From 326c9dc08ceac7613c187f3d3c3609731823f8bc Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Wed, 2 Aug 2023 18:05:53 +0200 Subject: [PATCH 01/97] Changes in maven poms to build and test the project using Spark 3.4.x and scala 2.12 --- dhp-common/pom.xml | 19 ++- dhp-pace-core/pom.xml | 12 -- dhp-workflows/dhp-dedup-openaire/pom.xml | 24 --- dhp-workflows/dhp-graph-provision/pom.xml | 14 +- .../oa/provision/utils/TemplateFactory.java | 3 - pom.xml | 145 ++++++++++++------ 6 files changed, 116 insertions(+), 101 deletions(-) diff --git a/dhp-common/pom.xml b/dhp-common/pom.xml index 6198bd81e..d64e7e7a0 100644 --- a/dhp-common/pom.xml +++ b/dhp-common/pom.xml @@ -62,16 +62,17 @@ + + edu.cmu + secondstring + + eu.dnetlib.dhp dhp-pace-core ${project.version} - - org.apache.hadoop - hadoop-common - com.github.sisyphsu dateparser @@ -118,10 +119,6 @@ net.sf.saxon Saxon-HE - - org.slf4j - jcl-over-slf4j - org.apache.cxf cxf-rt-transports-http @@ -129,6 +126,12 @@ eu.dnetlib cnr-rmi-api + + + log4j + log4j + + diff --git a/dhp-pace-core/pom.xml b/dhp-pace-core/pom.xml index fd7f44fc9..a6d2538f2 100644 --- a/dhp-pace-core/pom.xml +++ b/dhp-pace-core/pom.xml @@ -53,14 +53,6 @@ edu.cmu secondstring - - com.google.guava - guava - - - com.google.code.gson - gson - org.apache.commons commons-lang3 @@ -85,10 +77,6 @@ com.fasterxml.jackson.core jackson-databind - - org.apache.commons - commons-math3 - com.jayway.jsonpath json-path diff --git a/dhp-workflows/dhp-dedup-openaire/pom.xml b/dhp-workflows/dhp-dedup-openaire/pom.xml index a271efe8e..2d40f44da 100644 --- a/dhp-workflows/dhp-dedup-openaire/pom.xml +++ b/dhp-workflows/dhp-dedup-openaire/pom.xml @@ -54,24 +54,10 @@ dhp-pace-core ${project.version} - org.apache.commons commons-lang3 - - - org.scala-lang.modules - scala-java8-compat_${scala.binary.version} - 1.0.2 - - - - org.scala-lang.modules - scala-collection-compat_${scala.binary.version} - 2.11.0 - - org.apache.spark spark-core_${scala.binary.version} @@ -80,16 +66,10 @@ org.apache.spark spark-sql_${scala.binary.version} - org.apache.spark spark-graphx_${scala.binary.version} - - - com.arakelian - java-jq - dom4j dom4j @@ -102,10 +82,6 @@ com.fasterxml.jackson.core jackson-databind - - com.fasterxml.jackson.core - jackson-core - org.apache.httpcomponents httpclient diff --git a/dhp-workflows/dhp-graph-provision/pom.xml b/dhp-workflows/dhp-graph-provision/pom.xml index e62fcdf19..47b056614 100644 --- a/dhp-workflows/dhp-graph-provision/pom.xml +++ b/dhp-workflows/dhp-graph-provision/pom.xml @@ -59,12 +59,6 @@ com.jayway.jsonpath json-path - - - org.slf4j - slf4j-api - - dom4j @@ -160,6 +154,14 @@ org.apache.zookeeper zookeeper + + ant + org.apache.ant + + + antlr4-runtime + org.antlr + diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/TemplateFactory.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/TemplateFactory.java index 87c0261ac..7046b4cf0 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/TemplateFactory.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/TemplateFactory.java @@ -5,14 +5,11 @@ import static eu.dnetlib.dhp.oa.provision.utils.GraphMappingUtils.removePrefix; import static eu.dnetlib.dhp.oa.provision.utils.XmlSerializationUtils.escapeXml; import java.io.IOException; -import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; -import javax.swing.text.html.Option; - import org.apache.commons.lang3.StringUtils; import org.stringtemplate.v4.ST; diff --git a/pom.xml b/pom.xml index 3fd351c1d..fa4f16df3 100644 --- a/pom.xml +++ b/pom.xml @@ -204,10 +204,17 @@ test + + org.slf4j + slf4j-api + ${org.slf4j.version} + provided + + org.slf4j jcl-over-slf4j - 1.7.25 + ${org.slf4j.version} provided @@ -217,22 +224,29 @@ ${dhp.commons.lang.version} + + org.apache.commons + commons-beanutils + ${commons-beanutils.version} + + + commons-validator commons-validator - 1.7 + ${commons-validator.version} com.github.sisyphsu dateparser - 1.0.7 + ${dateparser.version} me.xuender unidecode - 0.0.7 + ${unidecode.version} @@ -245,13 +259,13 @@ commons-codec commons-codec - 1.9 + ${commons-codec.version} commons-io commons-io - 2.4 + ${commons-io.version} @@ -415,6 +429,7 @@ cxf-rt-transports-http 3.1.5 + javax.persistence javax.persistence-api @@ -504,16 +519,11 @@ commons-compress ${common.compress.version} - - org.apache.commons commons-csv ${common.csv.version} - - - org.apache.poi poi-ooxml @@ -568,14 +578,12 @@ provided - org.apache.commons commons-math3 3.6.1 - com.google.code.gson gson @@ -596,7 +604,7 @@ org.reflections reflections - 0.9.10 + ${reflections.version} @@ -610,6 +618,12 @@ icu4j 70.1 + + + org.javassist + javassist + ${javassist.version} + @@ -866,46 +880,62 @@ sftp://dnet-hadoop@static-web.d4science.org/dnet-hadoop UTF-8 UTF-8 - 3.6.0 1.8 1.8 - 2.22.2 - 2.0.1 - cdh5.9.2 - 2.6.0-${dhp.cdh.version} - 4.1.0-${dhp.cdh.version} - dhp-schemas - 3.6.0 - 2.4.0.cloudera2 - 2.9.6 - 3.5 - true - 11.0.2 + + 2.11.12 2.11 - 1.3.0 - 5.6.1 - 3.3.3 - 3.4.2 - [2.12,3.0) - [4.17.2] + + + 3.6.0 + 2.22.2 + 2.0.1 + 4.0.1 + + + dhp-schemas + + 4.1.2 + [2.6.1] + 1.20 + 1.8 + 1.8 + 1.9.4 + 1.9 + 3.2.1 + 2.4 + 1.1.3 + 1.7 + 1.0.7 + [3.17.1] + cdh5.9.2 + 3.5 + 11.0.2 + 2.6.0-${dhp.cdh.version} + 2.9.6 + 4.1.0-${dhp.cdh.version} + true + 2.4.0.cloudera2 [4.0.3] [6.0.5] [3.1.6] - [2.6.1] - 7.5.0 - 4.7.2 - 1.20 + 2.2.2 + 3.19.0-GA 3.5.3 4.13.0 - 1.8 - 4.1.2 - 1.8 + 5.6.1 + 3.3.3 + 3.4.2 + 4.7.2 4.5.3 - 4.0.1 - 2.2.2 - 1.1.3 - 3.2.1 + 1.7.25 + 0.9.10 + 1.3.0 + 7.5.0 + 3.6.0 + 0.0.7 + [2.12,3.0) @@ -915,21 +945,40 @@ 2.12 2.12.18 - + 1.3.0 + + 4.8.1 + + 1.22 + 1.8 + 1.10.0 + 1.9.4 + 1.15 + 3.2.2 + 2.11.0 + 1.1.3 + 1.7 + + 14.0.1 + 8.11.0 4.0.2 3.4.1 2.14.2 3.12.0 3.7.0-M11 - 4.8.1 - + 3.25.0-GA + 4.10.0 + 2.0.6 + 0.10.2 + + - \ No newline at end of file + From 2fa78f6071206415b08b00c20a97c6ae8441a0fe Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Thu, 7 Sep 2023 11:58:59 +0200 Subject: [PATCH 02/97] Changes requires to build and run tests with Java 17 --- .../WritePredefinedProjectPropertiesTest.java | 10 ++- .../java/eu/dnetlib/pace/util/UtilTest.java | 4 +- .../oa/dedup/graph/ConnectedComponent.java | 24 +++++--- .../doiboost/orcid/OrcidClientTest.java | 6 -- dhp-workflows/dhp-graph-provision/pom.xml | 61 ++++++++++++++++++- .../dhp/oa/provision/XmlIndexingJob.java | 10 +-- .../dnetlib/dhp/sparksolr/DHPSolrSupport.java | 12 ++++ .../dnetlib/dhp/sparksolr/DHPSolrSupport.java | 12 ++++ .../dhp-usage-raw-data-update/pom.xml | 12 +++- dhp-workflows/dhp-usage-stats-build/pom.xml | 18 +++++- pom.xml | 38 ++++++++++-- 11 files changed, 168 insertions(+), 39 deletions(-) create mode 100644 dhp-workflows/dhp-graph-provision/src/main/sparksolr-3/eu/dnetlib/dhp/sparksolr/DHPSolrSupport.java create mode 100644 dhp-workflows/dhp-graph-provision/src/main/sparksolr-4/eu/dnetlib/dhp/sparksolr/DHPSolrSupport.java diff --git a/dhp-build/dhp-build-properties-maven-plugin/src/test/java/eu/dnetlib/maven/plugin/properties/WritePredefinedProjectPropertiesTest.java b/dhp-build/dhp-build-properties-maven-plugin/src/test/java/eu/dnetlib/maven/plugin/properties/WritePredefinedProjectPropertiesTest.java index 84b962b4b..19e9377af 100644 --- a/dhp-build/dhp-build-properties-maven-plugin/src/test/java/eu/dnetlib/maven/plugin/properties/WritePredefinedProjectPropertiesTest.java +++ b/dhp-build/dhp-build-properties-maven-plugin/src/test/java/eu/dnetlib/maven/plugin/properties/WritePredefinedProjectPropertiesTest.java @@ -80,7 +80,15 @@ class WritePredefinedProjectPropertiesTest { mojo.outputFile = testFolder; // execute - Assertions.assertThrows(MojoExecutionException.class, () -> mojo.execute()); + try { + mojo.execute(); + Assertions.assertTrue(false); // not reached + } catch (Exception e) { + Assertions + .assertTrue( + MojoExecutionException.class.isAssignableFrom(e.getClass()) || + IllegalArgumentException.class.isAssignableFrom(e.getClass())); + } } @Test diff --git a/dhp-pace-core/src/test/java/eu/dnetlib/pace/util/UtilTest.java b/dhp-pace-core/src/test/java/eu/dnetlib/pace/util/UtilTest.java index 6056c342d..c5c5eaba7 100644 --- a/dhp-pace-core/src/test/java/eu/dnetlib/pace/util/UtilTest.java +++ b/dhp-pace-core/src/test/java/eu/dnetlib/pace/util/UtilTest.java @@ -10,7 +10,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import eu.dnetlib.pace.model.Person; -import jdk.nashorn.internal.ir.annotations.Ignore; public class UtilTest { @@ -21,8 +20,7 @@ public class UtilTest { params = new HashMap<>(); } - @Test - @Ignore + // @Test public void paceResolverTest() { PaceResolver paceResolver = new PaceResolver(); paceResolver.getComparator("keywordMatch", params); diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/graph/ConnectedComponent.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/graph/ConnectedComponent.java index 4a39a175d..4fc0a25e8 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/graph/ConnectedComponent.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/graph/ConnectedComponent.java @@ -3,6 +3,9 @@ package eu.dnetlib.dhp.oa.dedup.graph; import java.io.IOException; import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -16,14 +19,16 @@ import eu.dnetlib.pace.util.PaceException; public class ConnectedComponent implements Serializable { - private String ccId; - private Set ids; + private String ccId = ""; + private List ids = Collections.EMPTY_LIST; private static final String CONNECTED_COMPONENT_ID_PREFIX = "connect_comp"; - public ConnectedComponent(Set ids, final int cut) { - this.ids = ids; + public ConnectedComponent() { + } + public ConnectedComponent(Set ids, final int cut) { + this.ids = new ArrayList<>(ids); this.ccId = createDefaultID(); if (cut > 0 && ids.size() > cut) { @@ -31,14 +36,15 @@ public class ConnectedComponent implements Serializable { .stream() .filter(id -> !ccId.equalsIgnoreCase(id)) .limit(cut - 1) - .collect(Collectors.toSet()); + .distinct() + .collect(Collectors.toList()); // this.ids.add(ccId); ?? } } public ConnectedComponent(String ccId, Set ids) { this.ccId = ccId; - this.ids = ids; + this.ids = new ArrayList<>(ids); } public String createDefaultID() { @@ -82,12 +88,12 @@ public class ConnectedComponent implements Serializable { } } - public Set getIds() { + public List getIds() { return ids; } - public void setIds(Set ids) { - this.ids = ids; + public void setIds(List ids) { + this.ids =ids; } public String getCcId() { diff --git a/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/OrcidClientTest.java b/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/OrcidClientTest.java index 70bbd066a..8aebeda0b 100644 --- a/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/OrcidClientTest.java +++ b/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/OrcidClientTest.java @@ -30,7 +30,6 @@ import eu.dnetlib.dhp.common.collection.HttpClientParams; import eu.dnetlib.dhp.schema.orcid.AuthorData; import eu.dnetlib.doiboost.orcid.util.DownloadsReport; import eu.dnetlib.doiboost.orcid.util.MultiAttemptsHttpConnector; -import jdk.nashorn.internal.ir.annotations.Ignore; public class OrcidClientTest { final int REQ_LIMIT = 24; @@ -152,7 +151,6 @@ public class OrcidClientTest { } // @Test - @Ignore private void testModifiedDate() throws ParseException { testDate(toRetrieveDate); testDate(toNotRetrieveDate); @@ -332,7 +330,6 @@ public class OrcidClientTest { } @Test - @Ignore void testUpdatedRecord() throws Exception { final String base64CompressedRecord = IOUtils .toString(getClass().getResourceAsStream("0000-0001-7281-6306.compressed.base64")); @@ -341,7 +338,6 @@ public class OrcidClientTest { } @Test - @Ignore void testUpdatedWork() throws Exception { final String base64CompressedWork = "H4sIAAAAAAAAAM1XS2/jNhC+51cQOuxJsiXZSR03Vmq0G6Bo013E6R56oyXaZiOJWpKy4y783zvUg5Ksh5uiCJogisX5Zjj85sHx3f1rFKI94YKyeGE4I9tAJPZZQOPtwvj9+cGaGUhIHAc4ZDFZGEcijHvv6u7A+MtcPVCSSgsUQObYzuzaccBEguVuYYxt+LHgbwKP6a11M3WnY6UzrpB7KuiahlQeF0aSrkPqGwhcisWcxpLwGIcLYydlMh+PD4fDiHGfBvDcjmMxLhGlBglSH8vsIH0qGlLqBFRIGvvDWjWQ1iMJJ2CKBANqGlNqMbkj3IpxRPq1KkypFZFoDRHa0aRfq8JoNjhnfIAJJS6xPouiIQJyeYmGQzE+cO5cXqITcItBlKyASExD0a93jiwtvJDjYXDDAqBPHoH2wMmVWGNf8xyyaEBiSTeUDHHWBpd2Nmmc10yfbgHQrHCyIRxKjQwRUoFKPRwEnIgBnQJQVdGeQgJaCRN0OMnPkaUFVbD9WkpaIndQJowf+8EFoIpTErJjBFQOBavElFpfUxwC9ZcqvQErdQXhe+oPFF8BaObupYzVsYEOARzSoZBWmKqaBMHcV0Wf8oG0beIqD+Gdkz0lhyE3NajUW6fhQFSV9Nw/MCBYyofYa0EN7wrBz13eP+Y+J6obWgE8Pdd2JpYD94P77Ezmjj13b0bu5PqPu3EXumEnxEJaEVxSUIHammsra+53z44zt2/m1/bItaeVtQ6dhs3c4XytvW75IYUchMKvEHVUyqmnWBFAS0VJrqSvQde6vp251ux2NtFuKcVOi+oK9YY0M0Cn6o4J6WkvtEK2XJ1vfPGAZxSoK8lb+SxJBbLQx1CohOLndjJUywQWUFmqEi3G6Zaqf/7buOyYJd5IYpfmf0XipfP18pDR9cQCeEuJQI/Lx36bFbVnpBeL2UwmqQw7ApAvf4GeGGQdEbENgolui/wdpjHaYCmPCIPPAmGBIsxfoLUhyRCB0SeCakEBJRKBtfJ+UBbI15TG4PaGBAhWthx8DmFYtHZQujv1CWbLLdzmmUKmHEOWCe1/zdu78bn/+YH+hCOqOzcXfFwuP6OVT/P710crwqGXFrpNaM2GT3MXarw01i15TIi3pmtJXgtbTVGf3h6HKfF+wBAnPyTfdCChudlm5gZaoG//F9pPZsGQcqqbyZN5hBau5OoIJ3PPwjTKDuG4s5MZp2rMzF5PZoK34IT6PIFOPrk+mTiVO5aJH2C+JJRjE/06eoRfpJxa4VgyYaLlaJUv/EhCfATMU/76gEOfmehL/qbJNNHjaFna+CQYB8wvo9PpPFJ5MOrJ1Ix7USBZqBl7KRNOx1d3jex7SG6zuijqCMWRusBsncjZSrM2u82UJmqzpGhvUJN2t6caIM9QQgO9c0t40UROnWsJd2Rbs+nsxpna9u30ttNkjechmzHjEST+X5CkkuNY0GzQkzyFseAf7lSZuLwdh1xSXKvvQJ4g4abTYgPV7uMt3rskohlJmMa82kQkshtyBEIYqQ+YB8X3oRHg7iFKi/bZP+Ao+T6BJhIT/vNPi8ffZs+flk+r2v0WNroZiyWn6xRmadHqTJXsjLJczElAZX6TnJdoWTM1SI2gfutv3rjeBt5t06rVvNuWup29246tlvluO+u2/G92bK9DXheL6uFd/Q3EaRDZqBIAAA=="; final String work = ArgumentApplicationParser.decompressValue(base64CompressedWork); @@ -413,7 +409,6 @@ public class OrcidClientTest { } @Test - @Ignore void testDownloadedAuthor() throws Exception { final String base64CompressedWork = "H4sIAAAAAAAAAI2Yy26jMBSG932KiD0hIe1MiwiVZjGLkWbX2XRHsFOsgs3YJmnefszFFy4+mUhtVPz9P/gcH/vQ9PWrrjYXzAVh9Bjst7tgg2nBEKEfx+DP28/wOdgImVOUV4ziY3DDInjNHlKOC8ZRMnxtmlyWxyDaqU+ofg7h/uX7IYwfn+Ngo25ARUKoxJzm1TEopWySKLper1vGC4LU74+IikgTWoFRW+SyfyyfxCBag4iQhBawyoGMDjdqJrnECJAZRquYLDEPaV5jv8oyWlXj+qTiXZLGr7KMiQbnjAOR6IY1W7C6hgIwjGt6SKGfHsY13ajHYipLIcIyJ5Xw6+akdvjEtyt4wxEwM6+VGph5N2zYr2ENhQRhKsmZYChmS1j7nFs6VIBPOwImKhyfMVeFg6GAWEjrcoQ4FoBmBGwVXYhagGHDBIEX+ZzUDiqyn35VN6rJUpUJ4zc/PAI2T03FbrUKJZQszWjV3zavVOjvVfoE01qB+YUUQPGNwHTt3luxJjdqh1AxJFBKLWOrSeCcF13RtxxYtlPOPqH6m+MLwVfoMQ2kdae2ArLajc6fTxkI1nIoegs0yB426pMO+0fSw07xDKMu0XKSde5C2VvrlVMijRzFwqY7XTJI1QMLWcmEzMxtDdxfHiYSgTNJnYJ1K9y5k0tUrMgrnGGaRiuXxxuClulYUbr0nBvpkYLjvgTCGsuSoex3f1CEvRPHKI184NJKtKeaiO7cD5E61bJ4F+9DFd7d01u8Tw6H5BBvvz8f3q3nXLGIeJULGdaqeVBBRK7rS7h/fNvvk/gpedxt4923dxP7Fc3KtKuc1BhlkrfYmeN4dcmrhmbw60+HmWw2CKgbTuqc32CXKTTmeTWT6bDBjPsQ0DTpnchdaYO0ayQ2FyLIiVREqs25aU8VKYLRbK0BsyZuqvr1MU2Sm/rDdhe/2CRN6FU/b+oBVyj1zqRtC5F8kAumfTclsl+s7EoNQu64nfOaVLeezX60Z3XCULLi6GI2IZGTEeey7fec9lBAuXawIHKcpifE7GABHWfoxLVfpUNPBXoMbZWrHFsR3bPAk9J9i2sw9nW6AQT1mpk++7JhW+v44Hmt8PomJqfD13jRnvFOSxCKtu6qHoyBbQ7cMFo750UEfGaXm6bEeplXIXj2hvL6mA7tzvIwmM9pbJFBG834POZdLGi2gH2u9u0K9HMwn5PTioFWLufzmrS4oNuU9Pkt2rf/2jMs7fMdm2rQTTM+j+49AzToAVuXYA1mD2k0+XdE9vAP+JYR5NcQAAA="; final String work = ArgumentApplicationParser.decompressValue(base64CompressedWork); @@ -421,7 +416,6 @@ public class OrcidClientTest { } @Test - @Ignore void testDownloadedWork() throws Exception { final String base64CompressedWork = "H4sIAAAAAAAAANVa63LiOBb+z1Oo+LVbhbkGAlTCLE1Id9IhTQV6unr/CVvB2tiWR5Khmal5rX2BfbE9ki3b3Jzt6Y13h6pQSPrOXTo6knL10zffQxvCBWXBdbVVb1YRCWzm0GB9Xf28vLX6VSQkDhzssYBcV3dEVH8aVa62jL8M1RcKI2kBAYwNLnrtXrMPFCGW7nW10YSPBX8dq3XRb1swNGgomkaG3FBBV9SjcnddDaOVR+0qApUCMaSBJDzA3nXVlTIcNhrb7bbOuE0d+F43AtEwCENBnMjGUhtyjiSFGBqHCkkDu5gqB0rpSMgJsCJOAVmKMVRMuoRbAfbJeaoMY6h84q8gQi4Nz1NlmNQbnDNe4Ak1bLA28/0iB8TjBg1GMV5gdzxu0CGoxSBKlkMkpp44T3eINBxeyG5bKDABpJb7QF1guRpOsd/iOWRRhwSSPlNS5LNjsOHzHAXxmjlHmwBSr3DyTDgsNVLkkAxk6LDjcCIKaBJAtoo2FCagFTJBiyf5IdJwUAv2PJUaNUgXlgnju/PgBJDFKfTYzgdXFgXLYAzVLxH2wPWvrfQ9mKEVhG+oXbD4EsD+3H1txqaxgQwBPqRFIc0w2WoSBHNbLfqIF0zbfVymIbQ52VCyLVIzBRm6VeQVRFWNHuoHDASLeJH3jqDVUQXB5yrOH0ObE5UNLQe+R+1mu2U1u1Z7sGy2hq3esN2tt5oXf79qnELv8fGwkJYPmxSswD1uA6vVXrY7w+5g2G3WuxedjNsJmj2escJx33G/ZXsU5iAs/AyRR0WcjpRXBLglc0lM1BjP59bX1qw9Hn/+dH87/dy9vBikeinKkyzVHjoqJNWIk7QuE3KU6pES6O7MwsarJh44QW1KowcWOCxAC9tlzEPsGX3YrYGQICgS0JKzENach2bEoTYNyKEQzaJyQnzSqesKSaV3IhRx92L8tLAm7GerjbZUujSwlFnIobqKkTuth+Q4ED4Vqqypp5JyfK8ah5Ji0f8AZVSGT2TZVGXfBLw/liOyqdRpJqfyXr8ldyEZrehKkm8Jr/2hc3Qb7EVk9DfMJbU98pu3k+6aETXXBebCZpt23tBaBUfSZRxdo98eYmgNfRxrh3zAnldDM/37FvZ+IiWtoQfddgiaEGBIDGCG7btA7jgBP9svAK2h90l4yYqIGop5jgMHXA4J0NB9ksR+YTX0qFtfqACO01jGjDHFPx552AW2W0P3uvGROk4NLfTvCeNS8X9MaDg1rL9Qz6PYh7En3f4ZNmKS6nUfQYFmE6PYe05IYBqPFGaq5wHlYpaoDbYqxokVK+JBerz51z+BIzc+SfSdTHVrTiSYtZzGFNOdGrr5ohsLF2+NUguqppkDoua6/S6yXwAYu44pM+/HiZ1BwEDWMqYbC5fjZ+MEBwMjb4PRLdTFYWrUwiUhJH/H+G3pMl/7fjqJhTGwSwU5lnfLsVDmxIPvmRetbJeCOsvfaxWXbXWxLVziqNky51BLW1OP2JKzgNoASSa7Gk1WAfrLI9mirzBBIUD1r/W/AgrMla7CjEMOzYBJolo30/mnxd0SzadPt5+eZtMb9O7rEN1wNINgEA8Ha+IxNMdrHLCQRR4TFRCudnmB7m6GqD0YDCqW+lQqlfnndw93iw/TJ/RwN5k+TqZDNJkAQyUvUlWvktjrdgbQEeI1EapN8Grd7MOeYJlfajSxWVOMfcIhVQXgfcFsqhcceobVA/U3GjsbDCYrjVSKSz0wHo8Xym6dArRvvjsbAfUGouFr8s5lG9o72DVVSy1saDqMqlarWW+12r2GiIXXMzuAU6AQcLLqWf3mZRf6iOlsNQdda9BudhQnvNNdPWN8XA7BgU5G2k3pLADA75XD3BSnn3y+3M90SbZWGczkxiRVmfSaJrd0V8u0yG3CeYRyht7O07Ste45weuqNmhcpLO44woEPRq1eilLN/f3ntEqGPFfzi2PmudHTO3EOEKf60LdTyUeDr7KIIzKfTfqtdr896JxklQtbES/IQD7UyL+SZIJSXYhLHkHZ9oqEjPR1MRzWu550cDYdCeI9n+S4hzouUU76+UeCQJ0fjkKn0+v3m703i0Eh/z97BCDH/XAAziTIt4rH94j7s4dHbSY/HJ90e3qriBQL+MMxCGETs9j/QxiSQ5PaS63/QsZqdS8vOxdvtj7Oc//fL4dTI2LvDAfVA6erSDKe3+cPxw70j4c5HHZlfLT9iAEZYKjZkxOYKZxymJy659l/t+QZllC5bvVJrzShD5GN0/NkiaZyqNcJh0NrdngtTfp7wviaHB+SS1Ng7O+Sk3h5HodT4S8RyY78pUmGM6eEg1l8tVCa1KnvY/SgrzDKsxRLF46j+uahNKH3BE6lsIb1lUxpUhdS3WUE+u6nPP/qiyAsklumMhMz9SBNqeus0oQ+QXqwIa7m3qy87IhXnBLPI8kVXXlZMaASm5vAEqWuKYkvHMtbPdiPiIdm6dVmeVMZjX+lfnKDWmaRAT7ev6ctTfhEF3RoWnJeXlKfSXcHcsf69rk0wTd4Qx30RV9yl5et2Ipwqe/SS5MJXiU8vbIv2b/qZaC8PZ65AUwj9QJR3vx1mQ9b7VPy1FFebnSpWq7xi0qJuwA+fLYpL7rwJdLXobcSa97kM4Cl35f3YXmofp0+8R9gBc/XeXL9Vn38pH7mLTs27z9T8ky1n7ynlZ0I4le78rYzl6t/woG5krwQlpcRcLDD2UPkH5F73C9G5tFKfY0q/wa1TIHI0CgAAA=="; final String work = ArgumentApplicationParser.decompressValue(base64CompressedWork); diff --git a/dhp-workflows/dhp-graph-provision/pom.xml b/dhp-workflows/dhp-graph-provision/pom.xml index 47b056614..60c925227 100644 --- a/dhp-workflows/dhp-graph-provision/pom.xml +++ b/dhp-workflows/dhp-graph-provision/pom.xml @@ -18,7 +18,7 @@ scala-compile-first - initialize + process-resources add-source compile @@ -208,5 +208,64 @@ + + + scala-2.11 + + true + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.4.0 + + + generate-sources + + add-source + + + + src/main/sparksolr-3 + + + + + + + + + + + scala-2.12 + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.4.0 + + + generate-sources + + add-source + + + + src/main/sparksolr-4 + + + + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/XmlIndexingJob.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/XmlIndexingJob.java index cd401c6cb..220eb4f53 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/XmlIndexingJob.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/XmlIndexingJob.java @@ -27,12 +27,11 @@ import org.apache.spark.sql.SparkSession; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.lucidworks.spark.util.SolrSupport; - import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.oa.provision.model.SerializableSolrInputDocument; import eu.dnetlib.dhp.oa.provision.utils.ISLookupClient; import eu.dnetlib.dhp.oa.provision.utils.StreamingInputDocumentFactory; +import eu.dnetlib.dhp.sparksolr.DHPSolrSupport; import eu.dnetlib.dhp.utils.ISLookupClientFactory; import eu.dnetlib.dhp.utils.saxon.SaxonTransformerFactory; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; @@ -156,12 +155,7 @@ public class XmlIndexingJob { switch (outputFormat) { case SOLR: final String collection = ProvisionConstants.getCollectionName(format); - - // SparkSolr >= 4 - // com.lucidworks.spark.BatchSizeType bt = com.lucidworks.spark.BatchSizeType.NUM_DOCS; - // SolrSupport.indexDocs(zkHost, collection, batchSize, bt, docs.rdd()); - // SparkSolr < 4 - SolrSupport.indexDocs(zkHost, collection, batchSize, docs.rdd()); + DHPSolrSupport.indexDocs(zkHost, collection, batchSize, docs.rdd()); break; case HDFS: spark diff --git a/dhp-workflows/dhp-graph-provision/src/main/sparksolr-3/eu/dnetlib/dhp/sparksolr/DHPSolrSupport.java b/dhp-workflows/dhp-graph-provision/src/main/sparksolr-3/eu/dnetlib/dhp/sparksolr/DHPSolrSupport.java new file mode 100644 index 000000000..295f0f54d --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/main/sparksolr-3/eu/dnetlib/dhp/sparksolr/DHPSolrSupport.java @@ -0,0 +1,12 @@ +package eu.dnetlib.dhp.sparksolr; + +import com.lucidworks.spark.util.SolrSupport; +import org.apache.solr.common.SolrInputDocument; +import org.apache.spark.rdd.RDD; + +public class DHPSolrSupport { + + static public void indexDocs(String zkhost, String collection, int batchSize, RDD docs) { + SolrSupport.indexDocs(zkhost, collection, batchSize, docs); + } +} diff --git a/dhp-workflows/dhp-graph-provision/src/main/sparksolr-4/eu/dnetlib/dhp/sparksolr/DHPSolrSupport.java b/dhp-workflows/dhp-graph-provision/src/main/sparksolr-4/eu/dnetlib/dhp/sparksolr/DHPSolrSupport.java new file mode 100644 index 000000000..6b85176a3 --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/main/sparksolr-4/eu/dnetlib/dhp/sparksolr/DHPSolrSupport.java @@ -0,0 +1,12 @@ +package eu.dnetlib.dhp.sparksolr; + +import com.lucidworks.spark.util.SolrSupport; +import org.apache.solr.common.SolrInputDocument; +import org.apache.spark.rdd.RDD; + +public class DHPSolrSupport { + + static public void indexDocs(String zkhost, String collection, int batchSize, RDD docs) { + SolrSupport.indexDocs(zkhost, collection, batchSize, com.lucidworks.spark.BatchSizeType.NUM_DOCS, docs); + } +} diff --git a/dhp-workflows/dhp-usage-raw-data-update/pom.xml b/dhp-workflows/dhp-usage-raw-data-update/pom.xml index a9dbb09ae..8ce9826e2 100644 --- a/dhp-workflows/dhp-usage-raw-data-update/pom.xml +++ b/dhp-workflows/dhp-usage-raw-data-update/pom.xml @@ -39,8 +39,8 @@ UTF-8 UTF-8 - 0.13.1-cdh5.2.1 - 2.5.0-cdh5.2.1 + 1.1.0-cdh5.16.2 + 2.6.0-cdh5.16.2 @@ -72,7 +72,13 @@ org.apache.hadoop hadoop-common ${cdh.hadoop.version} - + + + jdk.tools + jdk.tools + + + eu.dnetlib.dhp dhp-common diff --git a/dhp-workflows/dhp-usage-stats-build/pom.xml b/dhp-workflows/dhp-usage-stats-build/pom.xml index 56aec73b7..4dd987f51 100644 --- a/dhp-workflows/dhp-usage-stats-build/pom.xml +++ b/dhp-workflows/dhp-usage-stats-build/pom.xml @@ -39,8 +39,8 @@ UTF-8 UTF-8 - 0.13.1-cdh5.2.1 - 2.5.0-cdh5.2.1 + 1.1.0-cdh5.16.2 + 2.6.0-cdh5.16.2 @@ -67,11 +67,23 @@ org.apache.hive hive-jdbc ${cdh.hive.version} - + + + jdk.tools + jdk.tools + + + org.apache.hadoop hadoop-common ${cdh.hadoop.version} + + + jdk.tools + jdk.tools + + eu.dnetlib.dhp diff --git a/pom.xml b/pom.xml index fa4f16df3..78dda8513 100644 --- a/pom.xml +++ b/pom.xml @@ -120,11 +120,18 @@ conjars conjars - https://conjars.wensel.net/repo/ + https://conjars.wensel.net/repo/ + + + org.projectlombok + lombok + 1.18.28 + provided + org.junit.jupiter junit-jupiter @@ -812,7 +819,7 @@ org.jacoco jacoco-maven-plugin - 0.7.9 + 0.8.10 **/schemas/* @@ -963,8 +970,8 @@ 14.0.1 8.11.0 - 4.0.2 - 3.4.1 + 4.0.4 + 3.4.2-SNAPSHOT 2.14.2 3.12.0 3.7.0-M11 @@ -977,8 +984,29 @@ 3.17.2-SNAPSHOT --> + - + + java17 + + 17 + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M4 + + + --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED + true + + + + + From 8c3e9a09d38fbb9d09d1a72d7bde2183c4a65967 Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Mon, 18 Sep 2023 12:51:18 +0200 Subject: [PATCH 03/97] added repository openaire-third-parties --- .../WritePredefinedProjectPropertiesTest.java | 2 +- .../eu/dnetlib/pace/util/DiffPatchMatch.java | 18 + .../oa/dedup/graph/ConnectedComponent.java | 2 +- pom.xml | 2047 +++++++++-------- 4 files changed, 1058 insertions(+), 1011 deletions(-) diff --git a/dhp-build/dhp-build-properties-maven-plugin/src/test/java/eu/dnetlib/maven/plugin/properties/WritePredefinedProjectPropertiesTest.java b/dhp-build/dhp-build-properties-maven-plugin/src/test/java/eu/dnetlib/maven/plugin/properties/WritePredefinedProjectPropertiesTest.java index 19e9377af..eddcd8867 100644 --- a/dhp-build/dhp-build-properties-maven-plugin/src/test/java/eu/dnetlib/maven/plugin/properties/WritePredefinedProjectPropertiesTest.java +++ b/dhp-build/dhp-build-properties-maven-plugin/src/test/java/eu/dnetlib/maven/plugin/properties/WritePredefinedProjectPropertiesTest.java @@ -88,7 +88,7 @@ class WritePredefinedProjectPropertiesTest { .assertTrue( MojoExecutionException.class.isAssignableFrom(e.getClass()) || IllegalArgumentException.class.isAssignableFrom(e.getClass())); - } + } } @Test diff --git a/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java b/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java index cfd9acd70..154bac62c 100644 --- a/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java +++ b/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java @@ -1,6 +1,24 @@ package eu.dnetlib.pace.util; +/* + * Diff Match and Patch + * Copyright 2018 The diff-match-patch Authors. + * https://github.com/google/diff-match-patch + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + /* * Diff Match and Patch * Copyright 2018 The diff-match-patch Authors. diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/graph/ConnectedComponent.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/graph/ConnectedComponent.java index 4fc0a25e8..f4b3c441a 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/graph/ConnectedComponent.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/graph/ConnectedComponent.java @@ -93,7 +93,7 @@ public class ConnectedComponent implements Serializable { } public void setIds(List ids) { - this.ids =ids; + this.ids = ids; } public String getCcId() { diff --git a/pom.xml b/pom.xml index 78dda8513..1480af2a6 100644 --- a/pom.xml +++ b/pom.xml @@ -1,1012 +1,1041 @@ - - - 4.0.0 - eu.dnetlib.dhp - dhp - 1.2.5-SNAPSHOT - pom - - - - GNU Affero General Public License v3.0 or later - https://spdx.org/licenses/AGPL-3.0-or-later.html#licenseText - repo - This program is free software: you can redistribute it and/or modify it under the terms of the - GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the - License, or (at your option) any later version. - - - - - dhp-build - dhp-pace-core - dhp-common - dhp-workflows - - - - Redmine - https://support.openaire.eu/projects/openaire - - - - jenkins - https://jenkins-dnet.d4science.org/ - - - - scm:git:gitea@code-repo.d4science.org:D-Net/dnet-hadoop.git - scm:git:gitea@code-repo.d4science.org:D-Net/dnet-hadoop.git - https://code-repo.d4science.org/D-Net/dnet-hadoop/ - HEAD - - - This module is the root descriptor for the dnet-hadoop project - - - - - - - dnet45-releases - D-Net 45 releases - https://maven.d4science.org/nexus/content/repositories/dnet45-releases - default - - false - - - true - - - - dnet45-snapshots - D-Net 45 snapshots - https://maven.d4science.org/nexus/content/repositories/dnet45-snapshots - default - - true - - - false - - - - dnet45-bootstrap-snapshot - D-Net 45 Bootstrap Snapshot - https://maven.d4science.org/nexus/content/repositories/dnet45-bootstrap-snapshot/ - - false - - - true - - default - - - dnet45-bootstrap-release - D-Net 45 Bootstrap Release - https://maven.d4science.org/nexus/content/repositories/dnet45-bootstrap-release/ - - true - - - false - - default - - - cloudera - Cloudera Repository - https://repository.cloudera.com/artifactory/cloudera-repos - - true - - - false - - - - dnet-deps - dnet-dependencies - https://maven.d4science.org/nexus/content/repositories/dnet-deps - default - - - maven-restlet - Restlet repository - https://maven.restlet.talend.com - - - conjars - conjars - https://conjars.wensel.net/repo/ - - - - - - - org.projectlombok - lombok - 1.18.28 - provided - - - org.junit.jupiter - junit-jupiter - ${junit-jupiter.version} - test - - - - org.mockito - mockito-core - ${mockito-core.version} - test - - - - org.mockito - mockito-junit-jupiter - ${mockito-core.version} - test - - - - - - - - eu.dnetlib.dhp - ${dhp-schemas.artifact} - ${dhp-schemas.version} - - - org.apache.hadoop - hadoop-hdfs - ${dhp.hadoop.version} - provided - - - org.apache.hadoop - hadoop-common - ${dhp.hadoop.version} - provided - - - org.apache.hadoop - hadoop-client - ${dhp.hadoop.version} - provided - - - org.apache.hadoop - hadoop-distcp - ${dhp.hadoop.version} - provided - - - org.apache.spark - spark-core_${scala.binary.version} - ${dhp.spark.version} - provided - - - org.apache.spark - spark-sql_${scala.binary.version} - ${dhp.spark.version} - provided - - - org.apache.spark - spark-graphx_${scala.binary.version} - ${dhp.spark.version} - provided - - - org.apache.spark - spark-hive_${scala.binary.version} - ${dhp.spark.version} - test - - - - org.slf4j - slf4j-api - ${org.slf4j.version} - provided - - - - org.slf4j - jcl-over-slf4j - ${org.slf4j.version} - provided - - - - org.apache.commons - commons-lang3 - ${dhp.commons.lang.version} - - - - org.apache.commons - commons-beanutils - ${commons-beanutils.version} - - - - - commons-validator - commons-validator - ${commons-validator.version} - - - - com.github.sisyphsu - dateparser - ${dateparser.version} - - - - me.xuender - unidecode - ${unidecode.version} - - - - com.google.guava - guava - ${dhp.guava.version} - - - - - commons-codec - commons-codec - ${commons-codec.version} - - - - commons-io - commons-io - ${commons-io.version} - - - - commons-cli - commons-cli - 1.2 - provided - - - - net.sf.saxon - Saxon-HE - 9.9.1-6 - - - - dom4j - dom4j - 1.6.1 - - - - xml-apis - xml-apis - 1.4.01 - - - - jaxen - jaxen - 1.1.6 - - - - com.mycila.xmltool - xmltool - 3.3 - - - - org.apache.solr - solr-solrj - ${solr.version} - - - * - * - - - - - com.lucidworks.spark - spark-solr - ${sparksolr.version} - - - * - * - - - - - org.apache.solr - solr-test-framework - ${solr.version} - test - - - io.dropwizard.metrics - metrics-core - 3.2.6 - test - - - - - org.apache.httpcomponents - httpclient - ${org.apache.httpcomponents.version} - - - org.apache.httpcomponents - httpmime - ${org.apache.httpcomponents.version} - - - org.noggit - noggit - 0.8 - - - org.apache.zookeeper - zookeeper - 3.4.11 - - - - net.schmizz - sshj - 0.10.0 - test - - - - com.fasterxml.jackson.core - jackson-core - ${dhp.jackson.version} - provided - - - - com.fasterxml.jackson.core - jackson-annotations - ${dhp.jackson.version} - provided - - - com.fasterxml.jackson.core - jackson-databind - ${dhp.jackson.version} - provided - - - - eu.dnetlib - dnet-actionmanager-common - ${dnet-actionmanager-common.version} - - - org.apache.hadoop - hadoop-common - - - - - eu.dnetlib - dnet-actionmanager-api - ${dnet-actionmanager-api.version} - - - eu.dnetlib - cnr-misc-utils - - - - - - eu.dnetlib - cnr-rmi-api - ${cnr-rmi-api.version} - - - - eu.dnetlib.dhp - dnet-openaire-broker-common - ${dnet-openaire-broker-common.version} - - - - org.apache.cxf - cxf-rt-transports-http - 3.1.5 - - - - javax.persistence - javax.persistence-api - 2.2 - provided - - - - com.jayway.jsonpath - json-path - 2.4.0 - - - com.arakelian - java-jq - 0.10.1 - - - edu.cmu - secondstring - 1.0.0 - - - org.mongodb - mongo-java-driver - ${mongodb.driver.version} - - - io.fares.junit.mongodb - mongodb-junit-test - 1.1.0 - - - org.postgresql - postgresql - 42.2.10 - - - - org.antlr - stringtemplate - 3.2.1 - - - - org.antlr - ST4 - 4.3.4 - - - - com.ximpleware - vtd-xml - ${vtd.version} - - - - org.elasticsearch - elasticsearch-hadoop - 7.6.0 - - - - - org.apache.oozie - oozie-client - ${dhp.oozie.version} - provided - - - - slf4j-simple - org.slf4j - - - - - - - com.squareup.okhttp3 - okhttp - ${okhttp.version} - - - - org.apache.commons - commons-compress - ${common.compress.version} - - - org.apache.commons - commons-csv - ${common.csv.version} - - - org.apache.poi - poi-ooxml - ${apache.poi.version} - - - - org.json - json - 20180813 - - - - org.json4s - json4s-jackson_${scala.binary.version} - ${json4s.version} - - - - com.github.victools - jsonschema-generator - ${jsonschemagenerator.version} - - - - org.apache.commons - commons-text - ${common.text.version} - - - - com.opencsv - opencsv - 5.5 - - - io.github.classgraph - classgraph - 4.8.71 - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-xml - ${jackson.version} - provided - - - com.fasterxml.jackson.module - jackson-module-jsonSchema - ${jackson.version} - provided - - - - org.apache.commons - commons-math3 - 3.6.1 - - - - com.google.code.gson - gson - ${google.gson.version} - - - - commons-collections - commons-collections - ${commons.collections.version} - - - commons-logging - commons-logging - ${commons.logging.version} - - - - org.reflections - reflections - ${reflections.version} - - - - org.scala-lang - scala-library - ${scala.version} - - - - com.ibm.icu - icu4j - 70.1 - - - - org.javassist - javassist - ${javassist.version} - - - - - - target - target/classes - ${project.artifactId}-${project.version} - target/test-classes - - - - org.apache.maven.plugins - maven-plugin-plugin - 3.3 - - - org.apache.maven.plugins - maven-project-info-reports-plugin - 3.0.0 - - - org.apache.maven.plugins - maven-site-plugin - 3.9.1 - - ${dhp.site.skip} - - - - - org.apache.maven.plugins - maven-compiler-plugin - ${maven.compiler.plugin.version} - - 1.8 - 1.8 - ${project.build.sourceEncoding} - - - - - org.apache.maven.plugins - maven-jar-plugin - 3.0.2 - - - - org.apache.maven.plugins - maven-source-plugin - 3.0.1 - - - attach-sources - verify - - jar-no-fork - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.0.0-M4 - - true - - - - org.apache.maven.plugins - maven-javadoc-plugin - 3.2.0 - - true - none - - - - org.apache.maven.plugins - maven-dependency-plugin - 3.6.0 - - - - net.revelc.code.formatter - formatter-maven-plugin - 2.11.0 - - - eu.dnetlib.dhp - dhp-code-style - ${project.version} - - - - - org.antipathy - mvn-scalafmt_${scala.binary.version} - 1.0.1640073709.733712b - - - eu.dnetlib.dhp - dhp-code-style - ${project.version} - - - - - - - - org.apache.maven.plugins - maven-site-plugin - - - org.apache.maven.plugins - maven-project-info-reports-plugin - - - net.revelc.code.formatter - formatter-maven-plugin - - - - format - - - eclipse/formatter_dnet.xml - - - - - - net.revelc.code - impsort-maven-plugin - 1.4.1 - - java.,javax.,org.,com. - java,* - - **/thrift/*.java - - - - - sort-imports - - sort - - - - - - org.antipathy - mvn-scalafmt_${scala.binary.version} - - https://code-repo.d4science.org/D-Net/dnet-hadoop/raw/branch/beta/dhp-build/dhp-code-style/src/main/resources/scalafmt/scalafmt.conf - false - false - - ${project.basedir}/src/main/scala - - - ${project.basedir}/src/test/scala - - false - false - : git rev-parse --abbrev-ref HEAD - false - - - - validate - - format - - - - - - org.apache.maven.plugins - maven-release-plugin - 2.5.3 - - - org.jacoco - jacoco-maven-plugin - 0.8.10 - - - **/schemas/* - **/com/cloudera/**/* - **/org/apache/avro/io/**/* - - - - - default-prepare-agent - - prepare-agent - - - - default-report - prepare-package - - report - - - - - - - - - - org.apache.maven.wagon - wagon-ssh - 2.10 - - - - - - dnet45-snapshots - DNet45 Snapshots - https://maven.d4science.org/nexus/content/repositories/dnet45-snapshots - default - - - dnet45-releases - https://maven.d4science.org/nexus/content/repositories/dnet45-releases - - - DHPSite - ${dhp.site.stage.path}/ - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - - true - none - - - - - - - sftp://dnet-hadoop@static-web.d4science.org/dnet-hadoop - UTF-8 - UTF-8 - 1.8 - 1.8 - - - 2.11.12 - 2.11 - - - 3.6.0 - 2.22.2 - 2.0.1 - 4.0.1 - - - dhp-schemas - - 4.1.2 - [2.6.1] - 1.20 - 1.8 - 1.8 - 1.9.4 - 1.9 - 3.2.1 - 2.4 - 1.1.3 - 1.7 - 1.0.7 - [3.17.1] - cdh5.9.2 - 3.5 - 11.0.2 - 2.6.0-${dhp.cdh.version} - 2.9.6 - 4.1.0-${dhp.cdh.version} - true - 2.4.0.cloudera2 - [4.0.3] - [6.0.5] - [3.1.6] - 2.2.2 - 3.19.0-GA - 3.5.3 - 4.13.0 - 5.6.1 - 3.3.3 - 3.4.2 - 4.7.2 - 4.5.3 - 1.7.25 - 0.9.10 - 1.3.0 - 7.5.0 - 3.6.0 - 0.0.7 - [2.12,3.0) - - - - - - scala-2.12 - - 2.12 - 2.12.18 - 1.3.0 - - - 4.8.1 - - - 1.22 - 1.8 - 1.10.0 - 1.9.4 - 1.15 - 3.2.2 - 2.11.0 - 1.1.3 - 1.7 - - 14.0.1 - 8.11.0 + + + 4.0.0 + eu.dnetlib.dhp + dhp + 1.2.5-SNAPSHOT + pom + + + + GNU Affero General Public License v3.0 or later + https://spdx.org/licenses/AGPL-3.0-or-later.html#licenseText + repo + This program is free software: you can redistribute it and/or modify it under the terms of the + GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + + + + + dhp-build + dhp-pace-core + dhp-common + dhp-workflows + + + + Redmine + https://support.openaire.eu/projects/openaire + + + + jenkins + https://jenkins-dnet.d4science.org/ + + + + scm:git:gitea@code-repo.d4science.org:D-Net/dnet-hadoop.git + scm:git:gitea@code-repo.d4science.org:D-Net/dnet-hadoop.git + https://code-repo.d4science.org/D-Net/dnet-hadoop/ + HEAD + + + This module is the root descriptor for the dnet-hadoop project + + + + + + + + Openaire-third-parties-snaphot + Openaire third parties Snapshot + https://maven.d4science.org/nexus/content/repositories/Openaire-third-parties-snaphot/ + + false + + + true + + + + + dnet45-releases + D-Net 45 releases + https://maven.d4science.org/nexus/content/repositories/dnet45-releases + default + + false + + + true + + + + dnet45-snapshots + D-Net 45 snapshots + https://maven.d4science.org/nexus/content/repositories/dnet45-snapshots + default + + true + + + false + + + + dnet45-bootstrap-snapshot + D-Net 45 Bootstrap Snapshot + https://maven.d4science.org/nexus/content/repositories/dnet45-bootstrap-snapshot/ + + false + + + true + + default + + + dnet45-bootstrap-release + D-Net 45 Bootstrap Release + https://maven.d4science.org/nexus/content/repositories/dnet45-bootstrap-release/ + + true + + + false + + default + + + cloudera + Cloudera Repository + https://repository.cloudera.com/artifactory/cloudera-repos + + true + + + false + + + + dnet-deps + dnet-dependencies + https://maven.d4science.org/nexus/content/repositories/dnet-deps + default + + + maven-restlet + Restlet repository + https://maven.restlet.talend.com + + + conjars + conjars + https://conjars.wensel.net/repo/ + + + + + + + org.projectlombok + lombok + 1.18.28 + provided + + + org.junit.jupiter + junit-jupiter + ${junit-jupiter.version} + test + + + + org.mockito + mockito-core + ${mockito-core.version} + test + + + + org.mockito + mockito-junit-jupiter + ${mockito-core.version} + test + + + + + + + + eu.dnetlib.dhp + ${dhp-schemas.artifact} + ${dhp-schemas.version} + + + org.apache.hadoop + hadoop-hdfs + ${dhp.hadoop.version} + provided + + + org.apache.hadoop + hadoop-common + ${dhp.hadoop.version} + provided + + + org.apache.hadoop + hadoop-client + ${dhp.hadoop.version} + provided + + + org.apache.hadoop + hadoop-distcp + ${dhp.hadoop.version} + provided + + + org.apache.spark + spark-core_${scala.binary.version} + ${dhp.spark.version} + provided + + + org.apache.spark + spark-sql_${scala.binary.version} + ${dhp.spark.version} + provided + + + org.apache.spark + spark-graphx_${scala.binary.version} + ${dhp.spark.version} + provided + + + org.apache.spark + spark-hive_${scala.binary.version} + ${dhp.spark.version} + test + + + + org.slf4j + slf4j-api + ${org.slf4j.version} + provided + + + + org.slf4j + jcl-over-slf4j + ${org.slf4j.version} + provided + + + + org.apache.commons + commons-lang3 + ${dhp.commons.lang.version} + + + + org.apache.commons + commons-beanutils + ${commons-beanutils.version} + + + + + commons-validator + commons-validator + ${commons-validator.version} + + + + com.github.sisyphsu + dateparser + ${dateparser.version} + + + + me.xuender + unidecode + ${unidecode.version} + + + + com.google.guava + guava + ${dhp.guava.version} + + + + + commons-codec + commons-codec + ${commons-codec.version} + + + + commons-io + commons-io + ${commons-io.version} + + + + commons-cli + commons-cli + 1.2 + provided + + + + net.sf.saxon + Saxon-HE + 9.9.1-6 + + + + dom4j + dom4j + 1.6.1 + + + + xml-apis + xml-apis + 1.4.01 + + + + jaxen + jaxen + 1.1.6 + + + + com.mycila.xmltool + xmltool + 3.3 + + + + org.apache.solr + solr-solrj + ${solr.version} + + + * + * + + + + + com.lucidworks.spark + spark-solr + ${sparksolr.version} + + + * + * + + + + + org.apache.solr + solr-test-framework + ${solr.version} + test + + + io.dropwizard.metrics + metrics-core + 3.2.6 + test + + + + + org.apache.httpcomponents + httpclient + ${org.apache.httpcomponents.version} + + + org.apache.httpcomponents + httpmime + ${org.apache.httpcomponents.version} + + + org.noggit + noggit + 0.8 + + + org.apache.zookeeper + zookeeper + 3.4.11 + + + + net.schmizz + sshj + 0.10.0 + test + + + + com.fasterxml.jackson.core + jackson-core + ${dhp.jackson.version} + provided + + + + com.fasterxml.jackson.core + jackson-annotations + ${dhp.jackson.version} + provided + + + com.fasterxml.jackson.core + jackson-databind + ${dhp.jackson.version} + provided + + + + eu.dnetlib + dnet-actionmanager-common + ${dnet-actionmanager-common.version} + + + org.apache.hadoop + hadoop-common + + + + + eu.dnetlib + dnet-actionmanager-api + ${dnet-actionmanager-api.version} + + + eu.dnetlib + cnr-misc-utils + + + + + + eu.dnetlib + cnr-rmi-api + ${cnr-rmi-api.version} + + + + eu.dnetlib.dhp + dnet-openaire-broker-common + ${dnet-openaire-broker-common.version} + + + + org.apache.cxf + cxf-rt-transports-http + 3.1.5 + + + + javax.persistence + javax.persistence-api + 2.2 + provided + + + + com.jayway.jsonpath + json-path + 2.4.0 + + + com.arakelian + java-jq + 0.10.1 + + + edu.cmu + secondstring + 1.0.0 + + + org.mongodb + mongo-java-driver + ${mongodb.driver.version} + + + io.fares.junit.mongodb + mongodb-junit-test + 1.1.0 + + + org.postgresql + postgresql + 42.2.10 + + + + org.antlr + stringtemplate + 3.2.1 + + + + org.antlr + ST4 + 4.3.4 + + + + com.ximpleware + vtd-xml + ${vtd.version} + + + + org.elasticsearch + elasticsearch-hadoop + 7.6.0 + + + + + org.apache.oozie + oozie-client + ${dhp.oozie.version} + provided + + + + slf4j-simple + org.slf4j + + + + + + + com.squareup.okhttp3 + okhttp + ${okhttp.version} + + + + org.apache.commons + commons-compress + ${common.compress.version} + + + org.apache.commons + commons-csv + ${common.csv.version} + + + org.apache.poi + poi-ooxml + ${apache.poi.version} + + + + org.json + json + 20180813 + + + + org.json4s + json4s-jackson_${scala.binary.version} + ${json4s.version} + + + + com.github.victools + jsonschema-generator + ${jsonschemagenerator.version} + + + + org.apache.commons + commons-text + ${common.text.version} + + + + com.opencsv + opencsv + 5.5 + + + io.github.classgraph + classgraph + 4.8.71 + + + + com.fasterxml.jackson.dataformat + jackson-dataformat-xml + ${jackson.version} + provided + + + com.fasterxml.jackson.module + jackson-module-jsonSchema + ${jackson.version} + provided + + + + org.apache.commons + commons-math3 + 3.6.1 + + + + com.google.code.gson + gson + ${google.gson.version} + + + + commons-collections + commons-collections + ${commons.collections.version} + + + commons-logging + commons-logging + ${commons.logging.version} + + + + org.reflections + reflections + ${reflections.version} + + + + org.scala-lang + scala-library + ${scala.version} + + + + com.ibm.icu + icu4j + 70.1 + + + + org.javassist + javassist + ${javassist.version} + + + + + + target + target/classes + ${project.artifactId}-${project.version} + target/test-classes + + + + org.apache.maven.plugins + maven-plugin-plugin + 3.3 + + + org.apache.maven.plugins + maven-project-info-reports-plugin + 3.0.0 + + + org.apache.maven.plugins + maven-site-plugin + 3.9.1 + + ${dhp.site.skip} + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${maven.compiler.plugin.version} + + 1.8 + 1.8 + ${project.build.sourceEncoding} + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.0.2 + + + + org.apache.maven.plugins + maven-source-plugin + 3.0.1 + + + attach-sources + verify + + jar-no-fork + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M4 + + true + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.2.0 + + true + none + + + + org.apache.maven.plugins + maven-dependency-plugin + 3.6.0 + + + + net.revelc.code.formatter + formatter-maven-plugin + 2.11.0 + + + eu.dnetlib.dhp + dhp-code-style + ${project.version} + + + + + org.antipathy + mvn-scalafmt_${scala.binary.version} + 1.0.1640073709.733712b + + + eu.dnetlib.dhp + dhp-code-style + ${project.version} + + + + + + + + org.apache.maven.plugins + maven-site-plugin + + + org.apache.maven.plugins + maven-project-info-reports-plugin + + + net.revelc.code.formatter + formatter-maven-plugin + + + + format + + + eclipse/formatter_dnet.xml + + + + + + net.revelc.code + impsort-maven-plugin + 1.4.1 + + java.,javax.,org.,com. + java,* + + **/thrift/*.java + + + + + sort-imports + + sort + + + + + + org.antipathy + mvn-scalafmt_${scala.binary.version} + + + https://code-repo.d4science.org/D-Net/dnet-hadoop/raw/branch/beta/dhp-build/dhp-code-style/src/main/resources/scalafmt/scalafmt.conf + + false + false + + ${project.basedir}/src/main/scala + + + ${project.basedir}/src/test/scala + + false + false + : git rev-parse --abbrev-ref HEAD + false + + + + validate + + format + + + + + + org.apache.maven.plugins + maven-release-plugin + 2.5.3 + + + org.jacoco + jacoco-maven-plugin + 0.8.10 + + + **/schemas/* + **/com/cloudera/**/* + **/org/apache/avro/io/**/* + + + + + default-prepare-agent + + prepare-agent + + + + default-report + prepare-package + + report + + + + + + + + + + org.apache.maven.wagon + wagon-ssh + 2.10 + + + + + + dnet45-snapshots + DNet45 Snapshots + https://maven.d4science.org/nexus/content/repositories/dnet45-snapshots + default + + + dnet45-releases + https://maven.d4science.org/nexus/content/repositories/dnet45-releases + + + DHPSite + ${dhp.site.stage.path}/ + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + true + none + + + + + + + sftp://dnet-hadoop@static-web.d4science.org/dnet-hadoop + UTF-8 + UTF-8 + 1.8 + 1.8 + + + 2.11.12 + 2.11 + + + 3.6.0 + 2.22.2 + 2.0.1 + 4.0.1 + + + dhp-schemas + + 4.1.2 + [2.6.1] + 1.20 + 1.8 + 1.8 + 1.9.4 + 1.9 + 3.2.1 + 2.4 + 1.1.3 + 1.7 + 1.0.7 + [3.17.1] + cdh5.9.2 + 3.5 + 11.0.2 + 2.6.0-${dhp.cdh.version} + 2.9.6 + 4.1.0-${dhp.cdh.version} + true + 2.4.0.cloudera2 + [4.0.3] + [6.0.5] + [3.1.6] + 2.2.2 + 3.19.0-GA + 3.5.3 + 4.13.0 + 5.6.1 + 3.3.3 + 3.4.2 + 4.7.2 + 4.5.3 + 1.7.25 + 0.9.10 + 1.3.0 + 7.5.0 + 3.6.0 + 0.0.7 + [2.12,3.0) + + + + + + scala-2.12 + + 2.12 + 2.12.18 + 1.3.0 + + + 4.8.1 + + + 1.22 + 1.8 + 1.10.0 + 1.9.4 + 1.15 + 3.2.2 + 2.11.0 + 1.1.3 + 1.7 + + 14.0.1 + 8.11.0 4.0.4 - 3.4.2-SNAPSHOT - 2.14.2 - 3.12.0 - 3.7.0-M11 - 3.25.0-GA - 4.10.0 - 2.0.6 - 0.10.2 - - - + 3.4.2.openaire-SNAPSHOT + 2.14.2 + 3.12.0 + 3.7.0-M11 + 3.25.0-GA + 4.10.0 + 2.0.6 + 0.10.2 + + + - - java17 - - 17 - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.0.0-M4 - - - --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED - true - - - - - - - + + java17 + + 17 + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M4 + + + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.lang.invoke=ALL-UNNAMED + --add-opens=java.base/java.lang.reflect=ALL-UNNAMED + --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED + --add-opens=java.base/sun.nio.cs=ALL-UNNAMED + --add-opens=java.base/sun.security.action=ALL-UNNAMED + --add-opens=java.base/sun.util.calendar=ALL-UNNAMED + + true + + + + + + + From 52495f2cd2f7acc4b5e8ba0e6bc9b99e27a3ade4 Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Mon, 18 Sep 2023 13:58:22 +0200 Subject: [PATCH 04/97] used javax.xml.stream.XMLEventReader instead of deprecated scala.xml.pull.XMLEventReader --- .../ebi/SparkCreateBaselineDataFrame.scala | 17 ++++++++-------- .../dnetlib/dhp/sx/bio/pubmed/PMParser.scala | 3 ++- .../dnetlib/dhp/sx/bio/BioScholixTest.scala | 20 +++++++++---------- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/ebi/SparkCreateBaselineDataFrame.scala b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/ebi/SparkCreateBaselineDataFrame.scala index 8ac8b00bf..6f5b7110f 100644 --- a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/ebi/SparkCreateBaselineDataFrame.scala +++ b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/ebi/SparkCreateBaselineDataFrame.scala @@ -3,7 +3,7 @@ package eu.dnetlib.dhp.sx.bio.ebi import eu.dnetlib.dhp.application.ArgumentApplicationParser import eu.dnetlib.dhp.collection.CollectionUtils import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup -import eu.dnetlib.dhp.schema.oaf.{Oaf, Result} +import eu.dnetlib.dhp.schema.oaf.Oaf import eu.dnetlib.dhp.sx.bio.pubmed._ import eu.dnetlib.dhp.utils.ISLookupClientFactory import org.apache.commons.io.IOUtils @@ -14,13 +14,13 @@ import org.apache.http.client.methods.HttpGet import org.apache.http.impl.client.HttpClientBuilder import org.apache.spark.SparkConf import org.apache.spark.rdd.RDD -import org.apache.spark.sql.expressions.Aggregator import org.apache.spark.sql._ +import org.apache.spark.sql.expressions.Aggregator import org.slf4j.{Logger, LoggerFactory} -import java.io.InputStream -import scala.io.Source -import scala.xml.pull.XMLEventReader +import java.io.{ByteArrayInputStream, InputStream} +import java.nio.charset.Charset +import javax.xml.stream.XMLInputFactory object SparkCreateBaselineDataFrame { @@ -83,7 +83,7 @@ object SparkCreateBaselineDataFrame { if (response.getStatusLine.getStatusCode > 400) { tries -= 1 } else - return IOUtils.toString(response.getEntity.getContent) + return IOUtils.toString(response.getEntity.getContent, Charset.defaultCharset()) } catch { case e: Throwable => println(s"Error on requesting ${r.getURI}") @@ -155,7 +155,7 @@ object SparkCreateBaselineDataFrame { IOUtils.toString( SparkEBILinksToOaf.getClass.getResourceAsStream( "/eu/dnetlib/dhp/sx/bio/ebi/baseline_to_oaf_params.json" - ) + ),Charset.defaultCharset() ) ) parser.parseArgument(args) @@ -194,10 +194,11 @@ object SparkCreateBaselineDataFrame { if (!"true".equalsIgnoreCase(skipUpdate)) { downloadBaseLineUpdate(s"$workingPath/baseline", hdfsServerUri) val k: RDD[(String, String)] = sc.wholeTextFiles(s"$workingPath/baseline", 2000) + val inputFactory = XMLInputFactory.newInstance val ds: Dataset[PMArticle] = spark.createDataset( k.filter(i => i._1.endsWith(".gz")) .flatMap(i => { - val xml = new XMLEventReader(Source.fromBytes(i._2.getBytes())) + val xml =inputFactory.createXMLEventReader(new ByteArrayInputStream(i._2.getBytes())) new PMParser(xml) }) ) diff --git a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/pubmed/PMParser.scala b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/pubmed/PMParser.scala index 9102c12c4..fb941a461 100644 --- a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/pubmed/PMParser.scala +++ b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/pubmed/PMParser.scala @@ -1,7 +1,8 @@ package eu.dnetlib.dhp.sx.bio.pubmed import scala.xml.MetaData -import scala.xml.pull.{EvElemEnd, EvElemStart, EvText, XMLEventReader} +import javax.xml.stream.XMLEventReader +import scala.xml.pull.{EvElemEnd, EvElemStart, EvText} /** @param xml */ diff --git a/dhp-workflows/dhp-aggregation/src/test/scala/eu/dnetlib/dhp/sx/bio/BioScholixTest.scala b/dhp-workflows/dhp-aggregation/src/test/scala/eu/dnetlib/dhp/sx/bio/BioScholixTest.scala index d1611300d..c4af14c40 100644 --- a/dhp-workflows/dhp-aggregation/src/test/scala/eu/dnetlib/dhp/sx/bio/BioScholixTest.scala +++ b/dhp-workflows/dhp-aggregation/src/test/scala/eu/dnetlib/dhp/sx/bio/BioScholixTest.scala @@ -16,6 +16,7 @@ import org.mockito.junit.jupiter.MockitoExtension import java.io.{BufferedReader, InputStream, InputStreamReader} import java.util.zip.GZIPInputStream +import javax.xml.stream.XMLInputFactory import scala.collection.JavaConverters._ import scala.collection.mutable.ListBuffer import scala.io.Source @@ -49,10 +50,8 @@ class BioScholixTest extends AbstractVocabularyTest { @Test def testEBIData() = { - val inputXML = Source - .fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/graph/bio/pubmed.xml")) - .mkString - val xml = new XMLEventReader(Source.fromBytes(inputXML.getBytes())) + val inputFactory = XMLInputFactory.newInstance + val xml = inputFactory.createXMLEventReader(getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/graph/bio/pubmed.xml")) new PMParser(xml).foreach(s => println(mapper.writeValueAsString(s))) } @@ -91,9 +90,10 @@ class BioScholixTest extends AbstractVocabularyTest { @Test def testParsingPubmedXML(): Unit = { - val xml = new XMLEventReader( - Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/graph/bio/pubmed.xml")) - ) + val inputFactory = XMLInputFactory.newInstance + + val xml = inputFactory.createXMLEventReader(getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/graph/bio/pubmed.xml")) + val parser = new PMParser(xml) parser.foreach(checkPMArticle) } @@ -156,9 +156,9 @@ class BioScholixTest extends AbstractVocabularyTest { @Test def testPubmedMapping(): Unit = { - val xml = new XMLEventReader( - Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/graph/bio/pubmed.xml")) - ) + val inputFactory = XMLInputFactory.newInstance + val xml = inputFactory.createXMLEventReader(getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/graph/bio/pubmed.xml")) + val parser = new PMParser(xml) val results = ListBuffer[Oaf]() parser.foreach(x => results += PubMedToOaf.convert(x, vocabularies)) From 613ec5ffceebb11740fc7ec29a406cbf7490ac14 Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Thu, 21 Sep 2023 14:23:37 +0200 Subject: [PATCH 05/97] Add profiles for different spark versions: spark-24, spark-34, spark-35 --- .../eu/dnetlib/dhp/common/PacePerson.java | 2 +- dhp-pace-core/pom.xml | 88 ++++++++++++++++- .../eu/dnetlib/pace/model/SparkModel.scala | 9 +- .../eu/dnetlib/pace/util/DiffPatchMatch.java | 1 - .../dnetlib/pace/util/SparkCompatUtils.scala | 12 +++ .../dnetlib/pace/util/SparkCompatUtils.scala | 12 +++ .../ebi/SparkCreateBaselineDataFrame.scala | 5 +- .../createunresolvedentities/ProduceTest.java | 5 +- .../opencitations/ReadCOCITest.java | 4 +- dhp-workflows/dhp-graph-provision/pom.xml | 42 ++++++++- .../dnetlib/dhp/swh/PrepareSWHActionsets.java | 3 +- pom.xml | 94 ++++++++++++++++--- 12 files changed, 245 insertions(+), 32 deletions(-) create mode 100644 dhp-pace-core/src/main/spark-2/eu/dnetlib/pace/util/SparkCompatUtils.scala create mode 100644 dhp-pace-core/src/main/spark-35/eu/dnetlib/pace/util/SparkCompatUtils.scala diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/PacePerson.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/PacePerson.java index fac9a7565..fbf586f8c 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/PacePerson.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/PacePerson.java @@ -38,7 +38,7 @@ public class PacePerson { PacePerson.class .getResourceAsStream( "/eu/dnetlib/dhp/common/name_particles.txt"))); - } catch (IOException e) { + } catch (Exception e) { throw new RuntimeException(e); } } diff --git a/dhp-pace-core/pom.xml b/dhp-pace-core/pom.xml index a6d2538f2..6449b7ec8 100644 --- a/dhp-pace-core/pom.xml +++ b/dhp-pace-core/pom.xml @@ -24,7 +24,7 @@ scala-compile-first - initialize + process-resources add-source compile @@ -95,4 +95,90 @@ + + + spark-24 + + true + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.4.0 + + + generate-sources + + add-source + + + + src/main/spark-2 + + + + + + + + + + + spark-34 + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.4.0 + + + generate-sources + + add-source + + + + src/main/spark-2 + + + + + + + + + + + spark-35 + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.4.0 + + + generate-sources + + add-source + + + + src/main/spark-35 + + + + + + + + + + diff --git a/dhp-pace-core/src/main/java/eu/dnetlib/pace/model/SparkModel.scala b/dhp-pace-core/src/main/java/eu/dnetlib/pace/model/SparkModel.scala index aa997c6e9..63322738f 100644 --- a/dhp-pace-core/src/main/java/eu/dnetlib/pace/model/SparkModel.scala +++ b/dhp-pace-core/src/main/java/eu/dnetlib/pace/model/SparkModel.scala @@ -2,11 +2,10 @@ package eu.dnetlib.pace.model import com.jayway.jsonpath.{Configuration, JsonPath} import eu.dnetlib.pace.config.{DedupConfig, Type} -import eu.dnetlib.pace.util.MapDocumentUtil -import org.apache.spark.sql.catalyst.encoders.RowEncoder +import eu.dnetlib.pace.util.{MapDocumentUtil, SparkCompatUtils} +import org.apache.spark.sql.{Dataset, Row} import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} -import org.apache.spark.sql.{Dataset, Row} import java.util.regex.Pattern import scala.collection.JavaConverters._ @@ -48,8 +47,8 @@ case class SparkModel(conf: DedupConfig) { val orderingFieldPosition: Int = schema.fieldIndex(orderingFieldName) - val parseJsonDataset: (Dataset[String] => Dataset[Row]) = df => { - df.map(r => rowFromJson(r))(RowEncoder(schema)) + val parseJsonDataset: (Dataset[String] => Dataset[Row]) = df => { + df.map(r => rowFromJson(r))(SparkCompatUtils.encoderFor(schema)) } def rowFromJson(json: String): Row = { diff --git a/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java b/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java index 154bac62c..ac37c5e5a 100644 --- a/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java +++ b/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java @@ -18,7 +18,6 @@ package eu.dnetlib.pace.util; * See the License for the specific language governing permissions and * limitations under the License. */ - /* * Diff Match and Patch * Copyright 2018 The diff-match-patch Authors. diff --git a/dhp-pace-core/src/main/spark-2/eu/dnetlib/pace/util/SparkCompatUtils.scala b/dhp-pace-core/src/main/spark-2/eu/dnetlib/pace/util/SparkCompatUtils.scala new file mode 100644 index 000000000..a426703d6 --- /dev/null +++ b/dhp-pace-core/src/main/spark-2/eu/dnetlib/pace/util/SparkCompatUtils.scala @@ -0,0 +1,12 @@ +package eu.dnetlib.pace.util + +import org.apache.spark.sql.Row +import org.apache.spark.sql.catalyst.encoders.{ExpressionEncoder, RowEncoder} +import org.apache.spark.sql.types.StructType + +object SparkCompatUtils { + + def encoderFor(schema: StructType): ExpressionEncoder[Row] = { + RowEncoder(schema) + } +} \ No newline at end of file diff --git a/dhp-pace-core/src/main/spark-35/eu/dnetlib/pace/util/SparkCompatUtils.scala b/dhp-pace-core/src/main/spark-35/eu/dnetlib/pace/util/SparkCompatUtils.scala new file mode 100644 index 000000000..cbc454ae2 --- /dev/null +++ b/dhp-pace-core/src/main/spark-35/eu/dnetlib/pace/util/SparkCompatUtils.scala @@ -0,0 +1,12 @@ +package eu.dnetlib.pace.util + +import org.apache.spark.sql.Row +import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder +import org.apache.spark.sql.types.StructType + +object SparkCompatUtils { + + def encoderFor(schema: StructType): ExpressionEncoder[Row] = { + ExpressionEncoder(schema) + } +} diff --git a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/ebi/SparkCreateBaselineDataFrame.scala b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/ebi/SparkCreateBaselineDataFrame.scala index 6f5b7110f..11d087583 100644 --- a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/ebi/SparkCreateBaselineDataFrame.scala +++ b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/ebi/SparkCreateBaselineDataFrame.scala @@ -155,7 +155,8 @@ object SparkCreateBaselineDataFrame { IOUtils.toString( SparkEBILinksToOaf.getClass.getResourceAsStream( "/eu/dnetlib/dhp/sx/bio/ebi/baseline_to_oaf_params.json" - ),Charset.defaultCharset() + ), + Charset.defaultCharset() ) ) parser.parseArgument(args) @@ -198,7 +199,7 @@ object SparkCreateBaselineDataFrame { val ds: Dataset[PMArticle] = spark.createDataset( k.filter(i => i._1.endsWith(".gz")) .flatMap(i => { - val xml =inputFactory.createXMLEventReader(new ByteArrayInputStream(i._2.getBytes())) + val xml = inputFactory.createXMLEventReader(new ByteArrayInputStream(i._2.getBytes())) new PMParser(xml) }) ) diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/ProduceTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/ProduceTest.java index ce116688a..0a4dfc00b 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/ProduceTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/ProduceTest.java @@ -15,10 +15,7 @@ import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SparkSession; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/opencitations/ReadCOCITest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/opencitations/ReadCOCITest.java index 3b416caf2..ebde0ed0c 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/opencitations/ReadCOCITest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/opencitations/ReadCOCITest.java @@ -119,7 +119,9 @@ public class ReadCOCITest { workingDir.toString() + "/COCI", "-outputPath", workingDir.toString() + "/COCI_json/", - "-inputFile", "input1;input2;input3;input4;input5" + "-inputFile", "input1;input2;input3;input4;input5", + "-format", + "COCI" }); final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/dhp-workflows/dhp-graph-provision/pom.xml b/dhp-workflows/dhp-graph-provision/pom.xml index 60c925227..4b4e6c1c4 100644 --- a/dhp-workflows/dhp-graph-provision/pom.xml +++ b/dhp-workflows/dhp-graph-provision/pom.xml @@ -162,6 +162,18 @@ antlr4-runtime org.antlr + + woodstox-core + com.fasterxml.woodstox + + + log4j + * + + + org.apache.logging.log4j + * + @@ -210,7 +222,7 @@ - scala-2.11 + spark-24 true @@ -240,7 +252,7 @@ - scala-2.12 + spark-34 @@ -266,6 +278,32 @@ + + spark-35 + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.4.0 + + + generate-sources + + add-source + + + + src/main/sparksolr-4 + + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/PrepareSWHActionsets.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/PrepareSWHActionsets.java index 2691d4b7e..230a077f7 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/PrepareSWHActionsets.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/PrepareSWHActionsets.java @@ -17,6 +17,7 @@ import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.api.java.function.FilterFunction; import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.sql.*; import org.apache.spark.sql.Dataset; @@ -117,7 +118,7 @@ public class PrepareSWHActionsets { .map( (MapFunction) t -> OBJECT_MAPPER.readValue(t, Software.class), Encoders.bean(Software.class)) - .filter(t -> t.getCodeRepositoryUrl() != null) + .filter((FilterFunction) t -> t.getCodeRepositoryUrl() != null) .select(col("id"), col("codeRepositoryUrl.value").as("repoUrl")); } diff --git a/pom.xml b/pom.xml index 1480af2a6..8c6bcd3d1 100644 --- a/pom.xml +++ b/pom.xml @@ -174,7 +174,7 @@ eu.dnetlib.dhp - ${dhp-schemas.artifact} + dhp-schemas ${dhp-schemas.version} @@ -233,6 +233,13 @@ provided + + org.slf4j + slf4j-log4j12 + ${org.slf4j.version} + provided + + org.slf4j jcl-over-slf4j @@ -240,6 +247,28 @@ provided + + org.apache.logging.log4j + log4j-slf4j2-impl + ${log4j.version} + + + org.apache.logging.log4j + log4j-api + ${log4j.version} + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + + + + org.apache.logging.log4j + log4j-1.2-api + ${log4j.version} + + org.apache.commons commons-lang3 @@ -381,7 +410,7 @@ org.apache.zookeeper zookeeper - 3.4.11 + ${zookeeper.version} @@ -713,6 +742,7 @@ 3.0.0-M4 true + false @@ -782,7 +812,7 @@ net.revelc.code impsort-maven-plugin - 1.4.1 + 1.6.2 java.,javax.,org.,com. java,* @@ -918,8 +948,6 @@ 4.0.1 - dhp-schemas - 4.1.2 [2.6.1] 1.20 @@ -932,7 +960,7 @@ 1.1.3 1.7 1.0.7 - [3.17.1] + 4.17.2 cdh5.9.2 3.5 11.0.2 @@ -945,6 +973,7 @@ [6.0.5] [3.1.6] 2.2.2 + 1.2.17 3.19.0-GA 3.5.3 4.13.0 @@ -960,12 +989,13 @@ 3.6.0 0.0.7 [2.12,3.0) + 3.4.6 - scala-2.12 + spark-34 2.12 2.12.18 @@ -988,25 +1018,60 @@ 14.0.1 8.11.0 4.0.4 - 3.4.2.openaire-SNAPSHOT + 3.4.2.openaire 2.14.2 3.12.0 + 2.19.0 3.7.0-M11 3.25.0-GA 4.10.0 2.0.6 0.10.2 - + 3.6.3 - java17 + spark-35 + + 2.12 + 2.12.18 + 1.3.0 + + + 4.8.1 + + + 1.23.0 + 1.8 + 1.10.0 + 1.9.4 + 1.16.0 + 3.2.2 + 2.13.0 + 1.1.3 + 1.7 + + 14.0.1 + 8.11.0 + 4.0.4 + 3.5.1.openaire-SNAPSHOT + 2.15.2 + 3.12.0 + 2.20.0 + 3.7.0-M11 + 3.25.0-GA + 4.10.0 + 2.0.7 + 0.10.2 + 3.6.3 + + + + + java11 - 17 + [11 @@ -1031,6 +1096,7 @@ --add-opens=java.base/sun.util.calendar=ALL-UNNAMED true + false From 342cb6189bbbfe44dfae772fc5308f419a6f8d09 Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Fri, 19 Apr 2024 12:13:26 +0200 Subject: [PATCH 06/97] fixed problem on changed signature on RowEncoder removed property dhp.schema.artifact --- dhp-common/pom.xml | 2 +- .../java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dhp-common/pom.xml b/dhp-common/pom.xml index d64e7e7a0..7c99ed527 100644 --- a/dhp-common/pom.xml +++ b/dhp-common/pom.xml @@ -164,7 +164,7 @@ eu.dnetlib.dhp - ${dhp-schemas.artifact} + dhp-schemas diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java index cb1c70059..bade4869f 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java @@ -3,6 +3,7 @@ package eu.dnetlib.dhp.oa.dedup; import static org.apache.spark.sql.functions.col; +import eu.dnetlib.pace.util.SparkCompatUtils; import org.apache.commons.io.IOUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.function.MapFunction; @@ -147,7 +148,7 @@ public class SparkPropagateRelation extends AbstractSparkAction { StructType idsSchema = StructType .fromDDL("`id` STRING, `dataInfo` STRUCT<`deletedbyinference`:BOOLEAN,`invisible`:BOOLEAN>"); - Dataset allIds = spark.emptyDataset(RowEncoder.apply(idsSchema)); + Dataset allIds = spark.emptyDataset(SparkCompatUtils.encoderFor(idsSchema)); for (EntityType entityType : ModelSupport.entityTypes.keySet()) { String entityPath = graphBasePath + '/' + entityType.name(); From 8dd9cf84e2ccbeec1db1d91193e813b35555bfb1 Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Fri, 19 Apr 2024 12:30:59 +0200 Subject: [PATCH 07/97] code formatted --- .../java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java index bade4869f..c5cb299b1 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java @@ -3,7 +3,6 @@ package eu.dnetlib.dhp.oa.dedup; import static org.apache.spark.sql.functions.col; -import eu.dnetlib.pace.util.SparkCompatUtils; import org.apache.commons.io.IOUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.function.MapFunction; @@ -23,6 +22,7 @@ import eu.dnetlib.dhp.schema.oaf.DataInfo; import eu.dnetlib.dhp.schema.oaf.Relation; import eu.dnetlib.dhp.utils.ISLookupClientFactory; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; +import eu.dnetlib.pace.util.SparkCompatUtils; import scala.Tuple2; import scala.Tuple3; From b72c3139e2b8439cf5b826638ff551d68f5da818 Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Fri, 19 Apr 2024 14:52:40 +0200 Subject: [PATCH 08/97] updated Ignore annotation that is deprecated to Disabled --- .../java/eu/dnetlib/pace/util/UtilTest.java | 5 +++-- .../doiboost/orcid/OrcidClientTest.java | 20 +++++++++---------- .../dnetlib/dhp/bulktag/SparkBulkTagJob.java | 10 ---------- 3 files changed, 13 insertions(+), 22 deletions(-) diff --git a/dhp-pace-core/src/test/java/eu/dnetlib/pace/util/UtilTest.java b/dhp-pace-core/src/test/java/eu/dnetlib/pace/util/UtilTest.java index 6056c342d..6d1300eae 100644 --- a/dhp-pace-core/src/test/java/eu/dnetlib/pace/util/UtilTest.java +++ b/dhp-pace-core/src/test/java/eu/dnetlib/pace/util/UtilTest.java @@ -7,10 +7,11 @@ import java.util.HashMap; import java.util.Map; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import eu.dnetlib.pace.model.Person; -import jdk.nashorn.internal.ir.annotations.Ignore; + public class UtilTest { @@ -22,7 +23,7 @@ public class UtilTest { } @Test - @Ignore + @Disabled public void paceResolverTest() { PaceResolver paceResolver = new PaceResolver(); paceResolver.getComparator("keywordMatch", params); diff --git a/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/OrcidClientTest.java b/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/OrcidClientTest.java index 70bbd066a..c4fd7dcba 100644 --- a/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/OrcidClientTest.java +++ b/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/OrcidClientTest.java @@ -30,7 +30,7 @@ import eu.dnetlib.dhp.common.collection.HttpClientParams; import eu.dnetlib.dhp.schema.orcid.AuthorData; import eu.dnetlib.doiboost.orcid.util.DownloadsReport; import eu.dnetlib.doiboost.orcid.util.MultiAttemptsHttpConnector; -import jdk.nashorn.internal.ir.annotations.Ignore; + public class OrcidClientTest { final int REQ_LIMIT = 24; @@ -48,7 +48,7 @@ public class OrcidClientTest { private static Path testPath; @BeforeAll - private static void setUp() throws IOException { + public static void setUp() throws IOException { testPath = Files.createTempDirectory(OrcidClientTest.class.getName()); System.out.println("using test path: " + testPath); } @@ -151,9 +151,9 @@ public class OrcidClientTest { System.out.println(valueDt.toString()); } - // @Test - @Ignore - private void testModifiedDate() throws ParseException { + @Test + @Disabled + public void testModifiedDate() throws ParseException { testDate(toRetrieveDate); testDate(toNotRetrieveDate); testDate(shortDate); @@ -226,7 +226,7 @@ public class OrcidClientTest { @Test @Disabled - private void slowedDownDownloadTest() throws Exception { + public void slowedDownDownloadTest() throws Exception { String orcid = "0000-0001-5496-1243"; String record = slowedDownDownload(orcid); String filename = "/tmp/downloaded_".concat(orcid).concat(".xml"); @@ -332,7 +332,7 @@ public class OrcidClientTest { } @Test - @Ignore + @Disabled void testUpdatedRecord() throws Exception { final String base64CompressedRecord = IOUtils .toString(getClass().getResourceAsStream("0000-0001-7281-6306.compressed.base64")); @@ -341,7 +341,7 @@ public class OrcidClientTest { } @Test - @Ignore + @Disabled void testUpdatedWork() throws Exception { final String base64CompressedWork = "H4sIAAAAAAAAAM1XS2/jNhC+51cQOuxJsiXZSR03Vmq0G6Bo013E6R56oyXaZiOJWpKy4y783zvUg5Ksh5uiCJogisX5Zjj85sHx3f1rFKI94YKyeGE4I9tAJPZZQOPtwvj9+cGaGUhIHAc4ZDFZGEcijHvv6u7A+MtcPVCSSgsUQObYzuzaccBEguVuYYxt+LHgbwKP6a11M3WnY6UzrpB7KuiahlQeF0aSrkPqGwhcisWcxpLwGIcLYydlMh+PD4fDiHGfBvDcjmMxLhGlBglSH8vsIH0qGlLqBFRIGvvDWjWQ1iMJJ2CKBANqGlNqMbkj3IpxRPq1KkypFZFoDRHa0aRfq8JoNjhnfIAJJS6xPouiIQJyeYmGQzE+cO5cXqITcItBlKyASExD0a93jiwtvJDjYXDDAqBPHoH2wMmVWGNf8xyyaEBiSTeUDHHWBpd2Nmmc10yfbgHQrHCyIRxKjQwRUoFKPRwEnIgBnQJQVdGeQgJaCRN0OMnPkaUFVbD9WkpaIndQJowf+8EFoIpTErJjBFQOBavElFpfUxwC9ZcqvQErdQXhe+oPFF8BaObupYzVsYEOARzSoZBWmKqaBMHcV0Wf8oG0beIqD+Gdkz0lhyE3NajUW6fhQFSV9Nw/MCBYyofYa0EN7wrBz13eP+Y+J6obWgE8Pdd2JpYD94P77Ezmjj13b0bu5PqPu3EXumEnxEJaEVxSUIHammsra+53z44zt2/m1/bItaeVtQ6dhs3c4XytvW75IYUchMKvEHVUyqmnWBFAS0VJrqSvQde6vp251ux2NtFuKcVOi+oK9YY0M0Cn6o4J6WkvtEK2XJ1vfPGAZxSoK8lb+SxJBbLQx1CohOLndjJUywQWUFmqEi3G6Zaqf/7buOyYJd5IYpfmf0XipfP18pDR9cQCeEuJQI/Lx36bFbVnpBeL2UwmqQw7ApAvf4GeGGQdEbENgolui/wdpjHaYCmPCIPPAmGBIsxfoLUhyRCB0SeCakEBJRKBtfJ+UBbI15TG4PaGBAhWthx8DmFYtHZQujv1CWbLLdzmmUKmHEOWCe1/zdu78bn/+YH+hCOqOzcXfFwuP6OVT/P710crwqGXFrpNaM2GT3MXarw01i15TIi3pmtJXgtbTVGf3h6HKfF+wBAnPyTfdCChudlm5gZaoG//F9pPZsGQcqqbyZN5hBau5OoIJ3PPwjTKDuG4s5MZp2rMzF5PZoK34IT6PIFOPrk+mTiVO5aJH2C+JJRjE/06eoRfpJxa4VgyYaLlaJUv/EhCfATMU/76gEOfmehL/qbJNNHjaFna+CQYB8wvo9PpPFJ5MOrJ1Ix7USBZqBl7KRNOx1d3jex7SG6zuijqCMWRusBsncjZSrM2u82UJmqzpGhvUJN2t6caIM9QQgO9c0t40UROnWsJd2Rbs+nsxpna9u30ttNkjechmzHjEST+X5CkkuNY0GzQkzyFseAf7lSZuLwdh1xSXKvvQJ4g4abTYgPV7uMt3rskohlJmMa82kQkshtyBEIYqQ+YB8X3oRHg7iFKi/bZP+Ao+T6BJhIT/vNPi8ffZs+flk+r2v0WNroZiyWn6xRmadHqTJXsjLJczElAZX6TnJdoWTM1SI2gfutv3rjeBt5t06rVvNuWup29246tlvluO+u2/G92bK9DXheL6uFd/Q3EaRDZqBIAAA=="; final String work = ArgumentApplicationParser.decompressValue(base64CompressedWork); @@ -413,7 +413,7 @@ public class OrcidClientTest { } @Test - @Ignore + @Disabled void testDownloadedAuthor() throws Exception { final String base64CompressedWork = "H4sIAAAAAAAAAI2Yy26jMBSG932KiD0hIe1MiwiVZjGLkWbX2XRHsFOsgs3YJmnefszFFy4+mUhtVPz9P/gcH/vQ9PWrrjYXzAVh9Bjst7tgg2nBEKEfx+DP28/wOdgImVOUV4ziY3DDInjNHlKOC8ZRMnxtmlyWxyDaqU+ofg7h/uX7IYwfn+Ngo25ARUKoxJzm1TEopWySKLper1vGC4LU74+IikgTWoFRW+SyfyyfxCBag4iQhBawyoGMDjdqJrnECJAZRquYLDEPaV5jv8oyWlXj+qTiXZLGr7KMiQbnjAOR6IY1W7C6hgIwjGt6SKGfHsY13ajHYipLIcIyJ5Xw6+akdvjEtyt4wxEwM6+VGph5N2zYr2ENhQRhKsmZYChmS1j7nFs6VIBPOwImKhyfMVeFg6GAWEjrcoQ4FoBmBGwVXYhagGHDBIEX+ZzUDiqyn35VN6rJUpUJ4zc/PAI2T03FbrUKJZQszWjV3zavVOjvVfoE01qB+YUUQPGNwHTt3luxJjdqh1AxJFBKLWOrSeCcF13RtxxYtlPOPqH6m+MLwVfoMQ2kdae2ArLajc6fTxkI1nIoegs0yB426pMO+0fSw07xDKMu0XKSde5C2VvrlVMijRzFwqY7XTJI1QMLWcmEzMxtDdxfHiYSgTNJnYJ1K9y5k0tUrMgrnGGaRiuXxxuClulYUbr0nBvpkYLjvgTCGsuSoex3f1CEvRPHKI184NJKtKeaiO7cD5E61bJ4F+9DFd7d01u8Tw6H5BBvvz8f3q3nXLGIeJULGdaqeVBBRK7rS7h/fNvvk/gpedxt4923dxP7Fc3KtKuc1BhlkrfYmeN4dcmrhmbw60+HmWw2CKgbTuqc32CXKTTmeTWT6bDBjPsQ0DTpnchdaYO0ayQ2FyLIiVREqs25aU8VKYLRbK0BsyZuqvr1MU2Sm/rDdhe/2CRN6FU/b+oBVyj1zqRtC5F8kAumfTclsl+s7EoNQu64nfOaVLeezX60Z3XCULLi6GI2IZGTEeey7fec9lBAuXawIHKcpifE7GABHWfoxLVfpUNPBXoMbZWrHFsR3bPAk9J9i2sw9nW6AQT1mpk++7JhW+v44Hmt8PomJqfD13jRnvFOSxCKtu6qHoyBbQ7cMFo750UEfGaXm6bEeplXIXj2hvL6mA7tzvIwmM9pbJFBG834POZdLGi2gH2u9u0K9HMwn5PTioFWLufzmrS4oNuU9Pkt2rf/2jMs7fMdm2rQTTM+j+49AzToAVuXYA1mD2k0+XdE9vAP+JYR5NcQAAA="; final String work = ArgumentApplicationParser.decompressValue(base64CompressedWork); @@ -421,7 +421,7 @@ public class OrcidClientTest { } @Test - @Ignore + @Disabled void testDownloadedWork() throws Exception { final String base64CompressedWork = "H4sIAAAAAAAAANVa63LiOBb+z1Oo+LVbhbkGAlTCLE1Id9IhTQV6unr/CVvB2tiWR5Khmal5rX2BfbE9ki3b3Jzt6Y13h6pQSPrOXTo6knL10zffQxvCBWXBdbVVb1YRCWzm0GB9Xf28vLX6VSQkDhzssYBcV3dEVH8aVa62jL8M1RcKI2kBAYwNLnrtXrMPFCGW7nW10YSPBX8dq3XRb1swNGgomkaG3FBBV9SjcnddDaOVR+0qApUCMaSBJDzA3nXVlTIcNhrb7bbOuE0d+F43AtEwCENBnMjGUhtyjiSFGBqHCkkDu5gqB0rpSMgJsCJOAVmKMVRMuoRbAfbJeaoMY6h84q8gQi4Nz1NlmNQbnDNe4Ak1bLA28/0iB8TjBg1GMV5gdzxu0CGoxSBKlkMkpp44T3eINBxeyG5bKDABpJb7QF1guRpOsd/iOWRRhwSSPlNS5LNjsOHzHAXxmjlHmwBSr3DyTDgsNVLkkAxk6LDjcCIKaBJAtoo2FCagFTJBiyf5IdJwUAv2PJUaNUgXlgnju/PgBJDFKfTYzgdXFgXLYAzVLxH2wPWvrfQ9mKEVhG+oXbD4EsD+3H1txqaxgQwBPqRFIc0w2WoSBHNbLfqIF0zbfVymIbQ52VCyLVIzBRm6VeQVRFWNHuoHDASLeJH3jqDVUQXB5yrOH0ObE5UNLQe+R+1mu2U1u1Z7sGy2hq3esN2tt5oXf79qnELv8fGwkJYPmxSswD1uA6vVXrY7w+5g2G3WuxedjNsJmj2escJx33G/ZXsU5iAs/AyRR0WcjpRXBLglc0lM1BjP59bX1qw9Hn/+dH87/dy9vBikeinKkyzVHjoqJNWIk7QuE3KU6pES6O7MwsarJh44QW1KowcWOCxAC9tlzEPsGX3YrYGQICgS0JKzENach2bEoTYNyKEQzaJyQnzSqesKSaV3IhRx92L8tLAm7GerjbZUujSwlFnIobqKkTuth+Q4ED4Vqqypp5JyfK8ah5Ji0f8AZVSGT2TZVGXfBLw/liOyqdRpJqfyXr8ldyEZrehKkm8Jr/2hc3Qb7EVk9DfMJbU98pu3k+6aETXXBebCZpt23tBaBUfSZRxdo98eYmgNfRxrh3zAnldDM/37FvZ+IiWtoQfddgiaEGBIDGCG7btA7jgBP9svAK2h90l4yYqIGop5jgMHXA4J0NB9ksR+YTX0qFtfqACO01jGjDHFPx552AW2W0P3uvGROk4NLfTvCeNS8X9MaDg1rL9Qz6PYh7En3f4ZNmKS6nUfQYFmE6PYe05IYBqPFGaq5wHlYpaoDbYqxokVK+JBerz51z+BIzc+SfSdTHVrTiSYtZzGFNOdGrr5ohsLF2+NUguqppkDoua6/S6yXwAYu44pM+/HiZ1BwEDWMqYbC5fjZ+MEBwMjb4PRLdTFYWrUwiUhJH/H+G3pMl/7fjqJhTGwSwU5lnfLsVDmxIPvmRetbJeCOsvfaxWXbXWxLVziqNky51BLW1OP2JKzgNoASSa7Gk1WAfrLI9mirzBBIUD1r/W/AgrMla7CjEMOzYBJolo30/mnxd0SzadPt5+eZtMb9O7rEN1wNINgEA8Ha+IxNMdrHLCQRR4TFRCudnmB7m6GqD0YDCqW+lQqlfnndw93iw/TJ/RwN5k+TqZDNJkAQyUvUlWvktjrdgbQEeI1EapN8Grd7MOeYJlfajSxWVOMfcIhVQXgfcFsqhcceobVA/U3GjsbDCYrjVSKSz0wHo8Xym6dArRvvjsbAfUGouFr8s5lG9o72DVVSy1saDqMqlarWW+12r2GiIXXMzuAU6AQcLLqWf3mZRf6iOlsNQdda9BudhQnvNNdPWN8XA7BgU5G2k3pLADA75XD3BSnn3y+3M90SbZWGczkxiRVmfSaJrd0V8u0yG3CeYRyht7O07Ste45weuqNmhcpLO44woEPRq1eilLN/f3ntEqGPFfzi2PmudHTO3EOEKf60LdTyUeDr7KIIzKfTfqtdr896JxklQtbES/IQD7UyL+SZIJSXYhLHkHZ9oqEjPR1MRzWu550cDYdCeI9n+S4hzouUU76+UeCQJ0fjkKn0+v3m703i0Eh/z97BCDH/XAAziTIt4rH94j7s4dHbSY/HJ90e3qriBQL+MMxCGETs9j/QxiSQ5PaS63/QsZqdS8vOxdvtj7Oc//fL4dTI2LvDAfVA6erSDKe3+cPxw70j4c5HHZlfLT9iAEZYKjZkxOYKZxymJy659l/t+QZllC5bvVJrzShD5GN0/NkiaZyqNcJh0NrdngtTfp7wviaHB+SS1Ng7O+Sk3h5HodT4S8RyY78pUmGM6eEg1l8tVCa1KnvY/SgrzDKsxRLF46j+uahNKH3BE6lsIb1lUxpUhdS3WUE+u6nPP/qiyAsklumMhMz9SBNqeus0oQ+QXqwIa7m3qy87IhXnBLPI8kVXXlZMaASm5vAEqWuKYkvHMtbPdiPiIdm6dVmeVMZjX+lfnKDWmaRAT7ev6ctTfhEF3RoWnJeXlKfSXcHcsf69rk0wTd4Qx30RV9yl5et2Ipwqe/SS5MJXiU8vbIv2b/qZaC8PZ65AUwj9QJR3vx1mQ9b7VPy1FFebnSpWq7xi0qJuwA+fLYpL7rwJdLXobcSa97kM4Cl35f3YXmofp0+8R9gBc/XeXL9Vn38pH7mLTs27z9T8ky1n7ynlZ0I4le78rYzl6t/woG5krwQlpcRcLDD2UPkH5F73C9G5tFKfY0q/wa1TIHI0CgAAA=="; final String work = ArgumentApplicationParser.decompressValue(base64CompressedWork); diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java index b09543da1..9e1acc7b2 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java @@ -4,21 +4,12 @@ package eu.dnetlib.dhp.bulktag; import static eu.dnetlib.dhp.PropagationConstant.removeOutputDir; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; -import java.io.BufferedOutputStream; -import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.*; import java.util.stream.Collectors; -import java.util.zip.GZIPOutputStream; -import org.apache.avro.TestAnnotation; -import org.apache.commons.compress.archivers.tar.TarArchiveEntry; -import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; -import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.spark.SparkConf; @@ -34,7 +25,6 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; -import com.sun.media.sound.ModelInstrumentComparator; import eu.dnetlib.dhp.api.Utils; import eu.dnetlib.dhp.api.model.CommunityEntityMap; From 3a027e97a70f278f8c8661ce7a2eabba9d39c6a3 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 19 Apr 2024 16:57:55 +0200 Subject: [PATCH 09/97] [graph indexing] sets spark memoryOverhead in the join operations to the same value used for the memory executor --- .../dhp/oa/provision/oozie_app/workflow.xml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml index f60c531e4..eb446ddd8 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml @@ -185,6 +185,7 @@ --executor-cores=${sparkExecutorCoresForJoining} --executor-memory=${sparkExecutorMemoryForJoining} --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.executor.memoryOverhead=${sparkExecutorMemoryForJoining} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -212,6 +213,7 @@ --executor-cores=${sparkExecutorCoresForJoining} --executor-memory=${sparkExecutorMemoryForJoining} --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.executor.memoryOverhead=${sparkExecutorMemoryForJoining} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -239,6 +241,7 @@ --executor-cores=${sparkExecutorCoresForJoining} --executor-memory=${sparkExecutorMemoryForJoining} --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.executor.memoryOverhead=${sparkExecutorMemoryForJoining} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -266,6 +269,7 @@ --executor-cores=${sparkExecutorCoresForJoining} --executor-memory=${sparkExecutorMemoryForJoining} --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.executor.memoryOverhead=${sparkExecutorMemoryForJoining} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -293,6 +297,7 @@ --executor-cores=${sparkExecutorCoresForJoining} --executor-memory=${sparkExecutorMemoryForJoining} --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.executor.memoryOverhead=${sparkExecutorMemoryForJoining} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -320,6 +325,7 @@ --executor-cores=${sparkExecutorCoresForJoining} --executor-memory=${sparkExecutorMemoryForJoining} --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.executor.memoryOverhead=${sparkExecutorMemoryForJoining} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -347,6 +353,7 @@ --executor-cores=${sparkExecutorCoresForJoining} --executor-memory=${sparkExecutorMemoryForJoining} --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.executor.memoryOverhead=${sparkExecutorMemoryForJoining} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -386,6 +393,7 @@ --executor-cores=${sparkExecutorCoresForJoining} --executor-memory=${sparkExecutorMemoryForJoining} --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.executor.memoryOverhead=${sparkExecutorMemoryForJoining} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -414,6 +422,7 @@ --executor-cores=${sparkExecutorCoresForJoining} --executor-memory=${sparkExecutorMemoryForJoining} --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.executor.memoryOverhead=${sparkExecutorMemoryForJoining} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -442,6 +451,7 @@ --executor-cores=${sparkExecutorCoresForJoining} --executor-memory=${sparkExecutorMemoryForJoining} --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.executor.memoryOverhead=${sparkExecutorMemoryForJoining} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -470,6 +480,7 @@ --executor-cores=${sparkExecutorCoresForJoining} --executor-memory=${sparkExecutorMemoryForJoining} --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.executor.memoryOverhead=${sparkExecutorMemoryForJoining} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -498,6 +509,7 @@ --executor-cores=${sparkExecutorCoresForJoining} --executor-memory=${sparkExecutorMemoryForJoining} --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.executor.memoryOverhead=${sparkExecutorMemoryForJoining} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -526,6 +538,7 @@ --executor-cores=${sparkExecutorCoresForJoining} --executor-memory=${sparkExecutorMemoryForJoining} --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.executor.memoryOverhead=${sparkExecutorMemoryForJoining} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -554,6 +567,7 @@ --executor-cores=${sparkExecutorCoresForJoining} --executor-memory=${sparkExecutorMemoryForJoining} --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.executor.memoryOverhead=${sparkExecutorMemoryForJoining} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} From e5879b68c731fe9a3444582b2b96e5301ff5c67f Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 19 Apr 2024 17:14:18 +0200 Subject: [PATCH 10/97] [transformative agreement] including reuslt-funder relations to the information imported from the TRs --- .../transformativeagreement/CreateActionSetSparkJob.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/transformativeagreement/CreateActionSetSparkJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/transformativeagreement/CreateActionSetSparkJob.java index e8443c033..9880d0260 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/transformativeagreement/CreateActionSetSparkJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/transformativeagreement/CreateActionSetSparkJob.java @@ -93,7 +93,7 @@ public class CreateActionSetSparkJob implements Serializable { .filter((FilterFunction) Objects::nonNull) .toJavaRDD() .map(p -> new AtomicAction(p.getClass(), p)); -//TODO relations in stand-by waiting to know if we need to create them or not In case we need just make a union before saving the sequence file + spark .read() .textFile(inputPath) @@ -108,6 +108,7 @@ public class CreateActionSetSparkJob implements Serializable { .filter((FilterFunction) r -> r != null) .toJavaRDD() .map(p -> new AtomicAction(p.getClass(), p)) + .union(relations) .mapToPair( aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()), new Text(OBJECT_MAPPER.writeValueAsString(aa)))) From 7a7e31315753e82ab2e99f3c0a4e26b0914fbe55 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 19 Apr 2024 17:30:25 +0200 Subject: [PATCH 11/97] updated schema version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8e6f16fe5..892382b9d 100644 --- a/pom.xml +++ b/pom.xml @@ -888,7 +888,7 @@ 3.3.3 3.4.2 [2.12,3.0) - [6.1.0] + [6.1.1] [4.0.3] [6.0.5] [3.1.6] From ab7f0855af09e25774fa62b4967b8ad9ed62ca10 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Sat, 20 Apr 2024 08:10:32 +0200 Subject: [PATCH 12/97] fixed query reading projects from the aggregator DB --- .../resources/eu/dnetlib/dhp/oa/graph/sql/queryProjects.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryProjects.sql b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryProjects.sql index fb584943f..7b664a696 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryProjects.sql +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryProjects.sql @@ -33,7 +33,7 @@ SELECT dc.officialname AS collectedfromname, p.contracttype || '@@@' || p.contracttypescheme AS contracttype, p.provenanceactionclass || '@@@' || p.provenanceactionscheme AS provenanceaction, - array_remove(array_agg(DISTINCT i.pid || '###' || i.issuertype || '@@@' || i.issuertype), NULL) AS pid,, + array_remove(array_agg(DISTINCT i.pid || '###' || i.issuertype || '@@@' || i.issuertype), NULL) AS pid, array_agg(DISTINCT s.name || '###' || s.semanticclass || '@@@' || s.semanticscheme) AS subjects, array_agg(DISTINCT fp.path) AS fundingtree From 0656ab28386f12f7482c369c002ce4dea2d4ec58 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Sat, 20 Apr 2024 08:10:58 +0200 Subject: [PATCH 13/97] code formatting --- .../eu/dnetlib/pace/tree/AuthorsMatch.java | 15 ++-- .../java/eu/dnetlib/pace/util/UtilTest.java | 1 - .../dhp/collection/mag/MagUtility.scala | 76 ++++++++++++------- .../mag/SparkMagOrganizationAS.scala | 26 ++++--- .../plugin/rest/OsfPreprintCollectorTest.java | 1 + .../dhp/collection/mag/MAGMappingTest.scala | 1 - .../dnetlib/dhp/oa/dedup/SparkDedupTest.java | 1 - .../doiboost/orcid/OrcidClientTest.java | 1 - .../raw/GenerateEntitiesApplication.java | 13 ++-- 9 files changed, 80 insertions(+), 55 deletions(-) diff --git a/dhp-pace-core/src/main/java/eu/dnetlib/pace/tree/AuthorsMatch.java b/dhp-pace-core/src/main/java/eu/dnetlib/pace/tree/AuthorsMatch.java index 0921d7a64..07080b09e 100644 --- a/dhp-pace-core/src/main/java/eu/dnetlib/pace/tree/AuthorsMatch.java +++ b/dhp-pace-core/src/main/java/eu/dnetlib/pace/tree/AuthorsMatch.java @@ -1,19 +1,20 @@ package eu.dnetlib.pace.tree; -import com.wcohen.ss.AbstractStringDistance; -import eu.dnetlib.pace.config.Config; -import eu.dnetlib.pace.model.Person; -import eu.dnetlib.pace.tree.support.AbstractListComparator; -import eu.dnetlib.pace.tree.support.ComparatorClass; -import eu.dnetlib.pace.util.AuthorMatchers; - import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.function.BiFunction; import java.util.stream.Collectors; +import com.wcohen.ss.AbstractStringDistance; + +import eu.dnetlib.pace.config.Config; +import eu.dnetlib.pace.model.Person; +import eu.dnetlib.pace.tree.support.AbstractListComparator; +import eu.dnetlib.pace.tree.support.ComparatorClass; +import eu.dnetlib.pace.util.AuthorMatchers; + @ComparatorClass("authorsMatch") public class AuthorsMatch extends AbstractListComparator { diff --git a/dhp-pace-core/src/test/java/eu/dnetlib/pace/util/UtilTest.java b/dhp-pace-core/src/test/java/eu/dnetlib/pace/util/UtilTest.java index 6d1300eae..93db552c1 100644 --- a/dhp-pace-core/src/test/java/eu/dnetlib/pace/util/UtilTest.java +++ b/dhp-pace-core/src/test/java/eu/dnetlib/pace/util/UtilTest.java @@ -12,7 +12,6 @@ import org.junit.jupiter.api.Test; import eu.dnetlib.pace.model.Person; - public class UtilTest { static Map params; diff --git a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/mag/MagUtility.scala b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/mag/MagUtility.scala index 48cb3276a..df22a6b84 100644 --- a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/mag/MagUtility.scala +++ b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/mag/MagUtility.scala @@ -5,7 +5,17 @@ import eu.dnetlib.dhp.schema.action.AtomicAction import eu.dnetlib.dhp.schema.common.ModelConstants import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils._ import eu.dnetlib.dhp.schema.oaf.utils.{OafMapperUtils, PidType} -import eu.dnetlib.dhp.schema.oaf.{Author, DataInfo, Instance, Journal, Organization, Publication, Relation, Result, Dataset => OafDataset} +import eu.dnetlib.dhp.schema.oaf.{ + Author, + DataInfo, + Instance, + Journal, + Organization, + Publication, + Relation, + Result, + Dataset => OafDataset +} import eu.dnetlib.dhp.utils.DHPUtils import org.apache.spark.sql.types._ import org.apache.spark.sql.{Dataset, Row, SparkSession} @@ -688,33 +698,45 @@ object MagUtility extends Serializable { o.setLegalname(field(r.getAs[String]("DisplayName"), null)) val gid = r.getAs[String]("GridId") if (gid != null) { - o.setPid(List( - structuredProperty(gid, qualifier( - PidType.GRID.toString, - PidType.GRID.toString, - ModelConstants.DNET_PID_TYPES, - ModelConstants.DNET_PID_TYPES - ), - null), - structuredProperty(r.getAs[Long]("AffiliationId").toString, qualifier( - PidType.mag_id.toString, - PidType.mag_id.toString, - ModelConstants.DNET_PID_TYPES, - ModelConstants.DNET_PID_TYPES - ), - null) - - ).asJava) + o.setPid( + List( + structuredProperty( + gid, + qualifier( + PidType.GRID.toString, + PidType.GRID.toString, + ModelConstants.DNET_PID_TYPES, + ModelConstants.DNET_PID_TYPES + ), + null + ), + structuredProperty( + r.getAs[Long]("AffiliationId").toString, + qualifier( + PidType.mag_id.toString, + PidType.mag_id.toString, + ModelConstants.DNET_PID_TYPES, + ModelConstants.DNET_PID_TYPES + ), + null + ) + ).asJava + ) } else { - o.setPid(List( - structuredProperty(r.getAs[Long]("AffiliationId").toString, qualifier( - PidType.mag_id.toString, - PidType.mag_id.toString, - ModelConstants.DNET_PID_TYPES, - ModelConstants.DNET_PID_TYPES - ), - null) - ).asJava) + o.setPid( + List( + structuredProperty( + r.getAs[Long]("AffiliationId").toString, + qualifier( + PidType.mag_id.toString, + PidType.mag_id.toString, + ModelConstants.DNET_PID_TYPES, + ModelConstants.DNET_PID_TYPES + ), + null + ) + ).asJava + ) } val c = r.getAs[String]("Iso3166Code") if (c != null) diff --git a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/mag/SparkMagOrganizationAS.scala b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/mag/SparkMagOrganizationAS.scala index 096a03f45..a9b0fac03 100644 --- a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/mag/SparkMagOrganizationAS.scala +++ b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/mag/SparkMagOrganizationAS.scala @@ -6,33 +6,37 @@ import eu.dnetlib.dhp.schema.oaf.Organization import org.apache.spark.sql.{Encoder, Encoders, SaveMode, SparkSession} import org.slf4j.{Logger, LoggerFactory} -class SparkMagOrganizationAS (propertyPath: String, args: Array[String], log: Logger) - extends AbstractScalaApplication(propertyPath, args, log: Logger) { +class SparkMagOrganizationAS(propertyPath: String, args: Array[String], log: Logger) + extends AbstractScalaApplication(propertyPath, args, log: Logger) { /** Here all the spark applications runs this method - * where the whole logic of the spark node is defined - */ + * where the whole logic of the spark node is defined + */ override def run(): Unit = { - val magBasePath:String = parser.get("magBasePath") + val magBasePath: String = parser.get("magBasePath") log.info(s"magBasePath is $magBasePath") - val outputPath:String = parser.get("outputPath") + val outputPath: String = parser.get("outputPath") log.info(s"outputPath is $outputPath") - generateAS(spark,magBasePath, outputPath) + generateAS(spark, magBasePath, outputPath) } - def generateAS(spark:SparkSession, magBasePath:String,outputPath:String ):Unit = { + def generateAS(spark: SparkSession, magBasePath: String, outputPath: String): Unit = { import spark.implicits._ - val organizations = MagUtility.loadMagEntity(spark,"Affiliations", magBasePath) - organizations.map(r => MagUtility.generateOrganization(r)).write.mode(SaveMode.Overwrite) + val organizations = MagUtility.loadMagEntity(spark, "Affiliations", magBasePath) + organizations + .map(r => MagUtility.generateOrganization(r)) + .write + .mode(SaveMode.Overwrite) .option("compression", "gzip") .text(outputPath) } } -object SparkMagOrganizationAS{ +object SparkMagOrganizationAS { val log: Logger = LoggerFactory.getLogger(SparkMagOrganizationAS.getClass) + def main(args: Array[String]): Unit = { new SparkMagOrganizationAS("/eu/dnetlib/dhp/collection/mag/create_organization_AS.json", args, log) .initialize() diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/OsfPreprintCollectorTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/OsfPreprintCollectorTest.java index 2f0263a0d..bc2d12661 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/OsfPreprintCollectorTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/OsfPreprintCollectorTest.java @@ -1,3 +1,4 @@ + package eu.dnetlib.dhp.collection.plugin.rest; import java.util.HashMap; diff --git a/dhp-workflows/dhp-aggregation/src/test/scala/eu/dnetlib/dhp/collection/mag/MAGMappingTest.scala b/dhp-workflows/dhp-aggregation/src/test/scala/eu/dnetlib/dhp/collection/mag/MAGMappingTest.scala index e41ccc41a..59b91d66b 100644 --- a/dhp-workflows/dhp-aggregation/src/test/scala/eu/dnetlib/dhp/collection/mag/MAGMappingTest.scala +++ b/dhp-workflows/dhp-aggregation/src/test/scala/eu/dnetlib/dhp/collection/mag/MAGMappingTest.scala @@ -10,7 +10,6 @@ class MAGMappingTest { val mapper = new ObjectMapper() - def mappingTest(): Unit = { val spark = SparkSession diff --git a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java index 2c96b7399..c80c98bb7 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java +++ b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java @@ -258,7 +258,6 @@ public class SparkDedupTest implements Serializable { assertEquals(115, sw_simrel.count()); } - // check if the first relation in the whitelist exists assertTrue( sw_simrel diff --git a/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/OrcidClientTest.java b/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/OrcidClientTest.java index c4fd7dcba..bc912b124 100644 --- a/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/OrcidClientTest.java +++ b/dhp-workflows/dhp-doiboost/src/test/java/eu/dnetlib/doiboost/orcid/OrcidClientTest.java @@ -31,7 +31,6 @@ import eu.dnetlib.dhp.schema.orcid.AuthorData; import eu.dnetlib.doiboost.orcid.util.DownloadsReport; import eu.dnetlib.doiboost.orcid.util.MultiAttemptsHttpConnector; - public class OrcidClientTest { final int REQ_LIMIT = 24; final int REQ_MAX_TEST = 100; diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java index 21d06692f..c3806c211 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplication.java @@ -130,12 +130,13 @@ public class GenerateEntitiesApplication extends AbstractMigrationApplication { switch (mode) { case claim: save( - inputRdd.keyBy(oaf -> ModelSupport.idFn().apply(oaf)) - .groupByKey() - .map(t -> MergeUtils.mergeGroup(t._1, t._2.iterator())), - //.mapToPair(oaf -> new Tuple2<>(ModelSupport.idFn().apply(oaf), oaf)) - //.reduceByKey(MergeUtils::merge) - //.map(Tuple2::_2), + inputRdd + .keyBy(oaf -> ModelSupport.idFn().apply(oaf)) + .groupByKey() + .map(t -> MergeUtils.mergeGroup(t._1, t._2.iterator())), + // .mapToPair(oaf -> new Tuple2<>(ModelSupport.idFn().apply(oaf), oaf)) + // .reduceByKey(MergeUtils::merge) + // .map(Tuple2::_2), targetPath); break; case graph: From 5857fd38c1bc8550e62fecfd5a4823b15e2202b3 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Sun, 21 Apr 2024 08:29:09 +0200 Subject: [PATCH 14/97] avoid NPEs in common Oaf merge utilities --- .../eu/dnetlib/dhp/oa/merge/AuthorMerger.java | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java index b413a0bb9..39725f6eb 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java @@ -10,6 +10,7 @@ import org.apache.commons.lang3.StringUtils; import com.wcohen.ss.JaroWinkler; import eu.dnetlib.dhp.schema.oaf.Author; +import eu.dnetlib.dhp.schema.oaf.Qualifier; import eu.dnetlib.dhp.schema.oaf.StructuredProperty; import eu.dnetlib.pace.model.Person; import scala.Tuple2; @@ -146,10 +147,20 @@ public class AuthorMerger { } public static String pidToComparableString(StructuredProperty pid) { - final String classid = pid.getQualifier().getClassid() != null ? pid.getQualifier().getClassid().toLowerCase() - : ""; - return (pid.getQualifier() != null ? classid : "") - + (pid.getValue() != null ? pid.getValue().toLowerCase() : ""); + final String classId = Optional + .ofNullable(pid) + .map( + p -> Optional + .ofNullable(p.getQualifier()) + .map(Qualifier::getClassid) + .map(String::toLowerCase) + .orElse("")) + .orElse(""); + return Optional + .ofNullable(pid) + .map(StructuredProperty::getValue) + .map(v -> String.join("|", v, classId)) + .orElse(""); } public static int countAuthorsPids(List authors) { From 776c898c4bd281ceca5824ec2ea4fa776df36cfb Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Mon, 22 Apr 2024 11:04:17 +0200 Subject: [PATCH 15/97] [WebCrawl] adding affiliation relations from web information --- .../CreateActionSetFromWebEntries.java | 272 +++++++++++++++++ .../actionmanager/webcrawl/as_parameters.json | 20 ++ .../dhp/actionmanager/webcrawl/job.properties | 2 + .../webcrawl/oozie_app/config-default.xml | 58 ++++ .../webcrawl/oozie_app/workflow.xml | 53 ++++ .../actionmanager/webcrawl/CreateASTest.java | 285 ++++++++++++++++++ .../dhp/actionmanager/webcrawl/part-00000 | 1 + .../dhp/actionmanager/webcrawl/part-00001 | 10 + .../dhp/actionmanager/webcrawl/part-00002 | 1 + 9 files changed, 702 insertions(+) create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateActionSetFromWebEntries.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/as_parameters.json create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/job.properties create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/oozie_app/config-default.xml create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/oozie_app/workflow.xml create mode 100644 dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateASTest.java create mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/webcrawl/part-00000 create mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/webcrawl/part-00001 create mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/webcrawl/part-00002 diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateActionSetFromWebEntries.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateActionSetFromWebEntries.java new file mode 100644 index 000000000..5a0be98d3 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateActionSetFromWebEntries.java @@ -0,0 +1,272 @@ + +package eu.dnetlib.dhp.actionmanager.webcrawl; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.io.Serializable; +import java.util.*; +import java.util.stream.Collectors; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.compress.GzipCodec; +import org.apache.hadoop.mapred.SequenceFileOutputFormat; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.FlatMapFunction; +import org.apache.spark.sql.*; +import org.apache.spark.sql.types.StructType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.schema.action.AtomicAction; +import eu.dnetlib.dhp.schema.common.ModelConstants; +import eu.dnetlib.dhp.schema.oaf.Relation; +import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; +import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils; +import eu.dnetlib.dhp.schema.oaf.utils.PidCleaner; +import eu.dnetlib.dhp.schema.oaf.utils.PidType; +import scala.Tuple2; + +/** + * @author miriam.baglioni + * @Date 18/04/24 + */ +public class CreateActionSetFromWebEntries implements Serializable { + private static final Logger log = LoggerFactory.getLogger(CreateActionSetFromWebEntries.class); + private static final String DOI_PREFIX = "50|doi_________::"; + + private static final String ROR_PREFIX = "20|ror_________::"; + + private static final String PMID_PREFIX = "50|pmid________::"; + + private static final String PMCID_PREFIX = "50|pmc_________::"; + private static final String WEB_CRAWL_ID = "10|openaire____::fb98a192f6a055ba495ef414c330834b"; + private static final String WEB_CRAWL_NAME = "Web Crawl"; + public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + public static void main(String[] args) throws Exception { + String jsonConfiguration = IOUtils + .toString( + CreateActionSetFromWebEntries.class + .getResourceAsStream( + "/eu/dnetlib/dhp/actionmanager/webcrawl/as_parameters.json")); + + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + parser.parseArgument(args); + + Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + final String inputPath = parser.get("sourcePath"); + log.info("inputPath: {}", inputPath); + + final String outputPath = parser.get("outputPath"); + log.info("outputPath: {}", outputPath); + + SparkConf conf = new SparkConf(); + + runWithSparkSession( + conf, + isSparkSessionManaged, + spark -> { + + createActionSet(spark, inputPath, outputPath + "actionSet"); + createPlainRelations(spark, inputPath, outputPath + "relations"); + }); + } + + private static void createPlainRelations(SparkSession spark, String inputPath, String outputPath) { + final Dataset dataset = readWebCrawl(spark, inputPath); + + dataset.flatMap((FlatMapFunction>) row -> { + List> ret = new ArrayList<>(); + + final String ror = row.getAs("ror"); + ret.addAll(createAffiliationRelationPairDOI(row.getAs("publication_year"), row.getAs("doi"), ror)); + ret.addAll(createAffiliationRelationPairPMID(row.getAs("publication_year"), row.getAs("pmid"), ror)); + ret.addAll(createAffiliationRelationPairPMCID(row.getAs("publication_year"), row.getAs("pmcid"), ror)); + + return ret + .iterator(); + }, Encoders.tuple(Encoders.STRING(), Encoders.bean(Relation.class))) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(outputPath); + } + + private static Collection> createAffiliationRelationPairPMCID( + String publication_year, String pmcid, String ror) { + if (pmcid == null) + return new ArrayList<>(); + + return createAffiliatioRelationPair("PMC" + pmcid, ror) + .stream() + .map(r -> new Tuple2(publication_year, r)) + .collect(Collectors.toList()); + } + + private static Collection> createAffiliationRelationPairPMID( + String publication_year, String pmid, String ror) { + if (pmid == null) + return new ArrayList<>(); + + return createAffiliatioRelationPair(pmid, ror) + .stream() + .map(r -> new Tuple2(publication_year, r)) + .collect(Collectors.toList()); + } + + private static Collection> createAffiliationRelationPairDOI( + String publication_year, String doi, String ror) { + if (doi == null) + return new ArrayList<>(); + + return createAffiliatioRelationPair(doi, ror) + .stream() + .map(r -> new Tuple2(publication_year, r)) + .collect(Collectors.toList()); + } + + public static void createActionSet(SparkSession spark, String inputPath, + String outputPath) { + + final Dataset dataset = readWebCrawl(spark, inputPath) + .filter("publication_year <= 2020 or country_code=='IE'") + .drop("publication_year"); + + dataset.flatMap((FlatMapFunction) row -> { + List ret = new ArrayList<>(); + final String ror = ROR_PREFIX + + IdentifierFactory.md5(PidCleaner.normalizePidValue("ROR", row.getAs("ror"))); + ret.addAll(createAffiliationRelationPairDOI(row.getAs("doi"), ror)); + ret.addAll(createAffiliationRelationPairPMID(row.getAs("pmid"), ror)); + ret.addAll(createAffiliationRelationPairPMCID(row.getAs("pmcid"), ror)); + + return ret + .iterator(); + }, Encoders.bean(Relation.class)) + .toJavaRDD() + .map(p -> new AtomicAction(p.getClass(), p)) + .mapToPair( + aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()), + new Text(OBJECT_MAPPER.writeValueAsString(aa)))) + .saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, GzipCodec.class); + + } + + private static Dataset readWebCrawl(SparkSession spark, String inputPath) { + StructType webInfo = StructType + .fromDDL( + "`id` STRING , `doi` STRING, `ids` STRUCT<`pmid` :STRING, `pmcid`: STRING >, `publication_year` STRING, " + + + "`authorships` ARRAY>>>"); + + return spark + .read() + .schema(webInfo) + .json(inputPath) + .withColumn( + "authors", functions + .explode( + functions.col("authorships"))) + .selectExpr("id", "doi", "ids", "publication_year", "authors.institutions as institutions") + .withColumn( + "institution", functions + .explode( + functions.col("institutions"))) + .selectExpr( + "id", "doi", "ids.pmcid as pmcid", "ids.pmid as pmid", "institution.ror as ror", + "institution.country_code as country_code", "publication_year") + // .where("country_code == 'IE'") + .distinct(); + + } + + private static List createAffiliationRelationPairPMCID(String pmcid, String ror) { + if (pmcid == null) + return new ArrayList<>(); + + return createAffiliatioRelationPair( + PMCID_PREFIX + + IdentifierFactory + .md5(PidCleaner.normalizePidValue(PidType.pmc.toString(), "PMC" + pmcid.substring(43))), + ror); + } + + private static List createAffiliationRelationPairPMID(String pmid, String ror) { + if (pmid == null) + return new ArrayList<>(); + + return createAffiliatioRelationPair( + PMID_PREFIX + + IdentifierFactory + .md5(PidCleaner.normalizePidValue(PidType.pmid.toString(), pmid.substring(33))), + ror); + } + + private static List createAffiliationRelationPairDOI(String doi, String ror) { + if (doi == null) + return new ArrayList<>(); + + return createAffiliatioRelationPair( + DOI_PREFIX + + IdentifierFactory + .md5(PidCleaner.normalizePidValue(PidType.doi.toString(), doi.substring(16))), + ror); + + } + + private static List createAffiliatioRelationPair(String resultId, String orgId) { + ArrayList newRelations = new ArrayList(); + + newRelations + .add( + OafMapperUtils + .getRelation( + orgId, resultId, ModelConstants.RESULT_ORGANIZATION, ModelConstants.AFFILIATION, + ModelConstants.IS_AUTHOR_INSTITUTION_OF, + Arrays + .asList( + OafMapperUtils.keyValue(WEB_CRAWL_ID, WEB_CRAWL_NAME)), + OafMapperUtils + .dataInfo( + false, null, false, false, + OafMapperUtils + .qualifier( + "sysimport:crasswalk:webcrawl", "Imported from Webcrawl", + ModelConstants.DNET_PROVENANCE_ACTIONS, ModelConstants.DNET_PROVENANCE_ACTIONS), + "0.9"), + null)); + + newRelations + .add( + OafMapperUtils + .getRelation( + resultId, orgId, ModelConstants.RESULT_ORGANIZATION, ModelConstants.AFFILIATION, + ModelConstants.HAS_AUTHOR_INSTITUTION, + Arrays + .asList( + OafMapperUtils.keyValue(WEB_CRAWL_ID, WEB_CRAWL_NAME)), + OafMapperUtils + .dataInfo( + false, null, false, false, + OafMapperUtils + .qualifier( + "sysimport:crasswalk:webcrawl", "Imported from Webcrawl", + ModelConstants.DNET_PROVENANCE_ACTIONS, ModelConstants.DNET_PROVENANCE_ACTIONS), + "0.9"), + null)); + + return newRelations; + + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/as_parameters.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/as_parameters.json new file mode 100644 index 000000000..3f056edf7 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/as_parameters.json @@ -0,0 +1,20 @@ +[ + { + "paramName": "sp", + "paramLongName": "sourcePath", + "paramDescription": "the zipped opencitations file", + "paramRequired": true + }, + { + "paramName": "op", + "paramLongName": "outputPath", + "paramDescription": "the working path", + "paramRequired": true + }, + { + "paramName": "issm", + "paramLongName": "isSparkSessionManaged", + "paramDescription": "the hdfs name node", + "paramRequired": false + } +] diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/job.properties b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/job.properties new file mode 100644 index 000000000..f616baea7 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/job.properties @@ -0,0 +1,2 @@ +sourcePath=/user/miriam.baglioni/openalex-snapshot/data/works/ +outputPath=/tmp/miriam/webcrawlComplete/ diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/oozie_app/config-default.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/oozie_app/config-default.xml new file mode 100644 index 000000000..a1755f329 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/oozie_app/config-default.xml @@ -0,0 +1,58 @@ + + + jobTracker + yarnRM + + + nameNode + hdfs://nameservice1 + + + oozie.use.system.libpath + true + + + oozie.action.sharelib.for.spark + spark2 + + + hive_metastore_uris + thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083 + + + spark2YarnHistoryServerAddress + http://iis-cdh5-test-gw.ocean.icm.edu.pl:18089 + + + spark2ExtraListeners + com.cloudera.spark.lineage.NavigatorAppListener + + + spark2SqlQueryExecutionListeners + com.cloudera.spark.lineage.NavigatorQueryListener + + + oozie.launcher.mapreduce.user.classpath.first + true + + + sparkExecutorNumber + 4 + + + spark2EventLogDir + /user/spark/spark2ApplicationHistory + + + sparkDriverMemory + 15G + + + sparkExecutorMemory + 6G + + + sparkExecutorCores + 1 + + \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/oozie_app/workflow.xml new file mode 100644 index 000000000..653a7d384 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/oozie_app/workflow.xml @@ -0,0 +1,53 @@ + + + + ${jobTracker} + ${nameNode} + + + mapreduce.job.queuename + ${queueName} + + + oozie.launcher.mapred.job.queue.name + ${oozieLauncherQueueName} + + + oozie.action.sharelib.for.spark + ${oozieActionShareLibForSpark2} + + + + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + yarn + cluster + Produces the AS for WC + eu.dnetlib.dhp.actionmanager.webcrawl.CreateActionSetFromWebEntries + dhp-aggregation-${projectVersion}.jar + + --executor-memory=${sparkExecutorMemory} + --executor-cores=${sparkExecutorCores} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} + + --sourcePath${sourcePath} + --outputPath${outputPath} + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateASTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateASTest.java new file mode 100644 index 000000000..a1cd69dcc --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateASTest.java @@ -0,0 +1,285 @@ +package eu.dnetlib.dhp.actionmanager.webcrawl; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; +import eu.dnetlib.dhp.schema.oaf.utils.PidCleaner; +import eu.dnetlib.dhp.schema.oaf.utils.PidType; +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.io.Text; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.schema.action.AtomicAction; +import eu.dnetlib.dhp.schema.oaf.Relation; + +/** + * @author miriam.baglioni + * @Date 22/04/24 + */ +public class CreateASTest { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static SparkSession spark; + + private static Path workingDir; + private static final Logger log = LoggerFactory + .getLogger(CreateASTest.class); + + @BeforeAll + public static void beforeAll() throws IOException { + workingDir = Files + .createTempDirectory(CreateASTest.class.getSimpleName()); + log.info("using work dir {}", workingDir); + + SparkConf conf = new SparkConf(); + conf.setAppName(CreateASTest.class.getSimpleName()); + + conf.setMaster("local[*]"); + conf.set("spark.driver.host", "localhost"); + conf.set("hive.metastore.local", "true"); + conf.set("spark.ui.enabled", "false"); + conf.set("spark.sql.warehouse.dir", workingDir.toString()); + conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString()); + + spark = SparkSession + .builder() + .appName(CreateASTest.class.getSimpleName()) + .config(conf) + .getOrCreate(); + } + + @AfterAll + public static void afterAll() throws IOException { + FileUtils.deleteDirectory(workingDir.toFile()); + spark.stop(); + } + @Test + void testNumberofRelations() throws Exception { + + String inputPath = getClass() + .getResource( + "/eu/dnetlib/dhp/actionmanager/webcrawl/") + .getPath(); + + CreateActionSetFromWebEntries + .main( + new String[] { + "-isSparkSessionManaged", + Boolean.FALSE.toString(), + "-sourcePath", + inputPath, + "-outputPath", + workingDir.toString() + "/actionSet1" + }); + + final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .sequenceFile(workingDir.toString() + "/actionSet1", Text.class, Text.class) + .map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class)) + .map(aa -> ((Relation) aa.getPayload())); + + Assertions.assertEquals(64, tmp.count()); + + } + @Test + void testRelations() throws Exception { + +// , "doi":"https://doi.org/10.1126/science.1188021", "pmid":"https://pubmed.ncbi.nlm.nih.gov/20448178", https://www.ncbi.nlm.nih.gov/pmc/articles/5100745 + + String inputPath = getClass() + .getResource( + "/eu/dnetlib/dhp/actionmanager/webcrawl/") + .getPath(); + + CreateActionSetFromWebEntries + .main( + new String[] { + "-isSparkSessionManaged", + Boolean.FALSE.toString(), + "-sourcePath", + inputPath, + "-outputPath", + workingDir.toString() + "/actionSet1" + }); + + final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .sequenceFile(workingDir.toString() + "/actionSet1", Text.class, Text.class) + .map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class)) + .map(aa -> ((Relation) aa.getPayload())); + + tmp.foreach(r -> System.out.println(new ObjectMapper().writeValueAsString(r))); + + Assertions + .assertEquals( + 1, tmp + .filter( + r -> r + .getSource() + .equals( + "50|doi_________::" + IdentifierFactory + .md5( + PidCleaner + .normalizePidValue(PidType.doi.toString(), "10.1098/rstl.1684.0023")))) + .count()); + + Assertions + .assertEquals( + 1, tmp + .filter( + r -> r + .getTarget() + .equals( + "50|doi_________::" + IdentifierFactory + .md5( + PidCleaner + .normalizePidValue(PidType.doi.toString(), "10.1098/rstl.1684.0023")))) + .count()); + + Assertions + .assertEquals( + 1, tmp + .filter( + r -> r + .getSource() + .equals( + "20|ror_________::" + IdentifierFactory + .md5( + PidCleaner + .normalizePidValue("ROR", "https://ror.org/03argrj65")))) + .count()); + + Assertions + .assertEquals( + 1, tmp + .filter( + r -> r + .getTarget() + .equals( + "20|ror_________::" + IdentifierFactory + .md5( + PidCleaner + .normalizePidValue("ROR", "https://ror.org/03argrj65")))) + .count()); + + Assertions + .assertEquals( + 5, tmp + .filter( + r -> r + .getSource() + .equals( + "20|ror_________::" + IdentifierFactory + .md5( + PidCleaner + .normalizePidValue("ROR", "https://ror.org/03265fv13")))) + .count()); + + Assertions + .assertEquals( + 5, tmp + .filter( + r -> r + .getTarget() + .equals( + "20|ror_________::" + IdentifierFactory + .md5( + PidCleaner + .normalizePidValue("ROR", "https://ror.org/03265fv13")))) + .count()); + + Assertions + .assertEquals( + 2, tmp + .filter( + r -> r + .getTarget() + .equals( + "20|ror_________::" + IdentifierFactory + .md5( + PidCleaner + .normalizePidValue(PidType.doi.toString(), "https://ror.org/03265fv13"))) + && r.getSource().startsWith("50|doi")) + .count()); + + Assertions + .assertEquals( + 2, tmp + .filter( + r -> r + .getTarget() + .equals( + "20|ror_________::" + IdentifierFactory + .md5( + PidCleaner + .normalizePidValue(PidType.doi.toString(), "https://ror.org/03265fv13"))) + && r.getSource().startsWith("50|pmid")) + .count()); + + Assertions + .assertEquals( + 1, tmp + .filter( + r -> r + .getTarget() + .equals( + "20|ror_________::" + IdentifierFactory + .md5( + PidCleaner + .normalizePidValue(PidType.doi.toString(), "https://ror.org/03265fv13"))) + && r.getSource().startsWith("50|pmc")) + .count()); + } + + @Test + void testRelationsCollectedFrom() throws Exception { + + String inputPath = getClass() + .getResource( + "/eu/dnetlib/dhp/actionmanager/webcrawl") + .getPath(); + + CreateActionSetFromWebEntries + .main( + new String[] { + "-isSparkSessionManaged", + Boolean.FALSE.toString(), + "-sourcePath", + inputPath, + "-outputPath", + workingDir.toString() + "/actionSet1" + }); + + final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .sequenceFile(workingDir.toString() + "/actionSet1", Text.class, Text.class) + .map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class)) + .map(aa -> ((Relation) aa.getPayload())); + + tmp.foreach(r -> { + assertEquals("Web Crawl", r.getCollectedfrom().get(0).getValue()); + assertEquals("10|openaire____::fb98a192f6a055ba495ef414c330834b", r.getCollectedfrom().get(0).getKey()); + }); + + } + + + +} diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/webcrawl/part-00000 b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/webcrawl/part-00000 new file mode 100644 index 000000000..a94baacb4 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/webcrawl/part-00000 @@ -0,0 +1 @@ +{"id": "https://openalex.org/W4214628335", "doi": "https://doi.org/10.1098/rstl.1684.0023", "title": "A letter from the learned and ingenious Mr. Will. Molyneux Secrctary to the Society of Dublin, to Will. Musgrave LL. B. Fellow of New Colledge, and Secretary to the Philosophical Society of Oxford, for advertisement of natural Knowledge; concerning Lough Neagh in Ireland, and its petrifying Qualitys", "display_name": "A letter from the learned and ingenious Mr. Will. Molyneux Secrctary to the Society of Dublin, to Will. Musgrave LL. B. Fellow of New Colledge, and Secretary to the Philosophical Society of Oxford, for advertisement of natural Knowledge; concerning Lough Neagh in Ireland, and its petrifying Qualitys", "publication_year": 1684, "publication_date": "1684-04-20", "ids": {"openalex": "https://openalex.org/W4214628335", "doi": "https://doi.org/10.1098/rstl.1684.0023"}, "language": "en", "primary_location": {"is_oa": true, "landing_page_url": "https://doi.org/10.1098/rstl.1684.0023", "pdf_url": "https://royalsocietypublishing.org/doi/pdf/10.1098/rstl.1684.0023", "source": {"id": "https://openalex.org/S4210177916", "display_name": "Philosophical transactions of the Royal Society of London", "issn_l": "0261-0523", "issn": ["0261-0523", "2053-9223"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310319787", "host_organization_name": "Royal Society", "host_organization_lineage": ["https://openalex.org/P4310319787"], "host_organization_lineage_names": ["Royal Society"], "type": "journal"}, "license": null, "version": "publishedVersion", "is_accepted": true, "is_published": true}, "type": "article", "type_crossref": "journal-article", "open_access": {"is_oa": true, "oa_status": "bronze", "oa_url": "https://royalsocietypublishing.org/doi/pdf/10.1098/rstl.1684.0023", "any_repository_has_fulltext": false}, "authorships": [{"author_position": "first", "author": {"id": "https://openalex.org/A5049974193", "display_name": "William Molyneux", "orcid": null}, "institutions": [{"id": "https://openalex.org/I2799941125", "display_name": "Royal Dublin Society", "ror": "https://ror.org/03argrj65", "country_code": "IE", "type": "nonprofit", "lineage": ["https://openalex.org/I2799941125"]}], "countries": ["IE"], "is_corresponding": true, "raw_author_name": "William Molyneux", "raw_affiliation_string": "Secretary to the Society of Dublin", "raw_affiliation_strings": ["Secretary to the Society of Dublin"]}], "countries_distinct_count": 1, "institutions_distinct_count": 1, "corresponding_author_ids": ["https://openalex.org/A5049974193"], "corresponding_institution_ids": ["https://openalex.org/I2799941125"], "apc_list": null, "apc_paid": null, "has_fulltext": false, "cited_by_count": 1, "cited_by_percentile_year": {"min": 80.9, "max": 87.8}, "biblio": {"volume": "14", "issue": "158", "first_page": "551", "last_page": "554"}, "is_retracted": false, "is_paratext": false, "keywords": [{"keyword": "lough neagh", "score": 0.5009}, {"keyword": "molyneux secrctary", "score": 0.4217}, {"keyword": "dublin", "score": 0.4181}, {"keyword": "ireland", "score": 0.407}, {"keyword": "natural knowledge;", "score": 0.3635}], "concepts": [{"id": "https://openalex.org/C127413603", "wikidata": "https://www.wikidata.org/wiki/Q11023", "display_name": "Engineering", "level": 0, "score": 0.39233524}, {"id": "https://openalex.org/C55587333", "wikidata": "https://www.wikidata.org/wiki/Q1133029", "display_name": "Engineering ethics", "level": 1, "score": 0.38055933}, {"id": "https://openalex.org/C95124753", "wikidata": "https://www.wikidata.org/wiki/Q875686", "display_name": "Environmental ethics", "level": 1, "score": 0.36769745}, {"id": "https://openalex.org/C42475967", "wikidata": "https://www.wikidata.org/wiki/Q194292", "display_name": "Operations research", "level": 1, "score": 0.34142447}, {"id": "https://openalex.org/C187736073", "wikidata": "https://www.wikidata.org/wiki/Q2920921", "display_name": "Management", "level": 1, "score": 0.32285866}, {"id": "https://openalex.org/C138885662", "wikidata": "https://www.wikidata.org/wiki/Q5891", "display_name": "Philosophy", "level": 0, "score": 0.31861395}, {"id": "https://openalex.org/C162324750", "wikidata": "https://www.wikidata.org/wiki/Q8134", "display_name": "Economics", "level": 0, "score": 0.09431246}], "mesh": [], "locations_count": 1, "locations": [{"is_oa": true, "landing_page_url": "https://doi.org/10.1098/rstl.1684.0023", "pdf_url": "https://royalsocietypublishing.org/doi/pdf/10.1098/rstl.1684.0023", "source": {"id": "https://openalex.org/S4210177916", "display_name": "Philosophical transactions of the Royal Society of London", "issn_l": "0261-0523", "issn": ["0261-0523", "2053-9223"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310319787", "host_organization_name": "Royal Society", "host_organization_lineage": ["https://openalex.org/P4310319787"], "host_organization_lineage_names": ["Royal Society"], "type": "journal"}, "license": null, "version": "publishedVersion", "is_accepted": true, "is_published": true}], "best_oa_location": {"is_oa": true, "landing_page_url": "https://doi.org/10.1098/rstl.1684.0023", "pdf_url": "https://royalsocietypublishing.org/doi/pdf/10.1098/rstl.1684.0023", "source": {"id": "https://openalex.org/S4210177916", "display_name": "Philosophical transactions of the Royal Society of London", "issn_l": "0261-0523", "issn": ["0261-0523", "2053-9223"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310319787", "host_organization_name": "Royal Society", "host_organization_lineage": ["https://openalex.org/P4310319787"], "host_organization_lineage_names": ["Royal Society"], "type": "journal"}, "license": null, "version": "publishedVersion", "is_accepted": true, "is_published": true}, "sustainable_development_goals": [{"id": "https://metadata.un.org/sdg/14", "display_name": "Life below water", "score": 0.35}, {"id": "https://metadata.un.org/sdg/8", "display_name": "Decent work and economic growth", "score": 0.12}], "grants": [], "referenced_works_count": 0, "referenced_works": [], "related_works": ["https://openalex.org/W2899084033", "https://openalex.org/W1982082555", "https://openalex.org/W2280699036", "https://openalex.org/W3009813477", "https://openalex.org/W2509006912", "https://openalex.org/W1986737539", "https://openalex.org/W2351005416", "https://openalex.org/W2393830843", "https://openalex.org/W2388387213", "https://openalex.org/W1791744077"], "ngrams_url": "https://api.openalex.org/works/W4214628335/ngrams", "abstract_inverted_index": {"Sir,": [0], "In": [1], "Answer": [2], "to": [3], "the": [4], "Oxford": [5], "Society's": [6], "Query": [7], "concerning": [8], "our": [9], "Lough": [10], "Neagh": [11], "and": [12], "Its": [13], "Petrifying": [14], "Qualitys,": [15], "I": [16], "make": [17], "this": [18], "return.": [19]}, "cited_by_api_url": "https://api.openalex.org/works?filter=cites:W4214628335", "counts_by_year": [], "updated_date": "2023-11-10T17:20:33.930548", "created_date": "2022-03-02"} \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/webcrawl/part-00001 b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/webcrawl/part-00001 new file mode 100644 index 000000000..ad39c76d8 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/webcrawl/part-00001 @@ -0,0 +1,10 @@ +{"id": "https://openalex.org/W2124362779", "doi": "https://doi.org/10.1126/science.1188021", "title": "A Draft Sequence of the Neandertal Genome", "display_name": "A Draft Sequence of the Neandertal Genome", "publication_year": 2010, "publication_date": "2010-05-07", "ids": {"openalex": "https://openalex.org/W2124362779", "doi": "https://doi.org/10.1126/science.1188021", "mag": "2124362779", "pmid": "https://pubmed.ncbi.nlm.nih.gov/20448178", "pmcid": "https://www.ncbi.nlm.nih.gov/pmc/articles/5100745"}, "language": "en", "primary_location": {"is_oa": true, "landing_page_url": "https://doi.org/10.1126/science.1188021", "pdf_url": null, "source": {"id": "https://openalex.org/S3880285", "display_name": "Science", "issn_l": "0036-8075", "issn": ["0036-8075", "1095-9203"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310315823", "host_organization_name": "American Association for the Advancement of Science", "host_organization_lineage": ["https://openalex.org/P4310315823"], "host_organization_lineage_names": ["American Association for the Advancement of Science"], "type": "journal"}, "license": null, "version": "publishedVersion", "is_accepted": true, "is_published": true}, "type": "article", "type_crossref": "journal-article", "open_access": {"is_oa": true, "oa_status": "bronze", "oa_url": "https://doi.org/10.1126/science.1188021", "any_repository_has_fulltext": true}, "authorships": [{"author_position": "first", "author": {"id": "https://openalex.org/A5038814932", "display_name": "Edward Green", "orcid": "https://orcid.org/0000-0003-0516-5827"}, "institutions": [{"id": "https://openalex.org/I4210118560", "display_name": "Max Planck Institute for Evolutionary Anthropology", "ror": "https://ror.org/02a33b393", "country_code": "DE", "type": "facility", "lineage": ["https://openalex.org/I149899117", "https://openalex.org/I4210118560"]}], "countries": ["DE"], "is_corresponding": true, "raw_author_name": "Richard E. Green", "raw_affiliation_string": "Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany.", "raw_affiliation_strings": ["Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5091317187", "display_name": "Johannes Krause", "orcid": "https://orcid.org/0000-0001-9144-3920"}, "institutions": [{"id": "https://openalex.org/I4210118560", "display_name": "Max Planck Institute for Evolutionary Anthropology", "ror": "https://ror.org/02a33b393", "country_code": "DE", "type": "facility", "lineage": ["https://openalex.org/I149899117", "https://openalex.org/I4210118560"]}], "countries": ["DE"], "is_corresponding": true, "raw_author_name": "Johannes Krause", "raw_affiliation_string": "Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany.", "raw_affiliation_strings": ["Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5044203404", "display_name": "Adrian W. Briggs", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210118560", "display_name": "Max Planck Institute for Evolutionary Anthropology", "ror": "https://ror.org/02a33b393", "country_code": "DE", "type": "facility", "lineage": ["https://openalex.org/I149899117", "https://openalex.org/I4210118560"]}], "countries": ["DE"], "is_corresponding": true, "raw_author_name": "Adrian W. Briggs", "raw_affiliation_string": "Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany.", "raw_affiliation_strings": ["Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5044374591", "display_name": "Tomislav Maričić", "orcid": "https://orcid.org/0000-0003-3267-0474"}, "institutions": [{"id": "https://openalex.org/I4210118560", "display_name": "Max Planck Institute for Evolutionary Anthropology", "ror": "https://ror.org/02a33b393", "country_code": "DE", "type": "facility", "lineage": ["https://openalex.org/I149899117", "https://openalex.org/I4210118560"]}], "countries": ["DE"], "is_corresponding": true, "raw_author_name": "Tomislav Maricic", "raw_affiliation_string": "Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany.", "raw_affiliation_strings": ["Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5082219479", "display_name": "Udo Stenzel", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210118560", "display_name": "Max Planck Institute for Evolutionary Anthropology", "ror": "https://ror.org/02a33b393", "country_code": "DE", "type": "facility", "lineage": ["https://openalex.org/I149899117", "https://openalex.org/I4210118560"]}], "countries": ["DE"], "is_corresponding": true, "raw_author_name": "Udo Stenzel", "raw_affiliation_string": "Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany.", "raw_affiliation_strings": ["Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5082477584", "display_name": "Martin Kircher", "orcid": "https://orcid.org/0000-0001-9278-5471"}, "institutions": [{"id": "https://openalex.org/I4210118560", "display_name": "Max Planck Institute for Evolutionary Anthropology", "ror": "https://ror.org/02a33b393", "country_code": "DE", "type": "facility", "lineage": ["https://openalex.org/I149899117", "https://openalex.org/I4210118560"]}], "countries": ["DE"], "is_corresponding": true, "raw_author_name": "Martin Kircher", "raw_affiliation_string": "Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany.", "raw_affiliation_strings": ["Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5091212930", "display_name": "Nick Patterson", "orcid": "https://orcid.org/0000-0002-2220-3648"}, "institutions": [{"id": "https://openalex.org/I107606265", "display_name": "Broad Institute", "ror": "https://ror.org/05a0ya142", "country_code": "US", "type": "nonprofit", "lineage": ["https://openalex.org/I107606265"]}], "countries": ["US"], "is_corresponding": true, "raw_author_name": "Nick Patterson", "raw_affiliation_string": "Broad Institute of MIT and Harvard, Cambridge, MA 02142, USA.", "raw_affiliation_strings": ["Broad Institute of MIT and Harvard, Cambridge, MA 02142, USA."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5013795174", "display_name": "Heng Li", "orcid": "https://orcid.org/0000-0002-3187-9041"}, "institutions": [{"id": "https://openalex.org/I107606265", "display_name": "Broad Institute", "ror": "https://ror.org/05a0ya142", "country_code": "US", "type": "nonprofit", "lineage": ["https://openalex.org/I107606265"]}], "countries": ["US"], "is_corresponding": true, "raw_author_name": "Heng Li", "raw_affiliation_string": "Broad Institute of MIT and Harvard, Cambridge, MA 02142, USA.", "raw_affiliation_strings": ["Broad Institute of MIT and Harvard, Cambridge, MA 02142, USA."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5039441451", "display_name": "Weiwei Zhai", "orcid": "https://orcid.org/0000-0001-7938-0226"}, "institutions": [{"id": "https://openalex.org/I95457486", "display_name": "University of California, Berkeley", "ror": "https://ror.org/01an7q238", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I2803209242", "https://openalex.org/I95457486"]}, {"id": "https://openalex.org/I4210100046", "display_name": "Integra (United States)", "ror": "https://ror.org/00ynqbp15", "country_code": "US", "type": "company", "lineage": ["https://openalex.org/I4210100046"]}], "countries": ["US"], "is_corresponding": true, "raw_author_name": "Weiwei Zhai", "raw_affiliation_string": "Department of Integrative Biology, University of California, Berkeley, CA 94720, USA.", "raw_affiliation_strings": ["Department of Integrative Biology, University of California, Berkeley, CA 94720, USA."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5024812314", "display_name": "Markus Hsi Yang Fritz", "orcid": null}, "institutions": [{"id": "https://openalex.org/I1303153112", "display_name": "European Bioinformatics Institute", "ror": "https://ror.org/02catss52", "country_code": "GB", "type": "facility", "lineage": ["https://openalex.org/I1303153112", "https://openalex.org/I4210138560"]}, {"id": "https://openalex.org/I87048295", "display_name": "Wellcome Trust", "ror": "https://ror.org/029chgv08", "country_code": "GB", "type": "nonprofit", "lineage": ["https://openalex.org/I87048295"]}], "countries": ["GB"], "is_corresponding": true, "raw_author_name": "Markus Hsi Yang Fritz", "raw_affiliation_string": "European Molecular Biology Laboratory–European Bioinformatics Institute, Wellcome Trust Genome Campus, Hinxton, Cambridgeshire, CB10 1SD, UK.", "raw_affiliation_strings": ["European Molecular Biology Laboratory–European Bioinformatics Institute, Wellcome Trust Genome Campus, Hinxton, Cambridgeshire, CB10 1SD, UK."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5057230759", "display_name": "Nancy F. Hansen", "orcid": "https://orcid.org/0000-0002-0950-0699"}, "institutions": [{"id": "https://openalex.org/I4210090236", "display_name": "National Human Genome Research Institute", "ror": "https://ror.org/00baak391", "country_code": "US", "type": "facility", "lineage": ["https://openalex.org/I1299303238", "https://openalex.org/I4210090236"]}, {"id": "https://openalex.org/I1299303238", "display_name": "National Institutes of Health", "ror": "https://ror.org/01cwqze88", "country_code": "US", "type": "government", "lineage": ["https://openalex.org/I1299022934", "https://openalex.org/I1299303238"]}], "countries": ["US"], "is_corresponding": true, "raw_author_name": "Nancy F. Hansen", "raw_affiliation_string": "Genome Technology Branch, National Human Genome Research Institute, National Institutes of Health, Bethesda, MD 20892, USA.", "raw_affiliation_strings": ["Genome Technology Branch, National Human Genome Research Institute, National Institutes of Health, Bethesda, MD 20892, USA."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5057279261", "display_name": "Éric Durand", "orcid": "https://orcid.org/0000-0002-8117-0022"}, "institutions": [{"id": "https://openalex.org/I95457486", "display_name": "University of California, Berkeley", "ror": "https://ror.org/01an7q238", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I2803209242", "https://openalex.org/I95457486"]}, {"id": "https://openalex.org/I4210100046", "display_name": "Integra (United States)", "ror": "https://ror.org/00ynqbp15", "country_code": "US", "type": "company", "lineage": ["https://openalex.org/I4210100046"]}], "countries": ["US"], "is_corresponding": true, "raw_author_name": "Eric Y. Durand", "raw_affiliation_string": "Department of Integrative Biology, University of California, Berkeley, CA 94720, USA.", "raw_affiliation_strings": ["Department of Integrative Biology, University of California, Berkeley, CA 94720, USA."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5064314813", "display_name": "Anna-Sapfo Malaspinas", "orcid": "https://orcid.org/0000-0003-1001-7511"}, "institutions": [{"id": "https://openalex.org/I95457486", "display_name": "University of California, Berkeley", "ror": "https://ror.org/01an7q238", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I2803209242", "https://openalex.org/I95457486"]}, {"id": "https://openalex.org/I4210100046", "display_name": "Integra (United States)", "ror": "https://ror.org/00ynqbp15", "country_code": "US", "type": "company", "lineage": ["https://openalex.org/I4210100046"]}], "countries": ["US"], "is_corresponding": true, "raw_author_name": "Anna Sapfo Malaspinas", "raw_affiliation_string": "Department of Integrative Biology, University of California, Berkeley, CA 94720, USA.", "raw_affiliation_strings": ["Department of Integrative Biology, University of California, Berkeley, CA 94720, USA."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5020301143", "display_name": "Jeffrey D. Jensen", "orcid": "https://orcid.org/0000-0002-4786-8064"}, "institutions": [{"id": "https://openalex.org/I166722992", "display_name": "University of Massachusetts Chan Medical School", "ror": "https://ror.org/0464eyp60", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I166722992", "https://openalex.org/I2802841742"]}], "countries": ["US"], "is_corresponding": true, "raw_author_name": "Jeffrey D. Jensen", "raw_affiliation_string": "Program in Bioinformatics and Integrative Biology, University of Massachusetts Medical School, Worcester, MA 01655, USA.", "raw_affiliation_strings": ["Program in Bioinformatics and Integrative Biology, University of Massachusetts Medical School, Worcester, MA 01655, USA."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5056155543", "display_name": "Tomàs Marquès‐Bonet", "orcid": "https://orcid.org/0000-0002-5597-3075"}, "institutions": [{"id": "https://openalex.org/I1344073410", "display_name": "Howard Hughes Medical Institute", "ror": "https://ror.org/006w34k90", "country_code": "US", "type": "nonprofit", "lineage": ["https://openalex.org/I1344073410"]}, {"id": "https://openalex.org/I201448701", "display_name": "University of Washington", "ror": "https://ror.org/00cvxb145", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I201448701"]}, {"id": "https://openalex.org/I4210135169", "display_name": "Institute of Evolutionary Biology", "ror": "https://ror.org/044mj7r89", "country_code": "ES", "type": "facility", "lineage": ["https://openalex.org/I134820265", "https://openalex.org/I170486558", "https://openalex.org/I4210135169"]}], "countries": ["ES", "US"], "is_corresponding": true, "raw_author_name": "Tomas Marques-Bonet", "raw_affiliation_string": "Howard Hughes Medical Institute, Department of Genome Sciences, University of Washington, Seattle, WA 98195, USA.; Institute of Evolutionary Biology (UPF-CSIC), Dr. Aiguader 88, 08003 Barcelona, Spain.", "raw_affiliation_strings": ["Howard Hughes Medical Institute, Department of Genome Sciences, University of Washington, Seattle, WA 98195, USA.", "Institute of Evolutionary Biology (UPF-CSIC), Dr. Aiguader 88, 08003 Barcelona, Spain."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5074885623", "display_name": "Can Alkan", "orcid": "https://orcid.org/0000-0002-5443-0706"}, "institutions": [{"id": "https://openalex.org/I1344073410", "display_name": "Howard Hughes Medical Institute", "ror": "https://ror.org/006w34k90", "country_code": "US", "type": "nonprofit", "lineage": ["https://openalex.org/I1344073410"]}, {"id": "https://openalex.org/I201448701", "display_name": "University of Washington", "ror": "https://ror.org/00cvxb145", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I201448701"]}], "countries": ["US"], "is_corresponding": true, "raw_author_name": "Can Alkan", "raw_affiliation_string": "Howard Hughes Medical Institute, Department of Genome Sciences, University of Washington, Seattle, WA 98195, USA.", "raw_affiliation_strings": ["Howard Hughes Medical Institute, Department of Genome Sciences, University of Washington, Seattle, WA 98195, USA."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5009866023", "display_name": "Kay Prüfer", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210118560", "display_name": "Max Planck Institute for Evolutionary Anthropology", "ror": "https://ror.org/02a33b393", "country_code": "DE", "type": "facility", "lineage": ["https://openalex.org/I149899117", "https://openalex.org/I4210118560"]}], "countries": ["DE"], "is_corresponding": true, "raw_author_name": "Kay Prüfer", "raw_affiliation_string": "Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany.", "raw_affiliation_strings": ["Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5008293185", "display_name": "Matthias Meyer", "orcid": "https://orcid.org/0000-0002-4760-558X"}, "institutions": [{"id": "https://openalex.org/I4210118560", "display_name": "Max Planck Institute for Evolutionary Anthropology", "ror": "https://ror.org/02a33b393", "country_code": "DE", "type": "facility", "lineage": ["https://openalex.org/I149899117", "https://openalex.org/I4210118560"]}], "countries": ["DE"], "is_corresponding": true, "raw_author_name": "Matthias Meyer", "raw_affiliation_string": "Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany.", "raw_affiliation_strings": ["Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5079209092", "display_name": "Hernán A. Burbano", "orcid": "https://orcid.org/0000-0003-3433-719X"}, "institutions": [{"id": "https://openalex.org/I4210118560", "display_name": "Max Planck Institute for Evolutionary Anthropology", "ror": "https://ror.org/02a33b393", "country_code": "DE", "type": "facility", "lineage": ["https://openalex.org/I149899117", "https://openalex.org/I4210118560"]}], "countries": ["DE"], "is_corresponding": true, "raw_author_name": "Hernán A. Burbano", "raw_affiliation_string": "Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany.", "raw_affiliation_strings": ["Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5046681707", "display_name": "Jeffrey M. Good", "orcid": "https://orcid.org/0000-0003-0707-5374"}, "institutions": [{"id": "https://openalex.org/I4210118560", "display_name": "Max Planck Institute for Evolutionary Anthropology", "ror": "https://ror.org/02a33b393", "country_code": "DE", "type": "facility", "lineage": ["https://openalex.org/I149899117", "https://openalex.org/I4210118560"]}, {"id": "https://openalex.org/I6750721", "display_name": "University of Montana", "ror": "https://ror.org/0078xmk34", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I2801213986", "https://openalex.org/I6750721"]}], "countries": ["DE", "US"], "is_corresponding": true, "raw_author_name": "Jeffrey M. Good", "raw_affiliation_string": "Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany.; Division of Biological Sciences, University of Montana, Missoula, MT 59812, USA.", "raw_affiliation_strings": ["Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany.", "Division of Biological Sciences, University of Montana, Missoula, MT 59812, USA."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5070466538", "display_name": "Rigo Schultz", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210118560", "display_name": "Max Planck Institute for Evolutionary Anthropology", "ror": "https://ror.org/02a33b393", "country_code": "DE", "type": "facility", "lineage": ["https://openalex.org/I149899117", "https://openalex.org/I4210118560"]}], "countries": ["DE"], "is_corresponding": false, "raw_author_name": "Rigo Schultz", "raw_affiliation_string": "Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany.", "raw_affiliation_strings": ["Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5035225714", "display_name": "Ayinuer Aximu‐Petri", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210118560", "display_name": "Max Planck Institute for Evolutionary Anthropology", "ror": "https://ror.org/02a33b393", "country_code": "DE", "type": "facility", "lineage": ["https://openalex.org/I149899117", "https://openalex.org/I4210118560"]}], "countries": ["DE"], "is_corresponding": false, "raw_author_name": "Ayinuer Aximu-Petri", "raw_affiliation_string": "Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany.", "raw_affiliation_strings": ["Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5050084486", "display_name": "Anne Butthof", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210118560", "display_name": "Max Planck Institute for Evolutionary Anthropology", "ror": "https://ror.org/02a33b393", "country_code": "DE", "type": "facility", "lineage": ["https://openalex.org/I149899117", "https://openalex.org/I4210118560"]}], "countries": ["DE"], "is_corresponding": false, "raw_author_name": "Anne Butthof", "raw_affiliation_string": "Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany.", "raw_affiliation_strings": ["Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5023786970", "display_name": "Barbara Höber", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210118560", "display_name": "Max Planck Institute for Evolutionary Anthropology", "ror": "https://ror.org/02a33b393", "country_code": "DE", "type": "facility", "lineage": ["https://openalex.org/I149899117", "https://openalex.org/I4210118560"]}], "countries": ["DE"], "is_corresponding": false, "raw_author_name": "Barbara Höber", "raw_affiliation_string": "Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany.", "raw_affiliation_strings": ["Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5041980776", "display_name": "Barbara Höffner", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210118560", "display_name": "Max Planck Institute for Evolutionary Anthropology", "ror": "https://ror.org/02a33b393", "country_code": "DE", "type": "facility", "lineage": ["https://openalex.org/I149899117", "https://openalex.org/I4210118560"]}], "countries": ["DE"], "is_corresponding": false, "raw_author_name": "Barbara Höffner", "raw_affiliation_string": "Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany.", "raw_affiliation_strings": ["Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5066851443", "display_name": "Madien Siegemund", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210118560", "display_name": "Max Planck Institute for Evolutionary Anthropology", "ror": "https://ror.org/02a33b393", "country_code": "DE", "type": "facility", "lineage": ["https://openalex.org/I149899117", "https://openalex.org/I4210118560"]}], "countries": ["DE"], "is_corresponding": false, "raw_author_name": "Madien Siegemund", "raw_affiliation_string": "Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany.", "raw_affiliation_strings": ["Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5032908748", "display_name": "Antje Weihmann", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210118560", "display_name": "Max Planck Institute for Evolutionary Anthropology", "ror": "https://ror.org/02a33b393", "country_code": "DE", "type": "facility", "lineage": ["https://openalex.org/I149899117", "https://openalex.org/I4210118560"]}], "countries": ["DE"], "is_corresponding": false, "raw_author_name": "Antje Weihmann", "raw_affiliation_string": "Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany.", "raw_affiliation_strings": ["Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5049581540", "display_name": "Chad Nusbaum", "orcid": null}, "institutions": [{"id": "https://openalex.org/I107606265", "display_name": "Broad Institute", "ror": "https://ror.org/05a0ya142", "country_code": "US", "type": "nonprofit", "lineage": ["https://openalex.org/I107606265"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Chad Nusbaum", "raw_affiliation_string": "Broad Institute of MIT and Harvard, Cambridge, MA 02142, USA.", "raw_affiliation_strings": ["Broad Institute of MIT and Harvard, Cambridge, MA 02142, USA."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5020748592", "display_name": "Eric S. Lander", "orcid": "https://orcid.org/0000-0003-2662-4631"}, "institutions": [{"id": "https://openalex.org/I107606265", "display_name": "Broad Institute", "ror": "https://ror.org/05a0ya142", "country_code": "US", "type": "nonprofit", "lineage": ["https://openalex.org/I107606265"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Eric S. Lander", "raw_affiliation_string": "Broad Institute of MIT and Harvard, Cambridge, MA 02142, USA.", "raw_affiliation_strings": ["Broad Institute of MIT and Harvard, Cambridge, MA 02142, USA."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5027605302", "display_name": "Carsten Russ", "orcid": null}, "institutions": [{"id": "https://openalex.org/I107606265", "display_name": "Broad Institute", "ror": "https://ror.org/05a0ya142", "country_code": "US", "type": "nonprofit", "lineage": ["https://openalex.org/I107606265"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Carsten Russ", "raw_affiliation_string": "Broad Institute of MIT and Harvard, Cambridge, MA 02142, USA.", "raw_affiliation_strings": ["Broad Institute of MIT and Harvard, Cambridge, MA 02142, USA."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5045532563", "display_name": "Nathaniel Novod", "orcid": null}, "institutions": [{"id": "https://openalex.org/I107606265", "display_name": "Broad Institute", "ror": "https://ror.org/05a0ya142", "country_code": "US", "type": "nonprofit", "lineage": ["https://openalex.org/I107606265"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Nathaniel Novod", "raw_affiliation_string": "Broad Institute of MIT and Harvard, Cambridge, MA 02142, USA.", "raw_affiliation_strings": ["Broad Institute of MIT and Harvard, Cambridge, MA 02142, USA."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5029924011", "display_name": "Jason P. Affourtit", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210102664", "display_name": "Enzo Life Sciences (United States)", "ror": "https://ror.org/01d7h6313", "country_code": "US", "type": "company", "lineage": ["https://openalex.org/I4210102664"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Jason Affourtit", "raw_affiliation_string": "454 Life Sciences, Branford, CT 06405, USA.", "raw_affiliation_strings": ["454 Life Sciences, Branford, CT 06405, USA."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5027613718", "display_name": "Michael Egholm", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210102664", "display_name": "Enzo Life Sciences (United States)", "ror": "https://ror.org/01d7h6313", "country_code": "US", "type": "company", "lineage": ["https://openalex.org/I4210102664"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Michael Egholm", "raw_affiliation_string": "454 Life Sciences, Branford, CT 06405, USA.", "raw_affiliation_strings": ["454 Life Sciences, Branford, CT 06405, USA."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5067975418", "display_name": "Christine Verna", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210118560", "display_name": "Max Planck Institute for Evolutionary Anthropology", "ror": "https://ror.org/02a33b393", "country_code": "DE", "type": "facility", "lineage": ["https://openalex.org/I149899117", "https://openalex.org/I4210118560"]}], "countries": ["DE"], "is_corresponding": false, "raw_author_name": "Christine Verna", "raw_affiliation_string": "Department of Human Evolution, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany.", "raw_affiliation_strings": ["Department of Human Evolution, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5063119535", "display_name": "Pavao Rudan", "orcid": null}, "institutions": [{"id": "https://openalex.org/I1301669915", "display_name": "Croatian Academy of Sciences and Arts", "ror": "https://ror.org/03d04qg82", "country_code": "HR", "type": "government", "lineage": ["https://openalex.org/I1301669915"]}], "countries": ["HR"], "is_corresponding": false, "raw_author_name": "Pavao Rudan", "raw_affiliation_string": "Croatian Academy of Sciences and Arts, Zrinski trg 11, HR-10000 Zagreb, Croatia.", "raw_affiliation_strings": ["Croatian Academy of Sciences and Arts, Zrinski trg 11, HR-10000 Zagreb, Croatia."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5053027797", "display_name": "Dejana Brajković", "orcid": null}, "institutions": [{"id": "https://openalex.org/I1301669915", "display_name": "Croatian Academy of Sciences and Arts", "ror": "https://ror.org/03d04qg82", "country_code": "HR", "type": "government", "lineage": ["https://openalex.org/I1301669915"]}], "countries": ["HR"], "is_corresponding": false, "raw_author_name": "Dejana Brajkovic", "raw_affiliation_string": "Croatian Academy of Sciences and Arts, Institute for Quaternary Paleontology and Geology, Ante Kovacica 5, HR-10000 Zagreb, Croatia.", "raw_affiliation_strings": ["Croatian Academy of Sciences and Arts, Institute for Quaternary Paleontology and Geology, Ante Kovacica 5, HR-10000 Zagreb, Croatia."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5077756602", "display_name": "Željko Kućan", "orcid": null}, "institutions": [{"id": "https://openalex.org/I1301669915", "display_name": "Croatian Academy of Sciences and Arts", "ror": "https://ror.org/03d04qg82", "country_code": "HR", "type": "government", "lineage": ["https://openalex.org/I1301669915"]}], "countries": ["HR"], "is_corresponding": false, "raw_author_name": "Željko Kucan", "raw_affiliation_string": "Croatian Academy of Sciences and Arts, Zrinski trg 11, HR-10000 Zagreb, Croatia.", "raw_affiliation_strings": ["Croatian Academy of Sciences and Arts, Zrinski trg 11, HR-10000 Zagreb, Croatia."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5070809643", "display_name": "Ivan Gušić", "orcid": null}, "institutions": [{"id": "https://openalex.org/I1301669915", "display_name": "Croatian Academy of Sciences and Arts", "ror": "https://ror.org/03d04qg82", "country_code": "HR", "type": "government", "lineage": ["https://openalex.org/I1301669915"]}], "countries": ["HR"], "is_corresponding": false, "raw_author_name": "Ivan Gušic", "raw_affiliation_string": "Croatian Academy of Sciences and Arts, Zrinski trg 11, HR-10000 Zagreb, Croatia.", "raw_affiliation_strings": ["Croatian Academy of Sciences and Arts, Zrinski trg 11, HR-10000 Zagreb, Croatia."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5025346300", "display_name": "Vladimir B. Doronichev", "orcid": "https://orcid.org/0000-0003-0198-0250"}, "institutions": [], "countries": ["RU"], "is_corresponding": false, "raw_author_name": "Vladimir B. Doronichev", "raw_affiliation_string": "ANO Laboratory of Prehistory, St. Petersburg, Russia.", "raw_affiliation_strings": ["ANO Laboratory of Prehistory, St. Petersburg, Russia."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5078468845", "display_name": "Liubov V. Golovanova", "orcid": "https://orcid.org/0000-0002-6099-4081"}, "institutions": [], "countries": ["RU"], "is_corresponding": false, "raw_author_name": "Liubov V. Golovanova", "raw_affiliation_string": "ANO Laboratory of Prehistory, St. Petersburg, Russia.", "raw_affiliation_strings": ["ANO Laboratory of Prehistory, St. Petersburg, Russia."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5056329756", "display_name": "Carles Lalueza-Fox", "orcid": "https://orcid.org/0000-0002-1730-5914"}, "institutions": [{"id": "https://openalex.org/I4210135169", "display_name": "Institute of Evolutionary Biology", "ror": "https://ror.org/044mj7r89", "country_code": "ES", "type": "facility", "lineage": ["https://openalex.org/I134820265", "https://openalex.org/I170486558", "https://openalex.org/I4210135169"]}], "countries": ["ES"], "is_corresponding": false, "raw_author_name": "Carles Lalueza-Fox", "raw_affiliation_string": "Institute of Evolutionary Biology (UPF-CSIC), Dr. Aiguader 88, 08003 Barcelona, Spain.", "raw_affiliation_strings": ["Institute of Evolutionary Biology (UPF-CSIC), Dr. Aiguader 88, 08003 Barcelona, Spain."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5088929398", "display_name": "Marco de la Rasilla Vives", "orcid": "https://orcid.org/0000-0002-5505-0625"}, "institutions": [{"id": "https://openalex.org/I165339363", "display_name": "University of Oviedo", "ror": "https://ror.org/006gksa02", "country_code": "ES", "type": "education", "lineage": ["https://openalex.org/I165339363"]}], "countries": ["ES"], "is_corresponding": false, "raw_author_name": "Marco De La Rasilla", "raw_affiliation_string": "Área de Prehistoria Departamento de Historia Universidad de Oviedo, Oviedo, Spain.", "raw_affiliation_strings": ["Área de Prehistoria Departamento de Historia Universidad de Oviedo, Oviedo, Spain."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5015927183", "display_name": "Javier Fortea", "orcid": null}, "institutions": [{"id": "https://openalex.org/I165339363", "display_name": "University of Oviedo", "ror": "https://ror.org/006gksa02", "country_code": "ES", "type": "education", "lineage": ["https://openalex.org/I165339363"]}], "countries": ["ES"], "is_corresponding": false, "raw_author_name": "Javier Fortea", "raw_affiliation_string": "Área de Prehistoria Departamento de Historia Universidad de Oviedo, Oviedo, Spain.", "raw_affiliation_strings": ["Área de Prehistoria Departamento de Historia Universidad de Oviedo, Oviedo, Spain."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5009367344", "display_name": "Antonio Rosas", "orcid": "https://orcid.org/0000-0002-5829-9952"}, "institutions": [{"id": "https://openalex.org/I4210120109", "display_name": "Museo Nacional de Ciencias Naturales", "ror": "https://ror.org/02v6zg374", "country_code": "ES", "type": "archive", "lineage": ["https://openalex.org/I134820265", "https://openalex.org/I4210120109"]}], "countries": ["ES"], "is_corresponding": false, "raw_author_name": "Antonio Rosas", "raw_affiliation_string": "Departamento de Paleobiología, Museo Nacional de Ciencias Naturales, CSIC, Madrid, Spain.", "raw_affiliation_strings": ["Departamento de Paleobiología, Museo Nacional de Ciencias Naturales, CSIC, Madrid, Spain."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5036086791", "display_name": "Ralf W. Schmitz", "orcid": null}, "institutions": [{"id": "https://openalex.org/I135140700", "display_name": "University of Bonn", "ror": "https://ror.org/041nas322", "country_code": "DE", "type": "education", "lineage": ["https://openalex.org/I135140700"]}], "countries": ["DE"], "is_corresponding": false, "raw_author_name": "Ralf W. Schmitz", "raw_affiliation_string": "Abteilung für Vor- und Frühgeschichtliche Archäologie, Universität Bonn, Germany.; Der Landschaftverband Rheinlund–Landesmuseum Bonn, Bachstrasse 5-9, D-53115 Bonn, Germany.", "raw_affiliation_strings": ["Abteilung für Vor- und Frühgeschichtliche Archäologie, Universität Bonn, Germany.", "Der Landschaftverband Rheinlund–Landesmuseum Bonn, Bachstrasse 5-9, D-53115 Bonn, Germany."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5020423008", "display_name": "Philip L. Johnson", "orcid": null}, "institutions": [{"id": "https://openalex.org/I150468666", "display_name": "Emory University", "ror": "https://ror.org/03czfpz43", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I150468666"]}], "countries": ["US"], "is_corresponding": true, "raw_author_name": "Philip L.F. Johnson", "raw_affiliation_string": "Department of Biology, Emory University, Atlanta, GA 30322, USA.", "raw_affiliation_strings": ["Department of Biology, Emory University, Atlanta, GA 30322, USA."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5014870107", "display_name": "Evan E. Eichler", "orcid": "https://orcid.org/0000-0002-8246-4014"}, "institutions": [{"id": "https://openalex.org/I1344073410", "display_name": "Howard Hughes Medical Institute", "ror": "https://ror.org/006w34k90", "country_code": "US", "type": "nonprofit", "lineage": ["https://openalex.org/I1344073410"]}, {"id": "https://openalex.org/I201448701", "display_name": "University of Washington", "ror": "https://ror.org/00cvxb145", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I201448701"]}], "countries": ["US"], "is_corresponding": true, "raw_author_name": "Evan E. Eichler", "raw_affiliation_string": "Howard Hughes Medical Institute, Department of Genome Sciences, University of Washington, Seattle, WA 98195, USA.", "raw_affiliation_strings": ["Howard Hughes Medical Institute, Department of Genome Sciences, University of Washington, Seattle, WA 98195, USA."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5087601456", "display_name": "Daniel Falush", "orcid": "https://orcid.org/0000-0002-2956-0795"}, "institutions": [{"id": "https://openalex.org/I27577105", "display_name": "University College Cork", "ror": "https://ror.org/03265fv13", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I181231927", "https://openalex.org/I27577105"]}], "countries": ["IE"], "is_corresponding": true, "raw_author_name": "Daniel Falush", "raw_affiliation_string": "Department of Microbiology, University College Cork, Cork, Ireland.", "raw_affiliation_strings": ["Department of Microbiology, University College Cork, Cork, Ireland."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5061379058", "display_name": "Ewan Birney", "orcid": "https://orcid.org/0000-0001-8314-8497"}, "institutions": [{"id": "https://openalex.org/I1303153112", "display_name": "European Bioinformatics Institute", "ror": "https://ror.org/02catss52", "country_code": "GB", "type": "facility", "lineage": ["https://openalex.org/I1303153112", "https://openalex.org/I4210138560"]}, {"id": "https://openalex.org/I87048295", "display_name": "Wellcome Trust", "ror": "https://ror.org/029chgv08", "country_code": "GB", "type": "nonprofit", "lineage": ["https://openalex.org/I87048295"]}], "countries": ["GB"], "is_corresponding": true, "raw_author_name": "Ewan Birney", "raw_affiliation_string": "European Molecular Biology Laboratory–European Bioinformatics Institute, Wellcome Trust Genome Campus, Hinxton, Cambridgeshire, CB10 1SD, UK.", "raw_affiliation_strings": ["European Molecular Biology Laboratory–European Bioinformatics Institute, Wellcome Trust Genome Campus, Hinxton, Cambridgeshire, CB10 1SD, UK."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5072393754", "display_name": "James C. Mullikin", "orcid": "https://orcid.org/0000-0003-0825-3750"}, "institutions": [{"id": "https://openalex.org/I4210090236", "display_name": "National Human Genome Research Institute", "ror": "https://ror.org/00baak391", "country_code": "US", "type": "facility", "lineage": ["https://openalex.org/I1299303238", "https://openalex.org/I4210090236"]}, {"id": "https://openalex.org/I1299303238", "display_name": "National Institutes of Health", "ror": "https://ror.org/01cwqze88", "country_code": "US", "type": "government", "lineage": ["https://openalex.org/I1299022934", "https://openalex.org/I1299303238"]}], "countries": ["US"], "is_corresponding": true, "raw_author_name": "James C. Mullikin", "raw_affiliation_string": "Genome Technology Branch, National Human Genome Research Institute, National Institutes of Health, Bethesda, MD 20892, USA.", "raw_affiliation_strings": ["Genome Technology Branch, National Human Genome Research Institute, National Institutes of Health, Bethesda, MD 20892, USA."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5067327311", "display_name": "Montgomery Slatkin", "orcid": null}, "institutions": [{"id": "https://openalex.org/I95457486", "display_name": "University of California, Berkeley", "ror": "https://ror.org/01an7q238", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I2803209242", "https://openalex.org/I95457486"]}, {"id": "https://openalex.org/I4210100046", "display_name": "Integra (United States)", "ror": "https://ror.org/00ynqbp15", "country_code": "US", "type": "company", "lineage": ["https://openalex.org/I4210100046"]}], "countries": ["US"], "is_corresponding": true, "raw_author_name": "Montgomery Slatkin", "raw_affiliation_string": "Department of Integrative Biology, University of California, Berkeley, CA 94720, USA.", "raw_affiliation_strings": ["Department of Integrative Biology, University of California, Berkeley, CA 94720, USA."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5088476239", "display_name": "Rasmus Nielsen", "orcid": "https://orcid.org/0000-0003-0513-6591"}, "institutions": [{"id": "https://openalex.org/I95457486", "display_name": "University of California, Berkeley", "ror": "https://ror.org/01an7q238", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I2803209242", "https://openalex.org/I95457486"]}, {"id": "https://openalex.org/I4210100046", "display_name": "Integra (United States)", "ror": "https://ror.org/00ynqbp15", "country_code": "US", "type": "company", "lineage": ["https://openalex.org/I4210100046"]}], "countries": ["US"], "is_corresponding": true, "raw_author_name": "Rasmus Nielsen", "raw_affiliation_string": "Department of Integrative Biology, University of California, Berkeley, CA 94720, USA.", "raw_affiliation_strings": ["Department of Integrative Biology, University of California, Berkeley, CA 94720, USA."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5080488747", "display_name": "Janet Kelso", "orcid": "https://orcid.org/0000-0002-3618-322X"}, "institutions": [{"id": "https://openalex.org/I4210118560", "display_name": "Max Planck Institute for Evolutionary Anthropology", "ror": "https://ror.org/02a33b393", "country_code": "DE", "type": "facility", "lineage": ["https://openalex.org/I149899117", "https://openalex.org/I4210118560"]}], "countries": ["DE"], "is_corresponding": true, "raw_author_name": "Janet Kelso", "raw_affiliation_string": "Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany.", "raw_affiliation_strings": ["Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5019027915", "display_name": "Michael Lachmann", "orcid": "https://orcid.org/0000-0002-1086-9717"}, "institutions": [{"id": "https://openalex.org/I4210118560", "display_name": "Max Planck Institute for Evolutionary Anthropology", "ror": "https://ror.org/02a33b393", "country_code": "DE", "type": "facility", "lineage": ["https://openalex.org/I149899117", "https://openalex.org/I4210118560"]}], "countries": ["DE"], "is_corresponding": true, "raw_author_name": "Michael Lachmann", "raw_affiliation_string": "Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany.", "raw_affiliation_strings": ["Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5011819951", "display_name": "David Reich", "orcid": "https://orcid.org/0000-0002-7037-5292"}, "institutions": [{"id": "https://openalex.org/I107606265", "display_name": "Broad Institute", "ror": "https://ror.org/05a0ya142", "country_code": "US", "type": "nonprofit", "lineage": ["https://openalex.org/I107606265"]}, {"id": "https://openalex.org/I136199984", "display_name": "Harvard University", "ror": "https://ror.org/03vek6s52", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I136199984"]}], "countries": ["US"], "is_corresponding": true, "raw_author_name": "David Reich", "raw_affiliation_string": "Broad Institute of MIT and Harvard, Cambridge, MA 02142, USA.; Department of Genetics, Harvard Medical School, Boston, MA 02115, USA.", "raw_affiliation_strings": ["Broad Institute of MIT and Harvard, Cambridge, MA 02142, USA.", "Department of Genetics, Harvard Medical School, Boston, MA 02115, USA."]}, {"author_position": "last", "author": {"id": "https://openalex.org/A5012216367", "display_name": "Svante Pääbo", "orcid": "https://orcid.org/0000-0002-4670-6311"}, "institutions": [{"id": "https://openalex.org/I4210118560", "display_name": "Max Planck Institute for Evolutionary Anthropology", "ror": "https://ror.org/02a33b393", "country_code": "DE", "type": "facility", "lineage": ["https://openalex.org/I149899117", "https://openalex.org/I4210118560"]}], "countries": ["DE"], "is_corresponding": true, "raw_author_name": "Svante Pääbo", "raw_affiliation_string": "Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany.", "raw_affiliation_strings": ["Department of Evolutionary Genetics, Max-Planck Institute for Evolutionary Anthropology, D-04103 Leipzig, Germany."]}], "countries_distinct_count": 7, "institutions_distinct_count": 21, "corresponding_author_ids": ["https://openalex.org/A5038814932", "https://openalex.org/A5091317187", "https://openalex.org/A5044203404", "https://openalex.org/A5044374591", "https://openalex.org/A5082219479", "https://openalex.org/A5082477584", "https://openalex.org/A5091212930", "https://openalex.org/A5013795174", "https://openalex.org/A5039441451", "https://openalex.org/A5024812314", "https://openalex.org/A5057230759", "https://openalex.org/A5057279261", "https://openalex.org/A5064314813", "https://openalex.org/A5020301143", "https://openalex.org/A5056155543", "https://openalex.org/A5074885623", "https://openalex.org/A5009866023", "https://openalex.org/A5008293185", "https://openalex.org/A5079209092", "https://openalex.org/A5046681707", "https://openalex.org/A5020423008", "https://openalex.org/A5014870107", "https://openalex.org/A5087601456", "https://openalex.org/A5061379058", "https://openalex.org/A5072393754", "https://openalex.org/A5067327311", "https://openalex.org/A5088476239", "https://openalex.org/A5080488747", "https://openalex.org/A5019027915", "https://openalex.org/A5011819951", "https://openalex.org/A5012216367"], "corresponding_institution_ids": ["https://openalex.org/I4210118560", "https://openalex.org/I4210118560", "https://openalex.org/I4210118560", "https://openalex.org/I4210118560", "https://openalex.org/I4210118560", "https://openalex.org/I4210118560", "https://openalex.org/I107606265", "https://openalex.org/I107606265", "https://openalex.org/I95457486", "https://openalex.org/I4210100046", "https://openalex.org/I1303153112", "https://openalex.org/I87048295", "https://openalex.org/I4210090236", "https://openalex.org/I1299303238", "https://openalex.org/I95457486", "https://openalex.org/I4210100046", "https://openalex.org/I95457486", "https://openalex.org/I4210100046", "https://openalex.org/I166722992", "https://openalex.org/I1344073410", "https://openalex.org/I201448701", "https://openalex.org/I4210135169", "https://openalex.org/I1344073410", "https://openalex.org/I201448701", "https://openalex.org/I4210118560", "https://openalex.org/I4210118560", "https://openalex.org/I4210118560", "https://openalex.org/I4210118560", "https://openalex.org/I6750721", "https://openalex.org/I150468666", "https://openalex.org/I1344073410", "https://openalex.org/I201448701", "https://openalex.org/I27577105", "https://openalex.org/I1303153112", "https://openalex.org/I87048295", "https://openalex.org/I4210090236", "https://openalex.org/I1299303238", "https://openalex.org/I95457486", "https://openalex.org/I4210100046", "https://openalex.org/I95457486", "https://openalex.org/I4210100046", "https://openalex.org/I4210118560", "https://openalex.org/I4210118560", "https://openalex.org/I107606265", "https://openalex.org/I136199984", "https://openalex.org/I4210118560"], "apc_list": null, "apc_paid": null, "has_fulltext": true, "fulltext_origin": "ngrams", "cited_by_count": 3542, "cited_by_percentile_year": {"min": 99.9, "max": 100.0}, "biblio": {"volume": "328", "issue": "5979", "first_page": "710", "last_page": "722"}, "is_retracted": false, "is_paratext": false, "keywords": [{"keyword": "neandertal genome", "score": 0.9198}, {"keyword": "draft sequence", "score": 0.4456}], "concepts": [{"id": "https://openalex.org/C86803240", "wikidata": "https://www.wikidata.org/wiki/Q420", "display_name": "Biology", "level": 0, "score": 0.6847178}, {"id": "https://openalex.org/C141231307", "wikidata": "https://www.wikidata.org/wiki/Q7020", "display_name": "Genome", "level": 3, "score": 0.66735774}, {"id": "https://openalex.org/C78458016", "wikidata": "https://www.wikidata.org/wiki/Q840400", "display_name": "Evolutionary biology", "level": 1, "score": 0.6581279}, {"id": "https://openalex.org/C2779987252", "wikidata": "https://www.wikidata.org/wiki/Q635162", "display_name": "Hominidae", "level": 3, "score": 0.533228}, {"id": "https://openalex.org/C2781271316", "wikidata": "https://www.wikidata.org/wiki/Q40171", "display_name": "Neanderthal", "level": 2, "score": 0.5064527}, {"id": "https://openalex.org/C54355233", "wikidata": "https://www.wikidata.org/wiki/Q7162", "display_name": "Genetics", "level": 1, "score": 0.42183834}, {"id": "https://openalex.org/C104317684", "wikidata": "https://www.wikidata.org/wiki/Q7187", "display_name": "Gene", "level": 2, "score": 0.39789662}, {"id": "https://openalex.org/C2988562018", "wikidata": "https://www.wikidata.org/wiki/Q1063", "display_name": "Biological evolution", "level": 2, "score": 0.31332722}, {"id": "https://openalex.org/C205649164", "wikidata": "https://www.wikidata.org/wiki/Q1071", "display_name": "Geography", "level": 0, "score": 0.1566945}, {"id": "https://openalex.org/C166957645", "wikidata": "https://www.wikidata.org/wiki/Q23498", "display_name": "Archaeology", "level": 1, "score": 0.0}], "mesh": [{"descriptor_ui": "D005580", "descriptor_name": "Fossils", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": true}, {"descriptor_ui": "D016678", "descriptor_name": "Genome", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": true}, {"descriptor_ui": "D015894", "descriptor_name": "Genome, Human", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": true}, {"descriptor_ui": "D015186", "descriptor_name": "Hominidae", "qualifier_ui": "Q000235", "qualifier_name": "genetics", "is_major_topic": true}, {"descriptor_ui": "D017422", "descriptor_name": "Sequence Analysis, DNA", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": true}, {"descriptor_ui": "D044383", "descriptor_name": "African Continental Ancestry Group", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D044383", "descriptor_name": "African Continental Ancestry Group", "qualifier_ui": "Q000235", "qualifier_name": "genetics", "is_major_topic": false}, {"descriptor_ui": "D000818", "descriptor_name": "Animals", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D044466", "descriptor_name": "Asian Continental Ancestry Group", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D044466", "descriptor_name": "Asian Continental Ancestry Group", "qualifier_ui": "Q000235", "qualifier_name": "genetics", "is_major_topic": false}, {"descriptor_ui": "D001483", "descriptor_name": "Base Sequence", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D001842", "descriptor_name": "Bone and Bones", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D004272", "descriptor_name": "DNA, Mitochondrial", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D004272", "descriptor_name": "DNA, Mitochondrial", "qualifier_ui": "Q000235", "qualifier_name": "genetics", "is_major_topic": false}, {"descriptor_ui": "D044465", "descriptor_name": "European Continental Ancestry Group", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D044465", "descriptor_name": "European Continental Ancestry Group", "qualifier_ui": "Q000235", "qualifier_name": "genetics", "is_major_topic": false}, {"descriptor_ui": "D019143", "descriptor_name": "Evolution, Molecular", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D053476", "descriptor_name": "Extinction, Biological", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D005260", "descriptor_name": "Female", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D018628", "descriptor_name": "Gene Dosage", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D051456", "descriptor_name": "Gene Flow", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D014644", "descriptor_name": "Genetic Variation", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D006239", "descriptor_name": "Haplotypes", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D015186", "descriptor_name": "Hominidae", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D006801", "descriptor_name": "Humans", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D002679", "descriptor_name": "Pan troglodytes", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D002679", "descriptor_name": "Pan troglodytes", "qualifier_ui": "Q000235", "qualifier_name": "genetics", "is_major_topic": false}, {"descriptor_ui": "D020641", "descriptor_name": "Polymorphism, Single Nucleotide", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D012641", "descriptor_name": "Selection, Genetic", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D016415", "descriptor_name": "Sequence Alignment", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D013995", "descriptor_name": "Time", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}], "locations_count": 4, "locations": [{"is_oa": true, "landing_page_url": "https://doi.org/10.1126/science.1188021", "pdf_url": null, "source": {"id": "https://openalex.org/S3880285", "display_name": "Science", "issn_l": "0036-8075", "issn": ["0036-8075", "1095-9203"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310315823", "host_organization_name": "American Association for the Advancement of Science", "host_organization_lineage": ["https://openalex.org/P4310315823"], "host_organization_lineage_names": ["American Association for the Advancement of Science"], "type": "journal"}, "license": null, "version": "publishedVersion", "is_accepted": true, "is_published": true}, {"is_oa": true, "landing_page_url": "https://europepmc.org/articles/pmc5100745", "pdf_url": "https://europepmc.org/articles/pmc5100745?pdf=render", "source": {"id": "https://openalex.org/S4306400806", "display_name": "Europe PMC (PubMed Central)", "issn_l": null, "issn": null, "is_oa": true, "is_in_doaj": false, "host_organization": "https://openalex.org/I1303153112", "host_organization_name": "European Bioinformatics Institute", "host_organization_lineage": ["https://openalex.org/I1303153112"], "host_organization_lineage_names": ["European Bioinformatics Institute"], "type": "repository"}, "license": null, "version": "acceptedVersion", "is_accepted": true, "is_published": false}, {"is_oa": true, "landing_page_url": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5100745", "pdf_url": null, "source": {"id": "https://openalex.org/S2764455111", "display_name": "PubMed Central", "issn_l": null, "issn": null, "is_oa": true, "is_in_doaj": false, "host_organization": "https://openalex.org/I1299303238", "host_organization_name": "National Institutes of Health", "host_organization_lineage": ["https://openalex.org/I1299303238"], "host_organization_lineage_names": ["National Institutes of Health"], "type": "repository"}, "license": null, "version": "acceptedVersion", "is_accepted": true, "is_published": false}, {"is_oa": false, "landing_page_url": "https://pubmed.ncbi.nlm.nih.gov/20448178", "pdf_url": null, "source": {"id": "https://openalex.org/S4306525036", "display_name": "PubMed", "issn_l": null, "issn": null, "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/I1299303238", "host_organization_name": "National Institutes of Health", "host_organization_lineage": ["https://openalex.org/I1299303238"], "host_organization_lineage_names": ["National Institutes of Health"], "type": "repository"}, "license": null, "version": null, "is_accepted": false, "is_published": false}], "best_oa_location": {"is_oa": true, "landing_page_url": "https://doi.org/10.1126/science.1188021", "pdf_url": null, "source": {"id": "https://openalex.org/S3880285", "display_name": "Science", "issn_l": "0036-8075", "issn": ["0036-8075", "1095-9203"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310315823", "host_organization_name": "American Association for the Advancement of Science", "host_organization_lineage": ["https://openalex.org/P4310315823"], "host_organization_lineage_names": ["American Association for the Advancement of Science"], "type": "journal"}, "license": null, "version": "publishedVersion", "is_accepted": true, "is_published": true}, "sustainable_development_goals": [{"id": "https://metadata.un.org/sdg/15", "display_name": "Life in Land", "score": 0.17}], "grants": [], "referenced_works_count": 80, "referenced_works": ["https://openalex.org/W24625822", "https://openalex.org/W1033235748", "https://openalex.org/W1966262033", "https://openalex.org/W1969869762", "https://openalex.org/W1973231394", "https://openalex.org/W1973685906", "https://openalex.org/W1974329144", "https://openalex.org/W1974900938", "https://openalex.org/W1975118655", "https://openalex.org/W1976588209", "https://openalex.org/W1977300115", "https://openalex.org/W1981232162", "https://openalex.org/W1987018175", "https://openalex.org/W1993435119", "https://openalex.org/W1994332522", "https://openalex.org/W1994573854", "https://openalex.org/W1996221326", "https://openalex.org/W1997407915", "https://openalex.org/W1998348937", "https://openalex.org/W2005743153", "https://openalex.org/W2006467490", "https://openalex.org/W2012016911", "https://openalex.org/W2013367562", "https://openalex.org/W2013714561", "https://openalex.org/W2015734214", "https://openalex.org/W2020823827", "https://openalex.org/W2023025638", "https://openalex.org/W2033242503", "https://openalex.org/W2034411088", "https://openalex.org/W2036384479", "https://openalex.org/W2038614704", "https://openalex.org/W2048313731", "https://openalex.org/W2050717506", "https://openalex.org/W2056170863", "https://openalex.org/W2061394869", "https://openalex.org/W2061837724", "https://openalex.org/W2065721819", "https://openalex.org/W2075074795", "https://openalex.org/W2076840114", "https://openalex.org/W2081229769", "https://openalex.org/W2084327894", "https://openalex.org/W2090684975", "https://openalex.org/W2092369215", "https://openalex.org/W2093463471", "https://openalex.org/W2098187463", "https://openalex.org/W2101294025", "https://openalex.org/W2106980598", "https://openalex.org/W2108485929", "https://openalex.org/W2110716173", "https://openalex.org/W2111310368", "https://openalex.org/W2113488183", "https://openalex.org/W2114976667", "https://openalex.org/W2122482936", "https://openalex.org/W2125526443", "https://openalex.org/W2125797555", "https://openalex.org/W2128114769", "https://openalex.org/W2130460780", "https://openalex.org/W2131088968", "https://openalex.org/W2132430052", "https://openalex.org/W2135016593", "https://openalex.org/W2137949197", "https://openalex.org/W2139417948", "https://openalex.org/W2139747296", "https://openalex.org/W2142642738", "https://openalex.org/W2144990191", "https://openalex.org/W2145371252", "https://openalex.org/W2146212748", "https://openalex.org/W2148994713", "https://openalex.org/W2151744354", "https://openalex.org/W2153784123", "https://openalex.org/W2155288921", "https://openalex.org/W2156330112", "https://openalex.org/W2156434996", "https://openalex.org/W2159623297", "https://openalex.org/W2165571729", "https://openalex.org/W2166465528", "https://openalex.org/W2167715883", "https://openalex.org/W2167983184", "https://openalex.org/W2170328376", "https://openalex.org/W4247527558"], "related_works": ["https://openalex.org/W2806047272", "https://openalex.org/W2034631686", "https://openalex.org/W2044120020", "https://openalex.org/W3004208933", "https://openalex.org/W2100397860", "https://openalex.org/W2000288374", "https://openalex.org/W2119165101", "https://openalex.org/W2185205737", "https://openalex.org/W2080162187", "https://openalex.org/W2184344526"], "ngrams_url": "https://api.openalex.org/works/W2124362779/ngrams", "abstract_inverted_index": {"Neandertals,": [0], "the": [1, 28, 43, 47, 57, 115, 121], "closest": [2], "evolutionary": [3], "relatives": [4], "of": [5, 12, 27, 32, 42, 49, 56, 62, 117, 123], "present-day": [6, 51, 97, 103], "humans,": [7, 76], "lived": [8], "in": [9, 73, 80, 83, 99, 105], "large": [10], "parts": [11, 55], "Europe": [13], "and": [14, 82, 85], "western": [15], "Asia": [16], "before": [17, 120], "disappearing": [18], "30,000": [19], "years": [20], "ago.": [21], "We": [22, 88], "present": [23], "a": [24, 60], "draft": [25], "sequence": [26], "Neandertal": [29, 44], "genome": [30, 45], "composed": [31], "more": [33, 93], "than": [34, 101], "4": [35], "billion": [36], "nucleotides": [37], "from": [38, 53, 112, 126], "three": [39], "individuals.": [40], "Comparisons": [41], "to": [46], "genomes": [48], "five": [50], "humans": [52, 98, 104], "different": [54], "world": [58], "identify": [59], "number": [61], "genomic": [63], "regions": [64], "that": [65, 90, 109], "may": [66], "have": [67], "been": [68], "affected": [69], "by": [70], "positive": [71], "selection": [72], "ancestral": [74], "modern": [75], "including": [77], "genes": [78], "involved": [79], "metabolism": [81], "cognitive": [84], "skeletal": [86], "development.": [87], "show": [89], "Neandertals": [91, 113], "shared": [92], "genetic": [94], "variants": [95], "with": [96, 102], "Eurasia": [100], "sub-Saharan": [106], "Africa,": [107], "suggesting": [108], "gene": [110], "flow": [111], "into": [114], "ancestors": [116], "non-Africans": [118], "occurred": [119], "divergence": [122], "Eurasian": [124], "groups": [125], "each": [127], "other.": [128]}, "cited_by_api_url": "https://api.openalex.org/works?filter=cites:W2124362779", "counts_by_year": [{"year": 2023, "cited_by_count": 233}, {"year": 2022, "cited_by_count": 269}, {"year": 2021, "cited_by_count": 299}, {"year": 2020, "cited_by_count": 273}, {"year": 2019, "cited_by_count": 271}, {"year": 2018, "cited_by_count": 284}, {"year": 2017, "cited_by_count": 271}, {"year": 2016, "cited_by_count": 259}, {"year": 2015, "cited_by_count": 254}, {"year": 2014, "cited_by_count": 283}, {"year": 2013, "cited_by_count": 275}, {"year": 2012, "cited_by_count": 263}], "updated_date": "2023-12-04T03:20:33.510203", "created_date": "2016-06-24"} +{"id": "https://openalex.org/W2115261608", "doi": "https://doi.org/10.1056/nejmoa0908721", "title": "Cisplatin plus Gemcitabine versus Gemcitabine for Biliary Tract Cancer", "display_name": "Cisplatin plus Gemcitabine versus Gemcitabine for Biliary Tract Cancer", "publication_year": 2010, "publication_date": "2010-04-08", "ids": {"openalex": "https://openalex.org/W2115261608", "doi": "https://doi.org/10.1056/nejmoa0908721", "mag": "2115261608", "pmid": "https://pubmed.ncbi.nlm.nih.gov/20375404"}, "language": "en", "primary_location": {"is_oa": true, "landing_page_url": "https://doi.org/10.1056/nejmoa0908721", "pdf_url": "https://www.nejm.org/doi/pdf/10.1056/NEJMoa0908721?articleTools=true", "source": {"id": "https://openalex.org/S62468778", "display_name": "The New England Journal of Medicine", "issn_l": "0028-4793", "issn": ["0028-4793", "1533-4406"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310320239", "host_organization_name": "Massachusetts Medical Society", "host_organization_lineage": ["https://openalex.org/P4310320239"], "host_organization_lineage_names": ["Massachusetts Medical Society"], "type": "journal"}, "license": null, "version": "publishedVersion", "is_accepted": true, "is_published": true}, "type": "article", "type_crossref": "journal-article", "open_access": {"is_oa": true, "oa_status": "green", "oa_url": "https://www.nejm.org/doi/pdf/10.1056/NEJMoa0908721?articleTools=true", "any_repository_has_fulltext": true}, "authorships": [{"author_position": "first", "author": {"id": "https://openalex.org/A5003454059", "display_name": "Juan W Valle", "orcid": "https://orcid.org/0000-0002-1999-0863"}, "institutions": [{"id": "https://openalex.org/I4210131968", "display_name": "The Christie Hospital", "ror": "https://ror.org/03nd63441", "country_code": "GB", "type": "healthcare", "lineage": ["https://openalex.org/I4210131968", "https://openalex.org/I4210133995"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Juan Ignacio Valle", "raw_affiliation_string": "Christie Hospital, Manchester, United Kingdom.", "raw_affiliation_strings": ["Christie Hospital, Manchester, United Kingdom."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5063738443", "display_name": "Harpreet Wasan", "orcid": "https://orcid.org/0000-0002-6268-2030"}, "institutions": [{"id": "https://openalex.org/I2801748203", "display_name": "Hammersmith Hospital", "ror": "https://ror.org/05jg8yp15", "country_code": "GB", "type": "healthcare", "lineage": ["https://openalex.org/I153355300", "https://openalex.org/I2801748203"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Harpreet Singh Wasan", "raw_affiliation_string": "Hammersmith Hospital, Imperial College Health Care Trust", "raw_affiliation_strings": ["Hammersmith Hospital, Imperial College Health Care Trust"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5025272654", "display_name": "Daniel H. Palmer", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210136696", "display_name": "NIHR Surgical Reconstruction and Microbiology Research Centre", "ror": "https://ror.org/042sjcz88", "country_code": "GB", "type": "facility", "lineage": ["https://openalex.org/I34931013", "https://openalex.org/I4210136696"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Daniel H Palmer", "raw_affiliation_string": "University Hospital Birmingham, Birmingham", "raw_affiliation_strings": ["University Hospital Birmingham, Birmingham"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5056345420", "display_name": "David Cunningham", "orcid": "https://orcid.org/0000-0001-5158-1069"}, "institutions": [{"id": "https://openalex.org/I4210121186", "display_name": "Royal Marsden Hospital", "ror": "https://ror.org/034vb5t35", "country_code": "GB", "type": "healthcare", "lineage": ["https://openalex.org/I1325846038", "https://openalex.org/I4210121186"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "David Cunningham", "raw_affiliation_string": "Royal Marsden Hospital", "raw_affiliation_strings": ["Royal Marsden Hospital"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5080693514", "display_name": "Alan Anthoney", "orcid": "https://orcid.org/0000-0001-5695-8312"}, "institutions": [{"id": "https://openalex.org/I2801331674", "display_name": "St James's University Hospital", "ror": "https://ror.org/013s89d74", "country_code": "GB", "type": "healthcare", "lineage": ["https://openalex.org/I2799390153", "https://openalex.org/I2801331674"]}, {"id": "https://openalex.org/I4210154619", "display_name": "St. James's Hospital", "ror": "https://ror.org/04c6bry31", "country_code": "IE", "type": "healthcare", "lineage": ["https://openalex.org/I4210154619"]}], "countries": ["GB", "IE"], "is_corresponding": false, "raw_author_name": "Alan Anthoney", "raw_affiliation_string": "St. James's Hospital, Leeds", "raw_affiliation_strings": ["St. James's Hospital, Leeds"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5060816944", "display_name": "Anthony Maraveyas", "orcid": null}, "institutions": [{"id": "https://openalex.org/I2801269832", "display_name": "Castle Hill Hospital", "ror": "https://ror.org/042asnw05", "country_code": "GB", "type": "healthcare", "lineage": ["https://openalex.org/I2800043709", "https://openalex.org/I2801269832"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Anthony Maraveyas", "raw_affiliation_string": "Castle Hill Hospital, Hull", "raw_affiliation_strings": ["Castle Hill Hospital, Hull"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5015937284", "display_name": "Srinivasan Madhusudan", "orcid": "https://orcid.org/0000-0002-5354-5480"}, "institutions": [{"id": "https://openalex.org/I1334287468", "display_name": "Nottingham University Hospitals NHS Trust", "ror": "https://ror.org/05y3qh794", "country_code": "GB", "type": "healthcare", "lineage": ["https://openalex.org/I1334287468"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Srinivasan Madhusudan", "raw_affiliation_string": "Nottingham University Hospitals, Nottingham", "raw_affiliation_strings": ["Nottingham University Hospitals, Nottingham"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5054875746", "display_name": "Tim Iveson", "orcid": "https://orcid.org/0000-0002-4681-2712"}, "institutions": [{"id": "https://openalex.org/I151328261", "display_name": "Hampton University", "ror": "https://ror.org/05fde5z47", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I151328261"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Tim Iveson", "raw_affiliation_string": "Southampton University Hospitals, Southampton", "raw_affiliation_strings": ["Southampton University Hospitals, Southampton"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5074785504", "display_name": "Sharon M. Hughes", "orcid": null}, "institutions": [{"id": "https://openalex.org/I45129253", "display_name": "University College London", "ror": "https://ror.org/02jx3x895", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I124357947", "https://openalex.org/I45129253"]}, {"id": "https://openalex.org/I4210088881", "display_name": "London Cancer", "ror": "https://ror.org/005kpb876", "country_code": "GB", "type": "other", "lineage": ["https://openalex.org/I4210088881"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Sharon M. Hughes", "raw_affiliation_string": "University College London Cancer Trials Centre", "raw_affiliation_strings": ["University College London Cancer Trials Centre"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5066537961", "display_name": "Stephen P. Pereira", "orcid": "https://orcid.org/0000-0003-0821-1809"}, "institutions": [{"id": "https://openalex.org/I45129253", "display_name": "University College London", "ror": "https://ror.org/02jx3x895", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I124357947", "https://openalex.org/I45129253"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Stephen P. Pereira", "raw_affiliation_string": "Institute of Hepatology, University College London", "raw_affiliation_strings": ["Institute of Hepatology, University College London"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5039406185", "display_name": "Michael Roughton", "orcid": null}, "institutions": [{"id": "https://openalex.org/I45129253", "display_name": "University College London", "ror": "https://ror.org/02jx3x895", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I124357947", "https://openalex.org/I45129253"]}, {"id": "https://openalex.org/I4210088881", "display_name": "London Cancer", "ror": "https://ror.org/005kpb876", "country_code": "GB", "type": "other", "lineage": ["https://openalex.org/I4210088881"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Michael Roughton", "raw_affiliation_string": "University College London Cancer Trials Centre", "raw_affiliation_strings": ["University College London Cancer Trials Centre"]}, {"author_position": "last", "author": {"id": "https://openalex.org/A5083898918", "display_name": "John Bridgewater", "orcid": "https://orcid.org/0000-0001-9186-1604"}, "institutions": [{"id": "https://openalex.org/I2801345345", "display_name": "Cancer Institute (WIA)", "ror": "https://ror.org/01tc10z29", "country_code": "IN", "type": "healthcare", "lineage": ["https://openalex.org/I2801345345"]}, {"id": "https://openalex.org/I4210088881", "display_name": "London Cancer", "ror": "https://ror.org/005kpb876", "country_code": "GB", "type": "other", "lineage": ["https://openalex.org/I4210088881"]}, {"id": "https://openalex.org/I45129253", "display_name": "University College London", "ror": "https://ror.org/02jx3x895", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I124357947", "https://openalex.org/I45129253"]}], "countries": ["GB", "IN"], "is_corresponding": false, "raw_author_name": "John Bridgewater", "raw_affiliation_string": "University College London Cancer Institute", "raw_affiliation_strings": ["University College London Cancer Institute"]}], "countries_distinct_count": 4, "institutions_distinct_count": 12, "corresponding_author_ids": [], "corresponding_institution_ids": [], "apc_list": null, "apc_paid": null, "has_fulltext": true, "fulltext_origin": "ngrams", "cited_by_count": 3169, "cited_by_percentile_year": {"min": 99.9, "max": 100.0}, "biblio": {"volume": "362", "issue": "14", "first_page": "1273", "last_page": "1281"}, "is_retracted": false, "is_paratext": false, "keywords": [{"keyword": "biliary tract cancer", "score": 0.5999}, {"keyword": "gemcitabine", "score": 0.4915}], "concepts": [{"id": "https://openalex.org/C2780258809", "wikidata": "https://www.wikidata.org/wiki/Q414143", "display_name": "Gemcitabine", "level": 3, "score": 0.9855536}, {"id": "https://openalex.org/C71924100", "wikidata": "https://www.wikidata.org/wiki/Q11190", "display_name": "Medicine", "level": 0, "score": 0.91816986}, {"id": "https://openalex.org/C2778239845", "wikidata": "https://www.wikidata.org/wiki/Q412415", "display_name": "Cisplatin", "level": 3, "score": 0.79349643}, {"id": "https://openalex.org/C3017919176", "wikidata": "https://www.wikidata.org/wiki/Q124292", "display_name": "Biliary tract cancer", "level": 4, "score": 0.7705759}, {"id": "https://openalex.org/C2776694085", "wikidata": "https://www.wikidata.org/wiki/Q974135", "display_name": "Chemotherapy", "level": 2, "score": 0.663098}, {"id": "https://openalex.org/C126322002", "wikidata": "https://www.wikidata.org/wiki/Q11180", "display_name": "Internal medicine", "level": 1, "score": 0.587602}, {"id": "https://openalex.org/C143998085", "wikidata": "https://www.wikidata.org/wiki/Q162555", "display_name": "Oncology", "level": 1, "score": 0.58636534}, {"id": "https://openalex.org/C2775982439", "wikidata": "https://www.wikidata.org/wiki/Q3562150", "display_name": "Biliary tract", "level": 2, "score": 0.5824834}, {"id": "https://openalex.org/C2777844706", "wikidata": "https://www.wikidata.org/wiki/Q422504", "display_name": "Deoxycytidine", "level": 4, "score": 0.48224363}], "mesh": [{"descriptor_ui": "D000971", "descriptor_name": "Antineoplastic Combined Chemotherapy Protocols", "qualifier_ui": "Q000627", "qualifier_name": "therapeutic use", "is_major_topic": true}, {"descriptor_ui": "D001661", "descriptor_name": "Biliary Tract Neoplasms", "qualifier_ui": "Q000188", "qualifier_name": "drug therapy", "is_major_topic": true}, {"descriptor_ui": "D002945", "descriptor_name": "Cisplatin", "qualifier_ui": "Q000627", "qualifier_name": "therapeutic use", "is_major_topic": true}, {"descriptor_ui": "D003841", "descriptor_name": "Deoxycytidine", "qualifier_ui": "Q000031", "qualifier_name": "analogs & derivatives", "is_major_topic": true}, {"descriptor_ui": "D000328", "descriptor_name": "Adult", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D000368", "descriptor_name": "Aged", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D000369", "descriptor_name": "Aged, 80 and over", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D000964", "descriptor_name": "Antimetabolites, Antineoplastic", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D000964", "descriptor_name": "Antimetabolites, Antineoplastic", "qualifier_ui": "Q000009", "qualifier_name": "adverse effects", "is_major_topic": false}, {"descriptor_ui": "D000964", "descriptor_name": "Antimetabolites, Antineoplastic", "qualifier_ui": "Q000627", "qualifier_name": "therapeutic use", "is_major_topic": false}, {"descriptor_ui": "D000971", "descriptor_name": "Antineoplastic Combined Chemotherapy Protocols", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D000971", "descriptor_name": "Antineoplastic Combined Chemotherapy Protocols", "qualifier_ui": "Q000009", "qualifier_name": "adverse effects", "is_major_topic": false}, {"descriptor_ui": "D001661", "descriptor_name": "Biliary Tract Neoplasms", "qualifier_ui": "Q000401", "qualifier_name": "mortality", "is_major_topic": false}, {"descriptor_ui": "D001661", "descriptor_name": "Biliary Tract Neoplasms", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D002945", "descriptor_name": "Cisplatin", "qualifier_ui": "Q000009", "qualifier_name": "adverse effects", "is_major_topic": false}, {"descriptor_ui": "D002945", "descriptor_name": "Cisplatin", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D003841", "descriptor_name": "Deoxycytidine", "qualifier_ui": "Q000009", "qualifier_name": "adverse effects", "is_major_topic": false}, {"descriptor_ui": "D003841", "descriptor_name": "Deoxycytidine", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D003841", "descriptor_name": "Deoxycytidine", "qualifier_ui": "Q000627", "qualifier_name": "therapeutic use", "is_major_topic": false}, {"descriptor_ui": "D018572", "descriptor_name": "Disease-Free Survival", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D005260", "descriptor_name": "Female", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D005500", "descriptor_name": "Follow-Up Studies", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D006801", "descriptor_name": "Humans", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D053208", "descriptor_name": "Kaplan-Meier Estimate", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D008297", "descriptor_name": "Male", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D008875", "descriptor_name": "Middle Aged", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D009503", "descriptor_name": "Neutropenia", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D009503", "descriptor_name": "Neutropenia", "qualifier_ui": "Q000139", "qualifier_name": "chemically induced", "is_major_topic": false}, {"descriptor_ui": "D010349", "descriptor_name": "Patient Compliance", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D010349", "descriptor_name": "Patient Compliance", "qualifier_ui": "Q000706", "qualifier_name": "statistics & numerical data", "is_major_topic": false}, {"descriptor_ui": "D016016", "descriptor_name": "Proportional Hazards Models", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}], "locations_count": 4, "locations": [{"is_oa": true, "landing_page_url": "https://doi.org/10.1056/nejmoa0908721", "pdf_url": "https://www.nejm.org/doi/pdf/10.1056/NEJMoa0908721?articleTools=true", "source": {"id": "https://openalex.org/S62468778", "display_name": "The New England Journal of Medicine", "issn_l": "0028-4793", "issn": ["0028-4793", "1533-4406"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310320239", "host_organization_name": "Massachusetts Medical Society", "host_organization_lineage": ["https://openalex.org/P4310320239"], "host_organization_lineage_names": ["Massachusetts Medical Society"], "type": "journal"}, "license": null, "version": "publishedVersion", "is_accepted": true, "is_published": true}, {"is_oa": true, "landing_page_url": "https://hull-repository.worktribe.com/output/467879", "pdf_url": "https://hull-repository.worktribe.com/preview/467908/nejmoa0908721.pdf", "source": {"id": "https://openalex.org/S4306400827", "display_name": "Repository@Hull (Worktribe) (University of Hull)", "issn_l": null, "issn": null, "is_oa": true, "is_in_doaj": false, "host_organization": "https://openalex.org/I191240316", "host_organization_name": "University of Hull", "host_organization_lineage": ["https://openalex.org/I191240316"], "host_organization_lineage_names": ["University of Hull"], "type": "repository"}, "license": "cc-by", "version": "acceptedVersion", "is_accepted": true, "is_published": false}, {"is_oa": true, "landing_page_url": "https://hull-repository.worktribe.com/file/467879/1/Article.pdf", "pdf_url": "https://hull-repository.worktribe.com/file/467879/1/Article.pdf", "source": {"id": "https://openalex.org/S4306400827", "display_name": "Repository@Hull (Worktribe) (University of Hull)", "issn_l": null, "issn": null, "is_oa": true, "is_in_doaj": false, "host_organization": "https://openalex.org/I191240316", "host_organization_name": "University of Hull", "host_organization_lineage": ["https://openalex.org/I191240316"], "host_organization_lineage_names": ["University of Hull"], "type": "repository"}, "license": null, "version": "acceptedVersion", "is_accepted": true, "is_published": false}, {"is_oa": false, "landing_page_url": "https://pubmed.ncbi.nlm.nih.gov/20375404", "pdf_url": null, "source": {"id": "https://openalex.org/S4306525036", "display_name": "PubMed", "issn_l": null, "issn": null, "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/I1299303238", "host_organization_name": "National Institutes of Health", "host_organization_lineage": ["https://openalex.org/I1299303238"], "host_organization_lineage_names": ["National Institutes of Health"], "type": "repository"}, "license": null, "version": null, "is_accepted": false, "is_published": false}], "best_oa_location": {"is_oa": true, "landing_page_url": "https://doi.org/10.1056/nejmoa0908721", "pdf_url": "https://www.nejm.org/doi/pdf/10.1056/NEJMoa0908721?articleTools=true", "source": {"id": "https://openalex.org/S62468778", "display_name": "The New England Journal of Medicine", "issn_l": "0028-4793", "issn": ["0028-4793", "1533-4406"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310320239", "host_organization_name": "Massachusetts Medical Society", "host_organization_lineage": ["https://openalex.org/P4310320239"], "host_organization_lineage_names": ["Massachusetts Medical Society"], "type": "journal"}, "license": null, "version": "publishedVersion", "is_accepted": true, "is_published": true}, "sustainable_development_goals": [{"id": "https://metadata.un.org/sdg/3", "display_name": "Good health and well-being", "score": 0.86}], "grants": [], "referenced_works_count": 18, "referenced_works": ["https://openalex.org/W1911108481", "https://openalex.org/W1938837918", "https://openalex.org/W1989145928", "https://openalex.org/W2003278673", "https://openalex.org/W2041941116", "https://openalex.org/W2075957912", "https://openalex.org/W2078797029", "https://openalex.org/W2082865236", "https://openalex.org/W2093388808", "https://openalex.org/W2101036284", "https://openalex.org/W2103257412", "https://openalex.org/W2116538964", "https://openalex.org/W2117834374", "https://openalex.org/W2128889451", "https://openalex.org/W2132129574", "https://openalex.org/W2139248078", "https://openalex.org/W2139947970", "https://openalex.org/W2319550695"], "related_works": ["https://openalex.org/W2100029565", "https://openalex.org/W4248275212", "https://openalex.org/W39641639", "https://openalex.org/W2334689178", "https://openalex.org/W2328187296", "https://openalex.org/W3088708979", "https://openalex.org/W2018086491", "https://openalex.org/W2799679442", "https://openalex.org/W2321409483", "https://openalex.org/W2918357092"], "ngrams_url": "https://api.openalex.org/works/W2115261608/ngrams", "abstract_inverted_index": {"There": [0], "is": [1], "no": [2], "established": [3], "standard": [4], "chemotherapy": [5], "for": [6], "patients": [7, 26], "with": [8, 32], "locally": [9], "advanced": [10], "or": [11], "metastatic": [12], "biliary": [13], "tract": [14], "cancer.": [15], "We": [16], "initially": [17], "conducted": [18], "a": [19], "randomized,": [20], "phase": [21, 49], "2": [22], "study": [23], "involving": [24], "86": [25], "to": [27, 47], "compare": [28], "cisplatin": [29], "plus": [30], "gemcitabine": [31, 33], "alone.": [34], "After": [35], "we": [36], "found": [37], "an": [38], "improvement": [39], "in": [40], "progression-free": [41], "survival,": [42], "the": [43, 48], "trial": [44, 51], "was": [45], "extended": [46], "3": [50], "reported": [52], "here.": [53]}, "cited_by_api_url": "https://api.openalex.org/works?filter=cites:W2115261608", "counts_by_year": [{"year": 2023, "cited_by_count": 378}, {"year": 2022, "cited_by_count": 411}, {"year": 2021, "cited_by_count": 416}, {"year": 2020, "cited_by_count": 338}, {"year": 2019, "cited_by_count": 265}, {"year": 2018, "cited_by_count": 212}, {"year": 2017, "cited_by_count": 218}, {"year": 2016, "cited_by_count": 197}, {"year": 2015, "cited_by_count": 183}, {"year": 2014, "cited_by_count": 174}, {"year": 2013, "cited_by_count": 134}, {"year": 2012, "cited_by_count": 133}], "updated_date": "2023-11-29T20:15:22.683044", "created_date": "2016-06-24"} +{"id": "https://openalex.org/W2157622195", "doi": "https://doi.org/10.1016/s0140-6736(10)60834-3", "title": "Risk factors for ischaemic and intracerebral haemorrhagic stroke in 22 countries (the INTERSTROKE study): a case-control study", "display_name": "Risk factors for ischaemic and intracerebral haemorrhagic stroke in 22 countries (the INTERSTROKE study): a case-control study", "publication_year": 2010, "publication_date": "2010-07-01", "ids": {"openalex": "https://openalex.org/W2157622195", "doi": "https://doi.org/10.1016/s0140-6736(10)60834-3", "mag": "2157622195", "pmid": "https://pubmed.ncbi.nlm.nih.gov/20561675"}, "language": "en", "primary_location": {"is_oa": false, "landing_page_url": "https://doi.org/10.1016/s0140-6736(10)60834-3", "pdf_url": null, "source": {"id": "https://openalex.org/S49861241", "display_name": "The Lancet", "issn_l": "0140-6736", "issn": ["1474-547X", "0099-5355", "0140-6736"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310320990", "host_organization_name": "Elsevier BV", "host_organization_lineage": ["https://openalex.org/P4310320990"], "host_organization_lineage_names": ["Elsevier BV"], "type": "journal"}, "license": null, "version": null, "is_accepted": false, "is_published": false}, "type": "article", "type_crossref": "journal-article", "open_access": {"is_oa": false, "oa_status": "closed", "oa_url": null, "any_repository_has_fulltext": false}, "authorships": [{"author_position": "first", "author": {"id": "https://openalex.org/A5035977661", "display_name": "Martin O’Donnell", "orcid": "https://orcid.org/0000-0002-7347-7761"}, "institutions": [{"id": "https://openalex.org/I2802834092", "display_name": "Population Health Research Institute", "ror": "https://ror.org/03kwaeq96", "country_code": "CA", "type": "healthcare", "lineage": ["https://openalex.org/I2802834092"]}, {"id": "https://openalex.org/I98251732", "display_name": "McMaster University", "ror": "https://ror.org/02fa3aq29", "country_code": "CA", "type": "education", "lineage": ["https://openalex.org/I98251732"]}, {"id": "https://openalex.org/I188760350", "display_name": "Ollscoil na Gaillimhe – University of Galway", "ror": "https://ror.org/03bea9k73", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I181231927", "https://openalex.org/I188760350"]}], "countries": ["CA", "IE"], "is_corresponding": true, "raw_author_name": "Martin J O'Donnell", "raw_affiliation_string": "HRB-Clinical Research Facility, NUI Galway, Ireland; Population Health Research Institute, McMaster University, Hamilton, ON, Canada", "raw_affiliation_strings": ["HRB-Clinical Research Facility, NUI Galway, Ireland", "Population Health Research Institute, McMaster University, Hamilton, ON, Canada"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5057705555", "display_name": "Denis Xavier", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210145302", "display_name": "St.John's Medical College Hospital", "ror": "https://ror.org/04z7fc725", "country_code": "IN", "type": "education", "lineage": ["https://openalex.org/I4210129261", "https://openalex.org/I4210145302"]}], "countries": ["IN"], "is_corresponding": false, "raw_author_name": "Denis Xavier", "raw_affiliation_string": "St John's Medical College and Research Institute, Bangalore, India", "raw_affiliation_strings": ["St John's Medical College and Research Institute, Bangalore, India"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5021591848", "display_name": "Lisheng Liu", "orcid": "https://orcid.org/0000-0002-8254-8031"}, "institutions": [{"id": "https://openalex.org/I4210161090", "display_name": "China National Centre for Food Safety Risk Assessment", "ror": "https://ror.org/058mseb02", "country_code": "CN", "type": "government", "lineage": ["https://openalex.org/I4210161090"]}], "countries": ["CN"], "is_corresponding": false, "raw_author_name": "Lisheng Liu", "raw_affiliation_string": "National Centre of Cardiovascular Disease, Beijing, China", "raw_affiliation_strings": ["National Centre of Cardiovascular Disease, Beijing, China"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5037935896", "display_name": "Hongye Zhang", "orcid": "https://orcid.org/0000-0002-8960-4614"}, "institutions": [{"id": "https://openalex.org/I4210129240", "display_name": "Shanghai Institute of Hypertension", "ror": "https://ror.org/038j9sn30", "country_code": "CN", "type": "facility", "lineage": ["https://openalex.org/I4210129240"]}], "countries": ["CN"], "is_corresponding": false, "raw_author_name": "Hongye Zhang", "raw_affiliation_string": "Beijing Hypertension League Institute, Beijing, China", "raw_affiliation_strings": ["Beijing Hypertension League Institute, Beijing, China"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5014226876", "display_name": "Siu Lim Chin", "orcid": null}, "institutions": [{"id": "https://openalex.org/I2802834092", "display_name": "Population Health Research Institute", "ror": "https://ror.org/03kwaeq96", "country_code": "CA", "type": "healthcare", "lineage": ["https://openalex.org/I2802834092"]}, {"id": "https://openalex.org/I98251732", "display_name": "McMaster University", "ror": "https://ror.org/02fa3aq29", "country_code": "CA", "type": "education", "lineage": ["https://openalex.org/I98251732"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "Siu Lim Chin", "raw_affiliation_string": "Population Health Research Institute, McMaster University, Hamilton, ON, Canada", "raw_affiliation_strings": ["Population Health Research Institute, McMaster University, Hamilton, ON, Canada"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5082026644", "display_name": "Purnima Rao‐Melacini", "orcid": "https://orcid.org/0000-0002-7537-9057"}, "institutions": [{"id": "https://openalex.org/I2802834092", "display_name": "Population Health Research Institute", "ror": "https://ror.org/03kwaeq96", "country_code": "CA", "type": "healthcare", "lineage": ["https://openalex.org/I2802834092"]}, {"id": "https://openalex.org/I98251732", "display_name": "McMaster University", "ror": "https://ror.org/02fa3aq29", "country_code": "CA", "type": "education", "lineage": ["https://openalex.org/I98251732"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "Purnima Rao-Melacini", "raw_affiliation_string": "Population Health Research Institute, McMaster University, Hamilton, ON, Canada", "raw_affiliation_strings": ["Population Health Research Institute, McMaster University, Hamilton, ON, Canada"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5069728608", "display_name": "Sumathy Rangarajan", "orcid": "https://orcid.org/0000-0003-2420-5986"}, "institutions": [{"id": "https://openalex.org/I2802834092", "display_name": "Population Health Research Institute", "ror": "https://ror.org/03kwaeq96", "country_code": "CA", "type": "healthcare", "lineage": ["https://openalex.org/I2802834092"]}, {"id": "https://openalex.org/I98251732", "display_name": "McMaster University", "ror": "https://ror.org/02fa3aq29", "country_code": "CA", "type": "education", "lineage": ["https://openalex.org/I98251732"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "Sumathy Rangarajan", "raw_affiliation_string": "Population Health Research Institute, McMaster University, Hamilton, ON, Canada", "raw_affiliation_strings": ["Population Health Research Institute, McMaster University, Hamilton, ON, Canada"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5068719503", "display_name": "Shofiqul Islam", "orcid": "https://orcid.org/0000-0001-8196-8598"}, "institutions": [{"id": "https://openalex.org/I2802834092", "display_name": "Population Health Research Institute", "ror": "https://ror.org/03kwaeq96", "country_code": "CA", "type": "healthcare", "lineage": ["https://openalex.org/I2802834092"]}, {"id": "https://openalex.org/I98251732", "display_name": "McMaster University", "ror": "https://ror.org/02fa3aq29", "country_code": "CA", "type": "education", "lineage": ["https://openalex.org/I98251732"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "Shofiqul Islam", "raw_affiliation_string": "Population Health Research Institute, McMaster University, Hamilton, ON, Canada", "raw_affiliation_strings": ["Population Health Research Institute, McMaster University, Hamilton, ON, Canada"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5082084514", "display_name": "Прем Пайс", "orcid": "https://orcid.org/0000-0001-5985-3137"}, "institutions": [{"id": "https://openalex.org/I4210145302", "display_name": "St.John's Medical College Hospital", "ror": "https://ror.org/04z7fc725", "country_code": "IN", "type": "education", "lineage": ["https://openalex.org/I4210129261", "https://openalex.org/I4210145302"]}], "countries": ["IN"], "is_corresponding": false, "raw_author_name": "Prem Pais", "raw_affiliation_string": "St John's Medical College and Research Institute, Bangalore, India", "raw_affiliation_strings": ["St John's Medical College and Research Institute, Bangalore, India"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5075405688", "display_name": "Matthew McQueen", "orcid": null}, "institutions": [{"id": "https://openalex.org/I2802834092", "display_name": "Population Health Research Institute", "ror": "https://ror.org/03kwaeq96", "country_code": "CA", "type": "healthcare", "lineage": ["https://openalex.org/I2802834092"]}, {"id": "https://openalex.org/I98251732", "display_name": "McMaster University", "ror": "https://ror.org/02fa3aq29", "country_code": "CA", "type": "education", "lineage": ["https://openalex.org/I98251732"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "Matthew J McQueen", "raw_affiliation_string": "Population Health Research Institute, McMaster University, Hamilton, ON, Canada", "raw_affiliation_strings": ["Population Health Research Institute, McMaster University, Hamilton, ON, Canada"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5076694100", "display_name": "Charles Mondo", "orcid": null}, "institutions": [{"id": "https://openalex.org/I2800872281", "display_name": "Mulago Hospital", "ror": "https://ror.org/02rhp5f96", "country_code": "UG", "type": "healthcare", "lineage": ["https://openalex.org/I2800872281"]}], "countries": ["UG"], "is_corresponding": false, "raw_author_name": "Charles Mondo", "raw_affiliation_string": "Uganda Heart Institute, Mulago Hospital, Kampala, Uganda", "raw_affiliation_strings": ["Uganda Heart Institute, Mulago Hospital, Kampala, Uganda"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5091066335", "display_name": "Albertino Damasceno", "orcid": "https://orcid.org/0000-0003-0925-494X"}, "institutions": [{"id": "https://openalex.org/I16904388", "display_name": "Eduardo Mondlane University", "ror": "https://ror.org/05n8n9378", "country_code": "MZ", "type": "education", "lineage": ["https://openalex.org/I16904388"]}], "countries": ["MZ"], "is_corresponding": false, "raw_author_name": "Albertino Damasceno", "raw_affiliation_string": "Eduardo Mondlane University, Maputo, Mozambique", "raw_affiliation_strings": ["Eduardo Mondlane University, Maputo, Mozambique"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5026568725", "display_name": "Patricio López‐Jaramillo", "orcid": "https://orcid.org/0000-0002-9122-8742"}, "institutions": [{"id": "https://openalex.org/I4210152199", "display_name": "Foscal Hospital", "ror": "https://ror.org/04wnzzd87", "country_code": "CO", "type": "healthcare", "lineage": ["https://openalex.org/I4210152199"]}], "countries": ["CO"], "is_corresponding": false, "raw_author_name": "Patricio Lopez-Jaramillo", "raw_affiliation_string": "Fundacion Oftalmologica de Santander-Clinica Carlos Ardila Lulle (FOSCAL), Bucaramanga, Colombia", "raw_affiliation_strings": ["Fundacion Oftalmologica de Santander-Clinica Carlos Ardila Lulle (FOSCAL), Bucaramanga, Colombia"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5014800121", "display_name": "Graeme J. Hankey", "orcid": "https://orcid.org/0000-0002-6044-7328"}, "institutions": [{"id": "https://openalex.org/I2799740997", "display_name": "Royal Perth Hospital", "ror": "https://ror.org/00zc2xc51", "country_code": "AU", "type": "healthcare", "lineage": ["https://openalex.org/I2799740997", "https://openalex.org/I4388446364"]}], "countries": ["AU"], "is_corresponding": false, "raw_author_name": "Graeme J Hankey", "raw_affiliation_string": "Department of Neurology, Royal Perth Hospital, Perth, WA, Australia", "raw_affiliation_strings": ["Department of Neurology, Royal Perth Hospital, Perth, WA, Australia"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5008606851", "display_name": "Antonio L. Dans", "orcid": null}, "institutions": [{"id": "https://openalex.org/I5791819", "display_name": "University of the Philippines Manila", "ror": "https://ror.org/01rrczv41", "country_code": "PH", "type": "education", "lineage": ["https://openalex.org/I103911934", "https://openalex.org/I5791819"]}], "countries": ["PH"], "is_corresponding": false, "raw_author_name": "Antonio L Dans", "raw_affiliation_string": "College of Medicine, University of Philippines, Manila, Philippines", "raw_affiliation_strings": ["College of Medicine, University of Philippines, Manila, Philippines"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5023361030", "display_name": "Khalid Yusoff", "orcid": null}, "institutions": [{"id": "https://openalex.org/I82724352", "display_name": "Universiti Teknologi MARA", "ror": "https://ror.org/05n8tts92", "country_code": "MY", "type": "education", "lineage": ["https://openalex.org/I4210138650", "https://openalex.org/I82724352"]}], "countries": ["MY"], "is_corresponding": false, "raw_author_name": "Khalid Yusoff", "raw_affiliation_string": "Universiti Teknologi MARA, Shah Alam, Malaysia", "raw_affiliation_strings": ["Universiti Teknologi MARA, Shah Alam, Malaysia"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5044800937", "display_name": "Thomas Truelsen", "orcid": "https://orcid.org/0000-0001-6648-7761"}, "institutions": [{"id": "https://openalex.org/I2801827564", "display_name": "Herlev Hospital", "ror": "https://ror.org/00wys9y90", "country_code": "DK", "type": "healthcare", "lineage": ["https://openalex.org/I2801827564"]}, {"id": "https://openalex.org/I2802567020", "display_name": "Copenhagen University Hospital", "ror": "https://ror.org/05bpbnx46", "country_code": "DK", "type": "healthcare", "lineage": ["https://openalex.org/I2802567020"]}], "countries": ["DK"], "is_corresponding": false, "raw_author_name": "Thomas Truelsen", "raw_affiliation_string": "Copenhagen University Hospital Herlev, Copenhagen, Denmark", "raw_affiliation_strings": ["Copenhagen University Hospital Herlev, Copenhagen, Denmark"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5033104412", "display_name": "Hans‐Christoph Diener", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210119759", "display_name": "Essen University Hospital", "ror": "https://ror.org/02na8dn90", "country_code": "DE", "type": "healthcare", "lineage": ["https://openalex.org/I4210119759"]}], "countries": ["DE"], "is_corresponding": false, "raw_author_name": "Hans-Christoph Diener", "raw_affiliation_string": "Department of Neurology, University Hospital, Essen, Germany", "raw_affiliation_strings": ["Department of Neurology, University Hospital, Essen, Germany"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5016216017", "display_name": "Ralph L. Sacco", "orcid": "https://orcid.org/0000-0003-4629-684X"}, "institutions": [{"id": "https://openalex.org/I145608581", "display_name": "University of Miami", "ror": "https://ror.org/02dgjyy92", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I145608581"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Ralph L Sacco", "raw_affiliation_string": "Miller School of Medicine, University of Miami, Miami, FL, USA", "raw_affiliation_strings": ["Miller School of Medicine, University of Miami, Miami, FL, USA"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5059642395", "display_name": "Danuta Ryglewicz", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210153937", "display_name": "Institute of Psychiatry and Neurology", "ror": "https://ror.org/0468k6j36", "country_code": "PL", "type": "facility", "lineage": ["https://openalex.org/I4210153937"]}], "countries": ["PL"], "is_corresponding": false, "raw_author_name": "Danuta Ryglewicz", "raw_affiliation_string": "Institute of Psychiatry and Neurology, Warsaw, Poland", "raw_affiliation_strings": ["Institute of Psychiatry and Neurology, Warsaw, Poland"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5073289723", "display_name": "Anna Członkowska", "orcid": "https://orcid.org/0000-0002-1956-1866"}, "institutions": [{"id": "https://openalex.org/I4210153937", "display_name": "Institute of Psychiatry and Neurology", "ror": "https://ror.org/0468k6j36", "country_code": "PL", "type": "facility", "lineage": ["https://openalex.org/I4210153937"]}], "countries": ["PL"], "is_corresponding": false, "raw_author_name": "Anna Czlonkowska", "raw_affiliation_string": "Institute of Psychiatry and Neurology, Warsaw, Poland", "raw_affiliation_strings": ["Institute of Psychiatry and Neurology, Warsaw, Poland"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5021425597", "display_name": "Christian Weimar", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210119759", "display_name": "Essen University Hospital", "ror": "https://ror.org/02na8dn90", "country_code": "DE", "type": "healthcare", "lineage": ["https://openalex.org/I4210119759"]}], "countries": ["DE"], "is_corresponding": false, "raw_author_name": "Christian Weimar", "raw_affiliation_string": "Department of Neurology, University Hospital, Essen, Germany", "raw_affiliation_strings": ["Department of Neurology, University Hospital, Essen, Germany"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5021376482", "display_name": "Xingyu Wang", "orcid": "https://orcid.org/0000-0001-8869-6150"}, "institutions": [{"id": "https://openalex.org/I4210129240", "display_name": "Shanghai Institute of Hypertension", "ror": "https://ror.org/038j9sn30", "country_code": "CN", "type": "facility", "lineage": ["https://openalex.org/I4210129240"]}], "countries": ["CN"], "is_corresponding": false, "raw_author_name": "Xingyu Wang", "raw_affiliation_string": "Beijing Hypertension League Institute, Beijing, China", "raw_affiliation_strings": ["Beijing Hypertension League Institute, Beijing, China"]}, {"author_position": "last", "author": {"id": "https://openalex.org/A5018419311", "display_name": "Salim Yusuf", "orcid": "https://orcid.org/0000-0002-9458-139X"}, "institutions": [{"id": "https://openalex.org/I2802834092", "display_name": "Population Health Research Institute", "ror": "https://ror.org/03kwaeq96", "country_code": "CA", "type": "healthcare", "lineage": ["https://openalex.org/I2802834092"]}, {"id": "https://openalex.org/I98251732", "display_name": "McMaster University", "ror": "https://ror.org/02fa3aq29", "country_code": "CA", "type": "education", "lineage": ["https://openalex.org/I98251732"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "Salim Yusuf", "raw_affiliation_string": "Population Health Research Institute, McMaster University, Hamilton, ON, Canada", "raw_affiliation_strings": ["Population Health Research Institute, McMaster University, Hamilton, ON, Canada"]}], "countries_distinct_count": 14, "institutions_distinct_count": 17, "corresponding_author_ids": ["https://openalex.org/A5035977661"], "corresponding_institution_ids": ["https://openalex.org/I2802834092", "https://openalex.org/I98251732", "https://openalex.org/I188760350"], "apc_list": {"value": 6830, "currency": "USD", "value_usd": 6830, "provenance": "doaj"}, "apc_paid": {"value": 6830, "currency": "USD", "value_usd": 6830, "provenance": "doaj"}, "has_fulltext": true, "fulltext_origin": "ngrams", "cited_by_count": 2644, "cited_by_percentile_year": {"min": 99.9, "max": 100.0}, "biblio": {"volume": "376", "issue": "9735", "first_page": "112", "last_page": "123"}, "is_retracted": false, "is_paratext": false, "keywords": [{"keyword": "intracerebral haemorrhagic stroke", "score": 0.6973}, {"keyword": "interstroke study", "score": 0.4026}, {"keyword": "risk factors", "score": 0.3921}, {"keyword": "case-control", "score": 0.25}], "concepts": [{"id": "https://openalex.org/C71924100", "wikidata": "https://www.wikidata.org/wiki/Q11190", "display_name": "Medicine", "level": 0, "score": 0.90962005}, {"id": "https://openalex.org/C2780645631", "wikidata": "https://www.wikidata.org/wiki/Q671554", "display_name": "Stroke (engine)", "level": 2, "score": 0.82413566}, {"id": "https://openalex.org/C156957248", "wikidata": "https://www.wikidata.org/wiki/Q1862216", "display_name": "Odds ratio", "level": 2, "score": 0.61042213}, {"id": "https://openalex.org/C50440223", "wikidata": "https://www.wikidata.org/wiki/Q1475848", "display_name": "Risk factor", "level": 2, "score": 0.53361964}, {"id": "https://openalex.org/C126322002", "wikidata": "https://www.wikidata.org/wiki/Q11180", "display_name": "Internal medicine", "level": 1, "score": 0.51701605}, {"id": "https://openalex.org/C146304588", "wikidata": "https://www.wikidata.org/wiki/Q961652", "display_name": "Case-control study", "level": 2, "score": 0.5135343}, {"id": "https://openalex.org/C2908647359", "wikidata": "https://www.wikidata.org/wiki/Q2625603", "display_name": "Population", "level": 2, "score": 0.48619908}, {"id": "https://openalex.org/C3018755981", "wikidata": "https://www.wikidata.org/wiki/Q12202", "display_name": "Ischaemic stroke", "level": 3, "score": 0.41758457}, {"id": "https://openalex.org/C1862650", "wikidata": "https://www.wikidata.org/wiki/Q186005", "display_name": "Physical therapy", "level": 1, "score": 0.3993664}, {"id": "https://openalex.org/C2779161974", "wikidata": "https://www.wikidata.org/wiki/Q815819", "display_name": "Atrial fibrillation", "level": 2, "score": 0.111358255}, {"id": "https://openalex.org/C99454951", "wikidata": "https://www.wikidata.org/wiki/Q932068", "display_name": "Environmental health", "level": 1, "score": 0.09639418}, {"id": "https://openalex.org/C78519656", "wikidata": "https://www.wikidata.org/wiki/Q101333", "display_name": "Mechanical engineering", "level": 1, "score": 0.0}, {"id": "https://openalex.org/C127413603", "wikidata": "https://www.wikidata.org/wiki/Q11023", "display_name": "Engineering", "level": 0, "score": 0.0}], "mesh": [{"descriptor_ui": "D002545", "descriptor_name": "Brain Ischemia", "qualifier_ui": "Q000150", "qualifier_name": "complications", "is_major_topic": true}, {"descriptor_ui": "D002543", "descriptor_name": "Cerebral Hemorrhage", "qualifier_ui": "Q000150", "qualifier_name": "complications", "is_major_topic": true}, {"descriptor_ui": "D020521", "descriptor_name": "Stroke", "qualifier_ui": "Q000209", "qualifier_name": "etiology", "is_major_topic": true}, {"descriptor_ui": "D001281", "descriptor_name": "Atrial Fibrillation", "qualifier_ui": "Q000150", "qualifier_name": "complications", "is_major_topic": false}, {"descriptor_ui": "D001281", "descriptor_name": "Atrial Fibrillation", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D002545", "descriptor_name": "Brain Ischemia", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D016022", "descriptor_name": "Case-Control Studies", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D002543", "descriptor_name": "Cerebral Hemorrhage", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D019049", "descriptor_name": "Developed Countries", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D003906", "descriptor_name": "Developing Countries", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D005260", "descriptor_name": "Female", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D006801", "descriptor_name": "Humans", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D006973", "descriptor_name": "Hypertension", "qualifier_ui": "Q000150", "qualifier_name": "complications", "is_major_topic": false}, {"descriptor_ui": "D006973", "descriptor_name": "Hypertension", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D008019", "descriptor_name": "Life Style", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D008297", "descriptor_name": "Male", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D008875", "descriptor_name": "Middle Aged", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D009203", "descriptor_name": "Myocardial Infarction", "qualifier_ui": "Q000453", "qualifier_name": "epidemiology", "is_major_topic": false}, {"descriptor_ui": "D009203", "descriptor_name": "Myocardial Infarction", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D009203", "descriptor_name": "Myocardial Infarction", "qualifier_ui": "Q000209", "qualifier_name": "etiology", "is_major_topic": false}, {"descriptor_ui": "D012307", "descriptor_name": "Risk Factors", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D020521", "descriptor_name": "Stroke", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D020521", "descriptor_name": "Stroke", "qualifier_ui": "Q000453", "qualifier_name": "epidemiology", "is_major_topic": false}, {"descriptor_ui": "D049629", "descriptor_name": "Waist-Hip Ratio", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}], "locations_count": 2, "locations": [{"is_oa": false, "landing_page_url": "https://doi.org/10.1016/s0140-6736(10)60834-3", "pdf_url": null, "source": {"id": "https://openalex.org/S49861241", "display_name": "The Lancet", "issn_l": "0140-6736", "issn": ["1474-547X", "0099-5355", "0140-6736"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310320990", "host_organization_name": "Elsevier BV", "host_organization_lineage": ["https://openalex.org/P4310320990"], "host_organization_lineage_names": ["Elsevier BV"], "type": "journal"}, "license": null, "version": null, "is_accepted": false, "is_published": false}, {"is_oa": false, "landing_page_url": "https://pubmed.ncbi.nlm.nih.gov/20561675", "pdf_url": null, "source": {"id": "https://openalex.org/S4306525036", "display_name": "PubMed", "issn_l": null, "issn": null, "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/I1299303238", "host_organization_name": "National Institutes of Health", "host_organization_lineage": ["https://openalex.org/I1299303238"], "host_organization_lineage_names": ["National Institutes of Health"], "type": "repository"}, "license": null, "version": null, "is_accepted": false, "is_published": false}], "best_oa_location": null, "sustainable_development_goals": [{"id": "https://metadata.un.org/sdg/3", "display_name": "Good health and well-being", "score": 0.56}, {"id": "https://metadata.un.org/sdg/1", "display_name": "No poverty", "score": 0.38}], "grants": [], "referenced_works_count": 37, "referenced_works": ["https://openalex.org/W1498763568", "https://openalex.org/W1605997428", "https://openalex.org/W1607804102", "https://openalex.org/W1971651068", "https://openalex.org/W1992510986", "https://openalex.org/W1995466981", "https://openalex.org/W1998234914", "https://openalex.org/W2002003793", "https://openalex.org/W2009389358", "https://openalex.org/W2009551665", "https://openalex.org/W2012820125", "https://openalex.org/W2015746229", "https://openalex.org/W2018784274", "https://openalex.org/W2044872902", "https://openalex.org/W2045642258", "https://openalex.org/W2048664920", "https://openalex.org/W2049327891", "https://openalex.org/W2050767863", "https://openalex.org/W2056068526", "https://openalex.org/W2061233470", "https://openalex.org/W2077187814", "https://openalex.org/W2078103827", "https://openalex.org/W2095955991", "https://openalex.org/W2096029390", "https://openalex.org/W2102227401", "https://openalex.org/W2110983878", "https://openalex.org/W2124565607", "https://openalex.org/W2127427274", "https://openalex.org/W2147009167", "https://openalex.org/W2147013342", "https://openalex.org/W2155121555", "https://openalex.org/W2160280522", "https://openalex.org/W2162805440", "https://openalex.org/W2171575573", "https://openalex.org/W2188950424", "https://openalex.org/W2322333854", "https://openalex.org/W4210973470"], "related_works": ["https://openalex.org/W1546768578", "https://openalex.org/W2907237707", "https://openalex.org/W2564778512", "https://openalex.org/W1998542753", "https://openalex.org/W2148413427", "https://openalex.org/W2350415074", "https://openalex.org/W2784027074", "https://openalex.org/W2041045730", "https://openalex.org/W2004209207", "https://openalex.org/W2114132640"], "ngrams_url": "https://api.openalex.org/works/W2157622195/ngrams", "abstract_inverted_index": {"The": [0], "contribution": [1, 42], "of": [2, 9, 17, 28, 43, 50, 91, 97, 104, 144, 183, 273, 297, 307, 312, 365, 368, 391, 394, 401], "various": [3], "risk": [4, 32, 45, 57, 155, 176, 213, 289, 330, 349, 359, 367], "factors": [5, 33, 46, 58, 177, 290, 331, 350, 360], "to": [6, 24, 47, 276], "the": [7, 26, 41, 48, 54, 142, 157, 298, 320, 366, 389], "burden": [8, 49, 390], "stroke": [10, 35, 60, 87, 152, 180], "worldwide": [11, 72], "is": [12], "unknown,": [13], "particularly": [14], "in": [15, 69], "countries": [16, 71], "low": [18], "and": [19, 30, 36, 52, 61, 77, 94, 106, 113, 121, 125, 129, 137, 149, 172, 259, 271, 344, 376, 378, 382, 398, 411], "middle": [20], "income.": [21], "We": [22, 132], "aimed": [23], "establish": [25], "association": [27, 143], "known": [29], "emerging": [31], "with": [34, 84, 109, 153, 163, 168, 363], "its": [37], "primary": [38], "subtypes,": [39], "assess": [40], "these": [44, 288], "stroke,": [51, 105, 146, 148, 337], "explore": [53], "differences": [55], "between": [56, 73], "for": [59, 111, 141, 178, 205, 217, 241, 280, 292, 300, 326, 335, 351], "myocardial": [62], "infarction.We": [63], "undertook": [64], "a": [65, 118, 122, 383], "standardised": [66], "case-control": [67], "study": [68], "22": [70], "March": [74], "1,": [75], "2007,": [76], "April": [78], "23,": [79], "2010.": [80], "Cases": [81], "were": [82, 107, 332, 347], "patients": [83], "acute": [85], "first": [86, 158], "(within": [88], "5": [89], "days": [90], "symptoms": [92], "onset": [93], "72": [95], "h": [96], "hospital": [98], "admission).": [99], "Controls": [100], "had": [101], "no": [102], "history": [103, 182], "matched": [108], "cases": [110, 160], "age": [112], "sex.": [114], "All": [115], "participants": [116], "completed": [117], "structured": [119], "questionnaire": [120], "physical": [123, 225, 380], "examination,": [124], "most": [126], "provided": [127], "blood": [128, 315, 374], "urine": [130], "samples.": [131], "calculated": [133], "odds": [134], "ratios": [135], "(ORs)": [136], "population-attributable": [138], "risks": [139], "(PARs)": [140], "all": [145, 179, 301, 327, 333], "ischaemic": [147, 164, 336], "intracerebral": [150, 169, 352], "haemorrhagic": [151, 170, 353], "selected": [154], "factors.In": [156], "3000": [159, 173], "(n=2337,": [161], "78%,": [162], "stroke;": [165], "n=663,": [166], "22%,": [167], "stroke)": [171], "controls,": [174], "significant": [175, 334, 348], "were:": [181], "hypertension": [184, 308, 313], "(OR": [185], "2.64,": [186], "99%": [187, 192], "CI": [188, 193, 295], "2.26-3.08;": [189], "PAR": [190, 299, 322], "34.6%,": [191], "30.4-39.1);": [194], "current": [195], "smoking": [196], "(2.09,": [197], "1.75-2.51;": [198], "18.9%,": [199], "15.3-23.1);": [200], "waist-to-hip": [201, 341], "ratio": [202, 272], "(1.65,": [203], "1.36-1.99": [204], "highest": [206, 218, 281], "vs": [207, 219, 282], "lowest": [208, 220, 283], "tertile;": [209, 221, 284], "26.5%,": [210], "18.8-36.0);": [211], "diet": [212], "score": [214], "(1.35,": [215, 261], "1.11-1.64": [216], "18.8%,": [222], "11.2-29.7);": [223], "regular": [224], "activity": [226, 381], "(0.69,": [227], "0.53-0.90;": [228], "28.5%,": [229], "14.5-48.5);": [230], "diabetes": [231], "mellitus": [232], "(1.36,": [233], "1.10-1.68;": [234], "5.0%,": [235], "2.6-9.5);": [236], "alcohol": [237, 345], "intake": [238, 346], "(1.51,": [239], "1.18-1.92": [240], "more": [242], "than": [243], "30": [244], "drinks": [245], "per": [246], "month": [247], "or": [248, 314], "binge": [249], "drinking;": [250], "3.8%,": [251], "0.9-14.4);": [252], "psychosocial": [253], "stress": [254], "(1.30,": [255], "1.06-1.60;": [256], "4.6%,": [257], "2.1-9.6)": [258], "depression": [260], "1.10-1.66;": [262], "5.2%,": [263], "2.7-9.8);": [264], "cardiac": [265], "causes": [266], "(2.38,": [267], "1.77-3.20;": [268], "6.7%,": [269], "4.8-9.1);": [270], "apolipoproteins": [274], "B": [275], "A1": [277], "(1.89,": [278], "1.49-2.40": [279], "24.9%,": [285], "15.7-37.1).": [286], "Collectively,": [287], "accounted": [291], "88.1%": [293], "(99%": [294], "82.3-92.2)": [296], "stroke.": [302, 328, 369], "When": [303], "an": [304], "alternate": [305], "definition": [306], "was": [309, 323], "used": [310], "(history": [311], "pressure": [316, 375], ">160/90": [317], "mm": [318], "Hg),": [319], "combined": [321], "90.3%": [324], "(85.3-93.7)": [325], "These": [329], "whereas": [338], "hypertension,": [339], "smoking,": [340, 377], "ratio,": [342], "diet,": [343, 385], "stroke.Our": [354], "findings": [355], "suggest": [356], "that": [357, 372], "ten": [358], "are": [361], "associated": [362], "90%": [364], "Targeted": [370], "interventions": [371], "reduce": [373, 388], "promote": [379], "healthy": [384], "could": [386], "substantially": [387], "stroke.Canadian": [392], "Institutes": [393], "Health": [395], "Research,": [396], "Heart": [397], "Stroke": [399, 404], "Foundation": [400], "Canada,": [402], "Canadian": [403], "Network,": [405], "Pfizer": [406], "Cardiovascular": [407], "Award,": [408], "Merck,": [409], "AstraZeneca,": [410], "Boehringer": [412], "Ingelheim.": [413]}, "cited_by_api_url": "https://api.openalex.org/works?filter=cites:W2157622195", "counts_by_year": [{"year": 2023, "cited_by_count": 177}, {"year": 2022, "cited_by_count": 220}, {"year": 2021, "cited_by_count": 205}, {"year": 2020, "cited_by_count": 176}, {"year": 2019, "cited_by_count": 195}, {"year": 2018, "cited_by_count": 190}, {"year": 2017, "cited_by_count": 243}, {"year": 2016, "cited_by_count": 216}, {"year": 2015, "cited_by_count": 265}, {"year": 2014, "cited_by_count": 234}, {"year": 2013, "cited_by_count": 187}, {"year": 2012, "cited_by_count": 176}], "updated_date": "2023-12-06T09:20:54.064206", "created_date": "2016-06-24"} +{"id": "https://openalex.org/W2104948944", "doi": "https://doi.org/10.1056/nejmoa0909494", "title": "A Placebo-Controlled Trial of Oral Fingolimod in Relapsing Multiple Sclerosis", "display_name": "A Placebo-Controlled Trial of Oral Fingolimod in Relapsing Multiple Sclerosis", "publication_year": 2010, "publication_date": "2010-02-04", "ids": {"openalex": "https://openalex.org/W2104948944", "doi": "https://doi.org/10.1056/nejmoa0909494", "mag": "2104948944", "pmid": "https://pubmed.ncbi.nlm.nih.gov/20089952"}, "language": "en", "primary_location": {"is_oa": true, "landing_page_url": "https://doi.org/10.1056/nejmoa0909494", "pdf_url": "https://www.nejm.org/doi/pdf/10.1056/NEJMoa0909494?articleTools=true", "source": {"id": "https://openalex.org/S62468778", "display_name": "The New England Journal of Medicine", "issn_l": "0028-4793", "issn": ["0028-4793", "1533-4406"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310320239", "host_organization_name": "Massachusetts Medical Society", "host_organization_lineage": ["https://openalex.org/P4310320239"], "host_organization_lineage_names": ["Massachusetts Medical Society"], "type": "journal"}, "license": null, "version": "publishedVersion", "is_accepted": true, "is_published": true}, "type": "article", "type_crossref": "journal-article", "open_access": {"is_oa": true, "oa_status": "bronze", "oa_url": "https://www.nejm.org/doi/pdf/10.1056/NEJMoa0909494?articleTools=true", "any_repository_has_fulltext": true}, "authorships": [{"author_position": "first", "author": {"id": "https://openalex.org/A5054874316", "display_name": "Ludwig Kappos", "orcid": "https://orcid.org/0000-0003-4175-5509"}, "institutions": [{"id": "https://openalex.org/I2802542264", "display_name": "University Hospital of Basel", "ror": "https://ror.org/04k51q396", "country_code": "CH", "type": "healthcare", "lineage": ["https://openalex.org/I2802542264"]}, {"id": "https://openalex.org/I1850255", "display_name": "University of Basel", "ror": "https://ror.org/02s6k3f65", "country_code": "CH", "type": "education", "lineage": ["https://openalex.org/I1850255"]}], "countries": ["CH"], "is_corresponding": false, "raw_author_name": "Ludwig Kappos", "raw_affiliation_string": "Departments of Neurology and Biomedicine, University Hospital, University of Basel", "raw_affiliation_strings": ["Departments of Neurology and Biomedicine, University Hospital, University of Basel"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5091677974", "display_name": "Ernst Wilhelm Radüe", "orcid": null}, "institutions": [{"id": "https://openalex.org/I1850255", "display_name": "University of Basel", "ror": "https://ror.org/02s6k3f65", "country_code": "CH", "type": "education", "lineage": ["https://openalex.org/I1850255"]}], "countries": ["CH"], "is_corresponding": false, "raw_author_name": "Ernst Wilhelm Radue", "raw_affiliation_string": "Medical Image Analysis Center, University Hospital, University of Basel", "raw_affiliation_strings": ["Medical Image Analysis Center, University Hospital, University of Basel"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5004974593", "display_name": "Paul O’Connor", "orcid": "https://orcid.org/0000-0003-0058-6905"}, "institutions": [{"id": "https://openalex.org/I4210089665", "display_name": "St Michael’s Hospital", "ror": "https://ror.org/008te2062", "country_code": "IE", "type": "healthcare", "lineage": ["https://openalex.org/I4210089665"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Paul O'Connor", "raw_affiliation_string": "St. Michael's Hospital, Toronto", "raw_affiliation_strings": ["St. Michael's Hospital, Toronto"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5076669758", "display_name": "Chris H. Polman", "orcid": null}, "institutions": [{"id": "https://openalex.org/I2802849423", "display_name": "University Medical Center", "ror": "https://ror.org/036pt7h44", "country_code": "US", "type": "healthcare", "lineage": ["https://openalex.org/I2802849423"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Chris Polman", "raw_affiliation_string": "Free University Medical Center, Amsterdam", "raw_affiliation_strings": ["Free University Medical Center, Amsterdam"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5028689651", "display_name": "Reinhard Hohlfeld", "orcid": "https://orcid.org/0000-0002-6302-1488"}, "institutions": [], "countries": [], "is_corresponding": false, "raw_author_name": "Reinhard Hohlfeld", "raw_affiliation_string": "", "raw_affiliation_strings": []}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5061851273", "display_name": "Peter A. Calabresi", "orcid": "https://orcid.org/0000-0002-7776-6472"}, "institutions": [{"id": "https://openalex.org/I4210150714", "display_name": "Johns Hopkins Hospital", "ror": "https://ror.org/05cb1k848", "country_code": "US", "type": "healthcare", "lineage": ["https://openalex.org/I2799853436", "https://openalex.org/I4210150714"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Peter Calabresi", "raw_affiliation_string": "Johns Hopkins Hospital, Baltimore", "raw_affiliation_strings": ["Johns Hopkins Hospital, Baltimore"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5020016996", "display_name": "Krzysztof Selmaj", "orcid": "https://orcid.org/0000-0003-1213-7218"}, "institutions": [{"id": "https://openalex.org/I4210122071", "display_name": "Medical University of Lodz", "ror": "https://ror.org/02t4ekc95", "country_code": "PL", "type": "education", "lineage": ["https://openalex.org/I4210122071"]}], "countries": ["PL"], "is_corresponding": false, "raw_author_name": "Krzysztof Selmaj", "raw_affiliation_string": "Medical University of Lodz, Lodz, Poland", "raw_affiliation_strings": ["Medical University of Lodz, Lodz, Poland"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5048230460", "display_name": "Catherine Agoropoulou", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210104729", "display_name": "Novartis (Netherlands)", "ror": "https://ror.org/01a80cj23", "country_code": "NL", "type": "company", "lineage": ["https://openalex.org/I1283582996", "https://openalex.org/I4210104729"]}], "countries": ["NL"], "is_corresponding": false, "raw_author_name": "Catherine Agoropoulou", "raw_affiliation_string": "Novartis Pharma", "raw_affiliation_strings": ["Novartis Pharma"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5009684493", "display_name": "Małgorzata Leyk", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210104729", "display_name": "Novartis (Netherlands)", "ror": "https://ror.org/01a80cj23", "country_code": "NL", "type": "company", "lineage": ["https://openalex.org/I1283582996", "https://openalex.org/I4210104729"]}], "countries": ["NL"], "is_corresponding": false, "raw_author_name": "Malgorzata Leyk", "raw_affiliation_string": "Novartis Pharma", "raw_affiliation_strings": ["Novartis Pharma"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5047089523", "display_name": "Lixin Zhang-Auberson", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210104729", "display_name": "Novartis (Netherlands)", "ror": "https://ror.org/01a80cj23", "country_code": "NL", "type": "company", "lineage": ["https://openalex.org/I1283582996", "https://openalex.org/I4210104729"]}], "countries": ["NL"], "is_corresponding": false, "raw_author_name": "Lixin Zhang-Auberson", "raw_affiliation_string": "Novartis Pharma", "raw_affiliation_strings": ["Novartis Pharma"]}, {"author_position": "last", "author": {"id": "https://openalex.org/A5073722339", "display_name": "P Burtin", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210104729", "display_name": "Novartis (Netherlands)", "ror": "https://ror.org/01a80cj23", "country_code": "NL", "type": "company", "lineage": ["https://openalex.org/I1283582996", "https://openalex.org/I4210104729"]}], "countries": ["NL"], "is_corresponding": false, "raw_author_name": "Pascale Burtin", "raw_affiliation_string": "Novartis Pharma", "raw_affiliation_strings": ["Novartis Pharma"]}], "countries_distinct_count": 5, "institutions_distinct_count": 7, "corresponding_author_ids": [], "corresponding_institution_ids": [], "apc_list": null, "apc_paid": null, "has_fulltext": true, "fulltext_origin": "ngrams", "cited_by_count": 2293, "cited_by_percentile_year": {"min": 99.9, "max": 100.0}, "biblio": {"volume": "362", "issue": "5", "first_page": "387", "last_page": "401"}, "is_retracted": false, "is_paratext": false, "keywords": [{"keyword": "oral fingolimod", "score": 0.5886}, {"keyword": "placebo-controlled", "score": 0.25}], "concepts": [{"id": "https://openalex.org/C2776036978", "wikidata": "https://www.wikidata.org/wiki/Q425137", "display_name": "Fingolimod", "level": 3, "score": 0.99043286}, {"id": "https://openalex.org/C71924100", "wikidata": "https://www.wikidata.org/wiki/Q11190", "display_name": "Medicine", "level": 0, "score": 0.8951483}, {"id": "https://openalex.org/C2780640218", "wikidata": "https://www.wikidata.org/wiki/Q8277", "display_name": "Multiple sclerosis", "level": 2, "score": 0.87099016}, {"id": "https://openalex.org/C27081682", "wikidata": "https://www.wikidata.org/wiki/Q269829", "display_name": "Placebo", "level": 3, "score": 0.71792376}, {"id": "https://openalex.org/C143409427", "wikidata": "https://www.wikidata.org/wiki/Q161238", "display_name": "Magnetic resonance imaging", "level": 2, "score": 0.59597236}, {"id": "https://openalex.org/C126322002", "wikidata": "https://www.wikidata.org/wiki/Q11180", "display_name": "Internal medicine", "level": 1, "score": 0.46105424}, {"id": "https://openalex.org/C2908698914", "wikidata": "https://www.wikidata.org/wiki/Q2450337", "display_name": "Interferon beta-1a", "level": 4, "score": 0.45331362}, {"id": "https://openalex.org/C2777056448", "wikidata": "https://www.wikidata.org/wiki/Q285166", "display_name": "Oral administration", "level": 2, "score": 0.418495}, {"id": "https://openalex.org/C98274493", "wikidata": "https://www.wikidata.org/wiki/Q128406", "display_name": "Pharmacology", "level": 1, "score": 0.37793812}, {"id": "https://openalex.org/C143998085", "wikidata": "https://www.wikidata.org/wiki/Q162555", "display_name": "Oncology", "level": 1, "score": 0.35167825}, {"id": "https://openalex.org/C203014093", "wikidata": "https://www.wikidata.org/wiki/Q101929", "display_name": "Immunology", "level": 1, "score": 0.24283966}, {"id": "https://openalex.org/C142724271", "wikidata": "https://www.wikidata.org/wiki/Q7208", "display_name": "Pathology", "level": 1, "score": 0.21543941}, {"id": "https://openalex.org/C126838900", "wikidata": "https://www.wikidata.org/wiki/Q77604", "display_name": "Radiology", "level": 1, "score": 0.14079526}, {"id": "https://openalex.org/C2994247566", "wikidata": "https://www.wikidata.org/wiki/Q2450337", "display_name": "Interferon beta", "level": 3, "score": 0.13114345}, {"id": "https://openalex.org/C204787440", "wikidata": "https://www.wikidata.org/wiki/Q188504", "display_name": "Alternative medicine", "level": 2, "score": 0.0}], "mesh": [], "locations_count": 4, "locations": [{"is_oa": true, "landing_page_url": "https://doi.org/10.1056/nejmoa0909494", "pdf_url": "https://www.nejm.org/doi/pdf/10.1056/NEJMoa0909494?articleTools=true", "source": {"id": "https://openalex.org/S62468778", "display_name": "The New England Journal of Medicine", "issn_l": "0028-4793", "issn": ["0028-4793", "1533-4406"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310320239", "host_organization_name": "Massachusetts Medical Society", "host_organization_lineage": ["https://openalex.org/P4310320239"], "host_organization_lineage_names": ["Massachusetts Medical Society"], "type": "journal"}, "license": null, "version": "publishedVersion", "is_accepted": true, "is_published": true}, {"is_oa": false, "landing_page_url": "https://hal.archives-ouvertes.fr/hal-00617764", "pdf_url": null, "source": {"id": "https://openalex.org/S4306402512", "display_name": "HAL (Le Centre pour la Communication Scientifique Directe)", "issn_l": null, "issn": null, "is_oa": true, "is_in_doaj": false, "host_organization": "https://openalex.org/I1294671590", "host_organization_name": "French National Centre for Scientific Research", "host_organization_lineage": ["https://openalex.org/I1294671590"], "host_organization_lineage_names": ["French National Centre for Scientific Research"], "type": "repository"}, "license": null, "version": null, "is_accepted": false, "is_published": false}, {"is_oa": false, "landing_page_url": "https://hal.science/hal-00617764", "pdf_url": null, "source": {"id": "https://openalex.org/S4306402512", "display_name": "HAL (Le Centre pour la Communication Scientifique Directe)", "issn_l": null, "issn": null, "is_oa": true, "is_in_doaj": false, "host_organization": "https://openalex.org/I1294671590", "host_organization_name": "French National Centre for Scientific Research", "host_organization_lineage": ["https://openalex.org/I1294671590"], "host_organization_lineage_names": ["French National Centre for Scientific Research"], "type": "repository"}, "license": null, "version": null, "is_accepted": false, "is_published": false}, {"is_oa": true, "landing_page_url": "http://hdl.handle.net/11858/00-001M-0000-0012-1FF5-A", "pdf_url": "https://pure.mpg.de/pubman/item/item_1130082_1/component/file_1130081/nejmoa0909494%5B1%5D.pdf", "source": {"id": "https://openalex.org/S4306400655", "display_name": "MPG.PuRe (Max Planck Society)", "issn_l": null, "issn": null, "is_oa": true, "is_in_doaj": false, "host_organization": "https://openalex.org/I149899117", "host_organization_name": "Max Planck Society", "host_organization_lineage": ["https://openalex.org/I149899117"], "host_organization_lineage_names": ["Max Planck Society"], "type": "repository"}, "license": null, "version": "publishedVersion", "is_accepted": true, "is_published": true}], "best_oa_location": {"is_oa": true, "landing_page_url": "https://doi.org/10.1056/nejmoa0909494", "pdf_url": "https://www.nejm.org/doi/pdf/10.1056/NEJMoa0909494?articleTools=true", "source": {"id": "https://openalex.org/S62468778", "display_name": "The New England Journal of Medicine", "issn_l": "0028-4793", "issn": ["0028-4793", "1533-4406"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310320239", "host_organization_name": "Massachusetts Medical Society", "host_organization_lineage": ["https://openalex.org/P4310320239"], "host_organization_lineage_names": ["Massachusetts Medical Society"], "type": "journal"}, "license": null, "version": "publishedVersion", "is_accepted": true, "is_published": true}, "sustainable_development_goals": [{"id": "https://metadata.un.org/sdg/3", "display_name": "Good health and well-being", "score": 0.57}], "grants": [], "referenced_works_count": 23, "referenced_works": ["https://openalex.org/W1992656196", "https://openalex.org/W1993746949", "https://openalex.org/W2006985918", "https://openalex.org/W2007541166", "https://openalex.org/W2008278384", "https://openalex.org/W2026657768", "https://openalex.org/W2035208392", "https://openalex.org/W2041823936", "https://openalex.org/W2049363669", "https://openalex.org/W2057391776", "https://openalex.org/W2059921365", "https://openalex.org/W2070532347", "https://openalex.org/W2087015799", "https://openalex.org/W2087589796", "https://openalex.org/W2107365133", "https://openalex.org/W2108813293", "https://openalex.org/W2112165124", "https://openalex.org/W2144958446", "https://openalex.org/W2154089780", "https://openalex.org/W2155245449", "https://openalex.org/W2916040650", "https://openalex.org/W3041047318", "https://openalex.org/W4211254511"], "related_works": ["https://openalex.org/W1716064475", "https://openalex.org/W3192960154", "https://openalex.org/W2083083635", "https://openalex.org/W2890515892", "https://openalex.org/W2607261421", "https://openalex.org/W2185538470", "https://openalex.org/W2928274480", "https://openalex.org/W1742875372", "https://openalex.org/W2119874037", "https://openalex.org/W160847151"], "ngrams_url": "https://api.openalex.org/works/W2104948944/ngrams", "abstract_inverted_index": {"Oral": [0], "fingolimod,": [1], "a": [2], "sphingosine-1-phosphate–receptor": [3], "modulator": [4], "that": [5], "prevents": [6], "the": [7], "egress": [8], "of": [9, 42], "lymphocytes": [10], "from": [11], "lymph": [12], "nodes,": [13], "significantly": [14], "improved": [15], "relapse": [16], "rates": [17], "and": [18, 39], "end": [19], "points": [20], "measured": [21], "on": [22], "magnetic": [23], "resonance": [24], "imaging": [25], "(MRI),": [26], "as": [27], "compared": [28], "with": [29], "either": [30], "placebo": [31], "or": [32], "intramuscular": [33], "interferon": [34], "beta-1a,": [35], "in": [36], "phase": [37], "2": [38], "3": [40], "studies": [41], "multiple": [43], "sclerosis.": [44]}, "cited_by_api_url": "https://api.openalex.org/works?filter=cites:W2104948944", "counts_by_year": [{"year": 2023, "cited_by_count": 88}, {"year": 2022, "cited_by_count": 108}, {"year": 2021, "cited_by_count": 137}, {"year": 2020, "cited_by_count": 157}, {"year": 2019, "cited_by_count": 166}, {"year": 2018, "cited_by_count": 151}, {"year": 2017, "cited_by_count": 161}, {"year": 2016, "cited_by_count": 206}, {"year": 2015, "cited_by_count": 212}, {"year": 2014, "cited_by_count": 213}, {"year": 2013, "cited_by_count": 192}, {"year": 2012, "cited_by_count": 208}], "updated_date": "2023-11-30T13:04:37.630499", "created_date": "2016-06-24"} +{"id": "https://openalex.org/W2071754162", "doi": "https://doi.org/10.1371/journal.pone.0009672", "title": "Source Partitioning Using Stable Isotopes: Coping with Too Much Variation", "display_name": "Source Partitioning Using Stable Isotopes: Coping with Too Much Variation", "publication_year": 2010, "publication_date": "2010-03-12", "ids": {"openalex": "https://openalex.org/W2071754162", "doi": "https://doi.org/10.1371/journal.pone.0009672", "mag": "2071754162", "pmid": "https://pubmed.ncbi.nlm.nih.gov/20300637", "pmcid": "https://www.ncbi.nlm.nih.gov/pmc/articles/2837382"}, "language": "en", "primary_location": {"is_oa": true, "landing_page_url": "https://doi.org/10.1371/journal.pone.0009672", "pdf_url": "https://journals.plos.org/plosone/article/file?id=10.1371/journal.pone.0009672&type=printable", "source": {"id": "https://openalex.org/S202381698", "display_name": "PLOS ONE", "issn_l": "1932-6203", "issn": ["1932-6203"], "is_oa": true, "is_in_doaj": true, "host_organization": "https://openalex.org/P4310315706", "host_organization_name": "Public Library of Science", "host_organization_lineage": ["https://openalex.org/P4310315706"], "host_organization_lineage_names": ["Public Library of Science"], "type": "journal"}, "license": "cc-by", "version": "publishedVersion", "is_accepted": true, "is_published": true}, "type": "article", "type_crossref": "journal-article", "open_access": {"is_oa": true, "oa_status": "gold", "oa_url": "https://journals.plos.org/plosone/article/file?id=10.1371/journal.pone.0009672&type=printable", "any_repository_has_fulltext": true}, "authorships": [{"author_position": "first", "author": {"id": "https://openalex.org/A5042898219", "display_name": "Andrew C. Parnell", "orcid": "https://orcid.org/0000-0001-7956-7939"}, "institutions": [{"id": "https://openalex.org/I100930933", "display_name": "University College Dublin", "ror": "https://ror.org/05m7pjf47", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I100930933", "https://openalex.org/I181231927"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Andrew C. Parnell", "raw_affiliation_string": "School of Mathematical Sciences, University College Dublin, Dublin, Ireland", "raw_affiliation_strings": ["School of Mathematical Sciences, University College Dublin, Dublin, Ireland"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5022890499", "display_name": "Richard Inger", "orcid": "https://orcid.org/0000-0003-1660-3706"}, "institutions": [{"id": "https://openalex.org/I23923803", "display_name": "University of Exeter", "ror": "https://ror.org/03yghzc09", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I23923803"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Richard Inger", "raw_affiliation_string": "Centre for Ecology and Conservation, School of Biosciences, University of Exeter, Penryn, Cornwall, United Kingdom", "raw_affiliation_strings": ["Centre for Ecology and Conservation, School of Biosciences, University of Exeter, Penryn, Cornwall, United Kingdom"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5052126292", "display_name": "Stuart Bearhop", "orcid": "https://orcid.org/0000-0002-5864-0129"}, "institutions": [{"id": "https://openalex.org/I23923803", "display_name": "University of Exeter", "ror": "https://ror.org/03yghzc09", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I23923803"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Stuart Bearhop", "raw_affiliation_string": "Centre for Ecology and Conservation, School of Biosciences, University of Exeter, Penryn, Cornwall, United Kingdom", "raw_affiliation_strings": ["Centre for Ecology and Conservation, School of Biosciences, University of Exeter, Penryn, Cornwall, United Kingdom"]}, {"author_position": "last", "author": {"id": "https://openalex.org/A5018495124", "display_name": "Andrew L. Jackson", "orcid": "https://orcid.org/0000-0001-7334-0434"}, "institutions": [{"id": "https://openalex.org/I205274468", "display_name": "Trinity College Dublin", "ror": "https://ror.org/02tyrky19", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I205274468"]}], "countries": ["IE"], "is_corresponding": true, "raw_author_name": "Andrew L. Jackson", "raw_affiliation_string": "Department of Zoology, School of Natural Sciences, Trinity College Dublin, Dublin, Ireland", "raw_affiliation_strings": ["Department of Zoology, School of Natural Sciences, Trinity College Dublin, Dublin, Ireland"]}], "countries_distinct_count": 2, "institutions_distinct_count": 3, "corresponding_author_ids": ["https://openalex.org/A5018495124"], "corresponding_institution_ids": ["https://openalex.org/I205274468"], "apc_list": {"value": 1805, "currency": "USD", "value_usd": 1805, "provenance": "doaj"}, "apc_paid": {"value": 1805, "currency": "USD", "value_usd": 1805, "provenance": "doaj"}, "has_fulltext": true, "fulltext_origin": "pdf", "cited_by_count": 2273, "cited_by_percentile_year": {"min": 99.9, "max": 100.0}, "biblio": {"volume": "5", "issue": "3", "first_page": "e9672", "last_page": "e9672"}, "is_retracted": false, "is_paratext": false, "keywords": [{"keyword": "stable isotopes", "score": 0.6628}, {"keyword": "variation", "score": 0.3364}], "concepts": [{"id": "https://openalex.org/C79581498", "wikidata": "https://www.wikidata.org/wiki/Q1367530", "display_name": "Suite", "level": 2, "score": 0.67660546}, {"id": "https://openalex.org/C22117777", "wikidata": "https://www.wikidata.org/wiki/Q17148629", "display_name": "Stable isotope ratio", "level": 2, "score": 0.6071187}, {"id": "https://openalex.org/C107673813", "wikidata": "https://www.wikidata.org/wiki/Q812534", "display_name": "Bayesian probability", "level": 2, "score": 0.5897422}, {"id": "https://openalex.org/C41008148", "wikidata": "https://www.wikidata.org/wiki/Q21198", "display_name": "Computer science", "level": 0, "score": 0.50436413}, {"id": "https://openalex.org/C2778334786", "wikidata": "https://www.wikidata.org/wiki/Q1586270", "display_name": "Variation (astronomy)", "level": 2, "score": 0.4680319}, {"id": "https://openalex.org/C51813073", "wikidata": "https://www.wikidata.org/wiki/Q518459", "display_name": "Isotope analysis", "level": 2, "score": 0.45165503}, {"id": "https://openalex.org/C138777275", "wikidata": "https://www.wikidata.org/wiki/Q6884054", "display_name": "Mixing (physics)", "level": 2, "score": 0.4511376}, {"id": "https://openalex.org/C2522767166", "wikidata": "https://www.wikidata.org/wiki/Q2374463", "display_name": "Data science", "level": 1, "score": 0.40296122}, {"id": "https://openalex.org/C149782125", "wikidata": "https://www.wikidata.org/wiki/Q160039", "display_name": "Econometrics", "level": 1, "score": 0.36945972}, {"id": "https://openalex.org/C18903297", "wikidata": "https://www.wikidata.org/wiki/Q7150", "display_name": "Ecology", "level": 1, "score": 0.3393201}, {"id": "https://openalex.org/C86803240", "wikidata": "https://www.wikidata.org/wiki/Q420", "display_name": "Biology", "level": 0, "score": 0.23656878}, {"id": "https://openalex.org/C33923547", "wikidata": "https://www.wikidata.org/wiki/Q395", "display_name": "Mathematics", "level": 0, "score": 0.17136538}, {"id": "https://openalex.org/C205649164", "wikidata": "https://www.wikidata.org/wiki/Q1071", "display_name": "Geography", "level": 0, "score": 0.16370574}, {"id": "https://openalex.org/C154945302", "wikidata": "https://www.wikidata.org/wiki/Q11660", "display_name": "Artificial intelligence", "level": 1, "score": 0.1228683}, {"id": "https://openalex.org/C121332964", "wikidata": "https://www.wikidata.org/wiki/Q413", "display_name": "Physics", "level": 0, "score": 0.11609924}, {"id": "https://openalex.org/C166957645", "wikidata": "https://www.wikidata.org/wiki/Q23498", "display_name": "Archaeology", "level": 1, "score": 0.0}, {"id": "https://openalex.org/C62520636", "wikidata": "https://www.wikidata.org/wiki/Q944", "display_name": "Quantum mechanics", "level": 1, "score": 0.0}, {"id": "https://openalex.org/C44870925", "wikidata": "https://www.wikidata.org/wiki/Q37547", "display_name": "Astrophysics", "level": 1, "score": 0.0}], "mesh": [{"descriptor_ui": "D007554", "descriptor_name": "Isotopes", "qualifier_ui": "Q000737", "qualifier_name": "chemistry", "is_major_topic": true}, {"descriptor_ui": "D000465", "descriptor_name": "Algorithms", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D001499", "descriptor_name": "Bayes Theorem", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D001695", "descriptor_name": "Biology", "qualifier_ui": "Q000379", "qualifier_name": "methods", "is_major_topic": false}, {"descriptor_ui": "D001695", "descriptor_name": "Biology", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D004463", "descriptor_name": "Ecology", "qualifier_ui": "Q000379", "qualifier_name": "methods", "is_major_topic": false}, {"descriptor_ui": "D004463", "descriptor_name": "Ecology", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D004784", "descriptor_name": "Environmental Monitoring", "qualifier_ui": "Q000379", "qualifier_name": "methods", "is_major_topic": false}, {"descriptor_ui": "D004784", "descriptor_name": "Environmental Monitoring", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D007554", "descriptor_name": "Isotopes", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D008390", "descriptor_name": "Markov Chains", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D015233", "descriptor_name": "Models, Statistical", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D008962", "descriptor_name": "Models, Theoretical", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}], "locations_count": 5, "locations": [{"is_oa": true, "landing_page_url": "https://doi.org/10.1371/journal.pone.0009672", "pdf_url": "https://journals.plos.org/plosone/article/file?id=10.1371/journal.pone.0009672&type=printable", "source": {"id": "https://openalex.org/S202381698", "display_name": "PLOS ONE", "issn_l": "1932-6203", "issn": ["1932-6203"], "is_oa": true, "is_in_doaj": true, "host_organization": "https://openalex.org/P4310315706", "host_organization_name": "Public Library of Science", "host_organization_lineage": ["https://openalex.org/P4310315706"], "host_organization_lineage_names": ["Public Library of Science"], "type": "journal"}, "license": "cc-by", "version": "publishedVersion", "is_accepted": true, "is_published": true}, {"is_oa": true, "landing_page_url": "https://europepmc.org/articles/pmc2837382", "pdf_url": "https://europepmc.org/articles/pmc2837382?pdf=render", "source": {"id": "https://openalex.org/S4306400806", "display_name": "Europe PMC (PubMed Central)", "issn_l": null, "issn": null, "is_oa": true, "is_in_doaj": false, "host_organization": "https://openalex.org/I1303153112", "host_organization_name": "European Bioinformatics Institute", "host_organization_lineage": ["https://openalex.org/I1303153112"], "host_organization_lineage_names": ["European Bioinformatics Institute"], "type": "repository"}, "license": "cc-by", "version": "publishedVersion", "is_accepted": true, "is_published": true}, {"is_oa": true, "landing_page_url": "https://hdl.handle.net/10871/8741", "pdf_url": "https://ore.exeter.ac.uk/repository/bitstream/10871/8741/2/PLoS%20ONE%202010.pdf", "source": {"id": "https://openalex.org/S4306401998", "display_name": "Open Research Exeter (University of Exeter)", "issn_l": null, "issn": null, "is_oa": true, "is_in_doaj": false, "host_organization": "https://openalex.org/I23923803", "host_organization_name": "University of Exeter", "host_organization_lineage": ["https://openalex.org/I23923803"], "host_organization_lineage_names": ["University of Exeter"], "type": "repository"}, "license": "cc-by", "version": "publishedVersion", "is_accepted": true, "is_published": true}, {"is_oa": true, "landing_page_url": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2837382", "pdf_url": null, "source": {"id": "https://openalex.org/S2764455111", "display_name": "PubMed Central", "issn_l": null, "issn": null, "is_oa": true, "is_in_doaj": false, "host_organization": "https://openalex.org/I1299303238", "host_organization_name": "National Institutes of Health", "host_organization_lineage": ["https://openalex.org/I1299303238"], "host_organization_lineage_names": ["National Institutes of Health"], "type": "repository"}, "license": null, "version": "publishedVersion", "is_accepted": true, "is_published": true}, {"is_oa": false, "landing_page_url": "https://pubmed.ncbi.nlm.nih.gov/20300637", "pdf_url": null, "source": {"id": "https://openalex.org/S4306525036", "display_name": "PubMed", "issn_l": null, "issn": null, "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/I1299303238", "host_organization_name": "National Institutes of Health", "host_organization_lineage": ["https://openalex.org/I1299303238"], "host_organization_lineage_names": ["National Institutes of Health"], "type": "repository"}, "license": null, "version": null, "is_accepted": false, "is_published": false}], "best_oa_location": {"is_oa": true, "landing_page_url": "https://doi.org/10.1371/journal.pone.0009672", "pdf_url": "https://journals.plos.org/plosone/article/file?id=10.1371/journal.pone.0009672&type=printable", "source": {"id": "https://openalex.org/S202381698", "display_name": "PLOS ONE", "issn_l": "1932-6203", "issn": ["1932-6203"], "is_oa": true, "is_in_doaj": true, "host_organization": "https://openalex.org/P4310315706", "host_organization_name": "Public Library of Science", "host_organization_lineage": ["https://openalex.org/P4310315706"], "host_organization_lineage_names": ["Public Library of Science"], "type": "journal"}, "license": "cc-by", "version": "publishedVersion", "is_accepted": true, "is_published": true}, "sustainable_development_goals": [{"id": "https://metadata.un.org/sdg/15", "display_name": "Life in Land", "score": 0.51}, {"id": "https://metadata.un.org/sdg/12", "display_name": "Responsible consumption and production", "score": 0.21}], "grants": [], "referenced_works_count": 24, "referenced_works": ["https://openalex.org/W1491359801", "https://openalex.org/W1523249066", "https://openalex.org/W1534220275", "https://openalex.org/W1964940068", "https://openalex.org/W1974731359", "https://openalex.org/W1978380534", "https://openalex.org/W1994785855", "https://openalex.org/W2029270868", "https://openalex.org/W2046393091", "https://openalex.org/W2066230046", "https://openalex.org/W2086369298", "https://openalex.org/W2103958724", "https://openalex.org/W2105706314", "https://openalex.org/W2109868247", "https://openalex.org/W2111441677", "https://openalex.org/W2115304230", "https://openalex.org/W2116820101", "https://openalex.org/W2126717016", "https://openalex.org/W2129228305", "https://openalex.org/W2137884063", "https://openalex.org/W2141505056", "https://openalex.org/W2149598290", "https://openalex.org/W2156150005", "https://openalex.org/W2161766528"], "related_works": ["https://openalex.org/W4238894882", "https://openalex.org/W2465616004", "https://openalex.org/W2589291232", "https://openalex.org/W1988675666", "https://openalex.org/W4387120660", "https://openalex.org/W2057087473", "https://openalex.org/W2392714184", "https://openalex.org/W2081245617", "https://openalex.org/W4362663347", "https://openalex.org/W2073999216"], "ngrams_url": "https://api.openalex.org/works/W2071754162/ngrams", "abstract_inverted_index": {"Background": [0], "Stable": [1], "isotope": [2, 65, 134], "analysis": [3, 130], "is": [4, 21], "increasingly": [5], "being": [6], "utilised": [7], "across": [8], "broad": [9], "areas": [10], "of": [11, 18, 24, 31, 55, 60, 76, 122], "ecology": [12], "and": [13, 48, 79, 86, 102, 119], "biology.": [14], "Key": [15], "to": [16, 27, 34, 50, 63, 69, 72], "much": [17], "this": [19, 123], "work": [20], "the": [22, 29, 58], "use": [23], "mixing": [25, 66, 100], "models": [26, 67, 101], "estimate": [28], "proportion": [30], "sources": [32], "contributing": [33], "a": [35, 91, 104], "mixture": [36], "such": [37], "as": [38], "in": [39, 113], "diet": [40], "estimation.": [41], "Methodology": [42], "By": [43], "accurately": [44], "reflecting": [45], "natural": [46], "variation": [47], "uncertainty": [49], "generate": [51], "robust": [52], "probability": [53], "estimates": [54], "source": [56, 107], "proportions,": [57], "application": [59], "Bayesian": [61, 98], "methods": [62], "stable": [64, 133], "promises": [68], "enable": [70], "researchers": [71], "address": [73], "an": [74, 127], "array": [75], "new": [77, 105], "questions,": [78], "approach": [80], "current": [81], "questions": [82], "with": [83], "greater": [84], "insight": [85], "honesty.": [87], "Conclusions": [88], "We": [89], "outline": [90], "framework": [92], "that": [93], "builds": [94], "on": [95], "recently": [96], "published": [97], "isotopic": [99], "present": [103], "open": [106], "R": [108, 114], "package,": [109], "SIAR.": [110], "The": [111], "formulation": [112], "will": [115], "allow": [116], "for": [117, 132], "continued": [118], "rapid": [120], "development": [121], "core": [124], "model": [125], "into": [126], "all-encompassing": [128], "single": [129], "suite": [131], "research.": [135]}, "cited_by_api_url": "https://api.openalex.org/works?filter=cites:W2071754162", "counts_by_year": [{"year": 2023, "cited_by_count": 162}, {"year": 2022, "cited_by_count": 183}, {"year": 2021, "cited_by_count": 204}, {"year": 2020, "cited_by_count": 213}, {"year": 2019, "cited_by_count": 183}, {"year": 2018, "cited_by_count": 207}, {"year": 2017, "cited_by_count": 208}, {"year": 2016, "cited_by_count": 191}, {"year": 2015, "cited_by_count": 211}, {"year": 2014, "cited_by_count": 174}, {"year": 2013, "cited_by_count": 138}, {"year": 2012, "cited_by_count": 105}], "updated_date": "2023-12-03T10:18:49.424449", "created_date": "2016-06-24"} +{"id": "https://openalex.org/W2144543496", "doi": "https://doi.org/10.1086/649858", "title": "Clinical Practice Guidelines for the Management of Cryptococcal Disease: 2010 Update by the Infectious Diseases Society of America", "display_name": "Clinical Practice Guidelines for the Management of Cryptococcal Disease: 2010 Update by the Infectious Diseases Society of America", "publication_year": 2010, "publication_date": "2010-02-01", "ids": {"openalex": "https://openalex.org/W2144543496", "doi": "https://doi.org/10.1086/649858", "mag": "2144543496", "pmid": "https://pubmed.ncbi.nlm.nih.gov/20047480", "pmcid": "https://www.ncbi.nlm.nih.gov/pmc/articles/5826644"}, "language": "en", "primary_location": {"is_oa": true, "landing_page_url": "https://doi.org/10.1086/649858", "pdf_url": "https://academic.oup.com/cid/article-pdf/50/3/291/34128197/50-3-291.pdf", "source": {"id": "https://openalex.org/S72350973", "display_name": "Clinical Infectious Diseases", "issn_l": "1058-4838", "issn": ["1058-4838", "1537-6591"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310311648", "host_organization_name": "Oxford University Press", "host_organization_lineage": ["https://openalex.org/P4310311647", "https://openalex.org/P4310311648"], "host_organization_lineage_names": ["University of Oxford", "Oxford University Press"], "type": "journal"}, "license": null, "version": "publishedVersion", "is_accepted": true, "is_published": true}, "type": "article", "type_crossref": "journal-article", "open_access": {"is_oa": true, "oa_status": "bronze", "oa_url": "https://academic.oup.com/cid/article-pdf/50/3/291/34128197/50-3-291.pdf", "any_repository_has_fulltext": true}, "authorships": [{"author_position": "first", "author": {"id": "https://openalex.org/A5028777325", "display_name": "John R. Perfect", "orcid": "https://orcid.org/0000-0002-8742-3676"}, "institutions": [{"id": "https://openalex.org/I4210126298", "display_name": "Duke Medical Center", "ror": "https://ror.org/03njmea73", "country_code": "US", "type": "healthcare", "lineage": ["https://openalex.org/I4210126298", "https://openalex.org/I4210144876"]}], "countries": ["US"], "is_corresponding": true, "raw_author_name": "John R. Perfect", "raw_affiliation_string": "Division of Infectious Diseases, Duke University Medical Center, Durham, North Carolina", "raw_affiliation_strings": ["Division of Infectious Diseases, Duke University Medical Center, Durham, North Carolina"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5043299051", "display_name": "William E. Dismukes", "orcid": null}, "institutions": [{"id": "https://openalex.org/I32389192", "display_name": "University of Alabama at Birmingham", "ror": "https://ror.org/008s83205", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I2800507078", "https://openalex.org/I32389192"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "William E. Dismukes", "raw_affiliation_string": "Division of Infectious Diseases, University of Alabama-Birmingham", "raw_affiliation_strings": ["Division of Infectious Diseases, University of Alabama-Birmingham"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5059831856", "display_name": "Françoise Dromer", "orcid": "https://orcid.org/0000-0003-1671-1475"}, "institutions": [{"id": "https://openalex.org/I157536573", "display_name": "Institut Pasteur", "ror": "https://ror.org/0495fxg12", "country_code": "FR", "type": "nonprofit", "lineage": ["https://openalex.org/I157536573"]}], "countries": ["FR"], "is_corresponding": false, "raw_author_name": "Francoise Dromer", "raw_affiliation_string": " Institut Pasteur, Centre National de Référence Mycologie et Antifongiques, Unité de Mycologie Moleculaire, Paris, France", "raw_affiliation_strings": [" Institut Pasteur, Centre National de Référence Mycologie et Antifongiques, Unité de Mycologie Moleculaire, Paris, France"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5041785259", "display_name": "David L. Goldman", "orcid": null}, "institutions": [{"id": "https://openalex.org/I129975664", "display_name": "Albert Einstein College of Medicine", "ror": "https://ror.org/05cf8a891", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I129975664", "https://openalex.org/I4210112371"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "David L. Goldman", "raw_affiliation_string": " Department of Pediatric Infectious Diseases, Albert Einstein College of Medicine, Bronx, New York", "raw_affiliation_strings": [" Department of Pediatric Infectious Diseases, Albert Einstein College of Medicine, Bronx, New York"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5039293604", "display_name": "John R. Graybill", "orcid": null}, "institutions": [{"id": "https://openalex.org/I308582824", "display_name": "Murphy Oil Corporation (United States)", "ror": "https://ror.org/00semp387", "country_code": "US", "type": "company", "lineage": ["https://openalex.org/I308582824"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "John R. Graybill", "raw_affiliation_string": "Division of Infectious Diseases, University of Texas San Antonio, Audie L. Murphy Veterans Affairs Hospital, San Antonio", "raw_affiliation_strings": ["Division of Infectious Diseases, University of Texas San Antonio, Audie L. Murphy Veterans Affairs Hospital, San Antonio"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5087595130", "display_name": "Richard J. Hamill", "orcid": null}, "institutions": [], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Richard J. Hamill", "raw_affiliation_string": " Division of Infectious Diseases, Veteran's Affairs (VA) Medical Center, Houston, Texas", "raw_affiliation_strings": [" Division of Infectious Diseases, Veteran's Affairs (VA) Medical Center, Houston, Texas"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5034898456", "display_name": "Thomas S. Harrison", "orcid": null}, "institutions": [{"id": "https://openalex.org/I1310053265", "display_name": "St George's Hospital", "ror": "https://ror.org/0001ke483", "country_code": "GB", "type": "healthcare", "lineage": ["https://openalex.org/I1310053265", "https://openalex.org/I2801196673"]}, {"id": "https://openalex.org/I165862685", "display_name": "St George's, University of London", "ror": "https://ror.org/040f08y74", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I124357947", "https://openalex.org/I165862685"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Thomas S. Harrison", "raw_affiliation_string": "Department of Infectious Diseases, St. George's Hospital Medical School, London, United Kingdom.", "raw_affiliation_strings": ["Department of Infectious Diseases, St. George's Hospital Medical School, London, United Kingdom."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5061402646", "display_name": "Robert A. Larsen", "orcid": null}, "institutions": [{"id": "https://openalex.org/I1174212", "display_name": "University of Southern California", "ror": "https://ror.org/03taz7m60", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I1174212"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Robert A. Larsen", "raw_affiliation_string": " Department of Medicine, University of Southern California School of Medicine, Los Angeles", "raw_affiliation_strings": [" Department of Medicine, University of Southern California School of Medicine, Los Angeles"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5061027575", "display_name": "Olivier Lortholary", "orcid": "https://orcid.org/0000-0002-8325-8060"}, "institutions": [{"id": "https://openalex.org/I157536573", "display_name": "Institut Pasteur", "ror": "https://ror.org/0495fxg12", "country_code": "FR", "type": "nonprofit", "lineage": ["https://openalex.org/I157536573"]}, {"id": "https://openalex.org/I1288880153", "display_name": "Hôpital Necker-Enfants Malades", "ror": "https://ror.org/05tr67282", "country_code": "FR", "type": "healthcare", "lineage": ["https://openalex.org/I1288880153", "https://openalex.org/I4210120235"]}, {"id": "https://openalex.org/I204730241", "display_name": "Université Paris Cité", "ror": "https://ror.org/05f82e368", "country_code": "FR", "type": "education", "lineage": ["https://openalex.org/I204730241"]}], "countries": ["FR"], "is_corresponding": false, "raw_author_name": "Olivier Lortholary", "raw_affiliation_string": " Institut Pasteur, Centre National de Référence Mycologie et Antifongiques, Unité de Mycologie Moleculaire, Paris, France;  Université Paris-Descartes, Service des Maladies Infectieuses et Tropicales, Hópital Necker-Enfants Malades, Centre d'Infectiologie Necker-Pasteur, Paris, France", "raw_affiliation_strings": [" Institut Pasteur, Centre National de Référence Mycologie et Antifongiques, Unité de Mycologie Moleculaire, Paris, France", " Université Paris-Descartes, Service des Maladies Infectieuses et Tropicales, Hópital Necker-Enfants Malades, Centre d'Infectiologie Necker-Pasteur, Paris, France"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5081382082", "display_name": "M. Hong Nguyen", "orcid": "https://orcid.org/0000-0002-4252-8319"}, "institutions": [{"id": "https://openalex.org/I170201317", "display_name": "University of Pittsburgh", "ror": "https://ror.org/01an3r305", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I170201317"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Minh Hong Nguyen", "raw_affiliation_string": "Division of Infectious Diseases, University of Pittsburgh College of Medicine, Pittsburgh, Pennsylvania", "raw_affiliation_strings": ["Division of Infectious Diseases, University of Pittsburgh College of Medicine, Pittsburgh, Pennsylvania"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5004769805", "display_name": "Peter G. Pappas", "orcid": null}, "institutions": [{"id": "https://openalex.org/I32389192", "display_name": "University of Alabama at Birmingham", "ror": "https://ror.org/008s83205", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I2800507078", "https://openalex.org/I32389192"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Peter G. Pappas", "raw_affiliation_string": "Division of Infectious Diseases, University of Alabama-Birmingham", "raw_affiliation_strings": ["Division of Infectious Diseases, University of Alabama-Birmingham"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5058226785", "display_name": "William G. Powderly", "orcid": "https://orcid.org/0000-0001-7808-3086"}, "institutions": [{"id": "https://openalex.org/I100930933", "display_name": "University College Dublin", "ror": "https://ror.org/05m7pjf47", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I100930933", "https://openalex.org/I181231927"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "William G. Powderly", "raw_affiliation_string": "University College; Dublin Ireland", "raw_affiliation_strings": ["University College; Dublin Ireland"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5079624082", "display_name": "Nina Singh", "orcid": "https://orcid.org/0000-0002-3690-2327"}, "institutions": [{"id": "https://openalex.org/I2801460292", "display_name": "Harper University Hospital", "ror": "https://ror.org/00sxe0e68", "country_code": "US", "type": "healthcare", "lineage": ["https://openalex.org/I2800326109", "https://openalex.org/I2801460292"]}, {"id": "https://openalex.org/I185443292", "display_name": "Wayne State University", "ror": "https://ror.org/01070mq45", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I185443292"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Nina Singh", "raw_affiliation_string": "Wayne State University/Harper Hospital, Detroit, Michigan.", "raw_affiliation_strings": ["Wayne State University/Harper Hospital, Detroit, Michigan."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5054286469", "display_name": "Jack D. Sobel", "orcid": "https://orcid.org/0000-0002-5589-4609"}, "institutions": [{"id": "https://openalex.org/I2801460292", "display_name": "Harper University Hospital", "ror": "https://ror.org/00sxe0e68", "country_code": "US", "type": "healthcare", "lineage": ["https://openalex.org/I2800326109", "https://openalex.org/I2801460292"]}, {"id": "https://openalex.org/I185443292", "display_name": "Wayne State University", "ror": "https://ror.org/01070mq45", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I185443292"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Jack D. Sobel", "raw_affiliation_string": "Wayne State University/Harper Hospital, Detroit, Michigan.", "raw_affiliation_strings": ["Wayne State University/Harper Hospital, Detroit, Michigan."]}, {"author_position": "last", "author": {"id": "https://openalex.org/A5022370216", "display_name": "Tania C. Sorrell", "orcid": "https://orcid.org/0000-0001-9460-0960"}, "institutions": [{"id": "https://openalex.org/I129604602", "display_name": "University of Sydney", "ror": "https://ror.org/0384j8v12", "country_code": "AU", "type": "education", "lineage": ["https://openalex.org/I129604602"]}], "countries": ["AU"], "is_corresponding": false, "raw_author_name": "Tania C. Sorrell", "raw_affiliation_string": " Centre for Infectious Diseases and Microbiology, University of Sydney at Westmead, Sydney, Australia", "raw_affiliation_strings": [" Centre for Infectious Diseases and Microbiology, University of Sydney at Westmead, Sydney, Australia"]}], "countries_distinct_count": 5, "institutions_distinct_count": 15, "corresponding_author_ids": ["https://openalex.org/A5028777325"], "corresponding_institution_ids": ["https://openalex.org/I4210126298"], "apc_list": {"value": 4320, "currency": "USD", "value_usd": 4320, "provenance": "doaj"}, "apc_paid": {"value": 4320, "currency": "USD", "value_usd": 4320, "provenance": "doaj"}, "has_fulltext": false, "cited_by_count": 2122, "cited_by_percentile_year": {"min": 99.9, "max": 100.0}, "biblio": {"volume": "50", "issue": "3", "first_page": "291", "last_page": "322"}, "is_retracted": false, "is_paratext": false, "keywords": [{"keyword": "cryptococcal disease", "score": 0.8556}, {"keyword": "clinical practice guidelines", "score": 0.3635}, {"keyword": "infectious diseases society", "score": 0.3554}, {"keyword": "clinical practice", "score": 0.2707}], "concepts": [{"id": "https://openalex.org/C2779413141", "wikidata": "https://www.wikidata.org/wiki/Q1470140", "display_name": "Cryptococcosis", "level": 2, "score": 0.8450383}, {"id": "https://openalex.org/C71924100", "wikidata": "https://www.wikidata.org/wiki/Q11190", "display_name": "Medicine", "level": 0, "score": 0.8307383}, {"id": "https://openalex.org/C2779778235", "wikidata": "https://www.wikidata.org/wiki/Q539986", "display_name": "Immune reconstitution inflammatory syndrome", "level": 5, "score": 0.81853914}, {"id": "https://openalex.org/C2780651595", "wikidata": "https://www.wikidata.org/wiki/Q411478", "display_name": "Fluconazole", "level": 3, "score": 0.64438725}, {"id": "https://openalex.org/C2777328456", "wikidata": "https://www.wikidata.org/wiki/Q238490", "display_name": "Flucytosine", "level": 4, "score": 0.6028642}, {"id": "https://openalex.org/C2778621254", "wikidata": "https://www.wikidata.org/wiki/Q2346415", "display_name": "Meningoencephalitis", "level": 2, "score": 0.59767985}, {"id": "https://openalex.org/C177713679", "wikidata": "https://www.wikidata.org/wiki/Q679690", "display_name": "Intensive care medicine", "level": 1, "score": 0.58169484}, {"id": "https://openalex.org/C2778952914", "wikidata": "https://www.wikidata.org/wiki/Q309498", "display_name": "Cryptococcus", "level": 2, "score": 0.54287726}, {"id": "https://openalex.org/C2779629538", "wikidata": "https://www.wikidata.org/wiki/Q412223", "display_name": "Amphotericin B", "level": 3, "score": 0.53595287}, {"id": "https://openalex.org/C2779286289", "wikidata": "https://www.wikidata.org/wiki/Q149791", "display_name": "Cryptococcus gattii", "level": 3, "score": 0.49812603}, {"id": "https://openalex.org/C203014093", "wikidata": "https://www.wikidata.org/wiki/Q101929", "display_name": "Immunology", "level": 1, "score": 0.49699023}, {"id": "https://openalex.org/C524204448", "wikidata": "https://www.wikidata.org/wiki/Q788926", "display_name": "Infectious disease (medical specialty)", "level": 3, "score": 0.43597233}, {"id": "https://openalex.org/C2779134260", "wikidata": "https://www.wikidata.org/wiki/Q12136", "display_name": "Disease", "level": 2, "score": 0.39056867}, {"id": "https://openalex.org/C126322002", "wikidata": "https://www.wikidata.org/wiki/Q11180", "display_name": "Internal medicine", "level": 1, "score": 0.32003656}, {"id": "https://openalex.org/C3013748606", "wikidata": "https://www.wikidata.org/wiki/Q15787", "display_name": "Human immunodeficiency virus (HIV)", "level": 2, "score": 0.23900056}, {"id": "https://openalex.org/C16005928", "wikidata": "https://www.wikidata.org/wiki/Q171171", "display_name": "Dermatology", "level": 1, "score": 0.22921193}, {"id": "https://openalex.org/C2779548794", "wikidata": "https://www.wikidata.org/wiki/Q578726", "display_name": "Antifungal", "level": 2, "score": 0.201556}, {"id": "https://openalex.org/C2993143319", "wikidata": "https://www.wikidata.org/wiki/Q583050", "display_name": "Antiretroviral therapy", "level": 4, "score": 0.16778731}, {"id": "https://openalex.org/C142462285", "wikidata": "https://www.wikidata.org/wiki/Q2528140", "display_name": "Viral load", "level": 3, "score": 0.15489352}, {"id": "https://openalex.org/C86803240", "wikidata": "https://www.wikidata.org/wiki/Q420", "display_name": "Biology", "level": 0, "score": 0.07502103}, {"id": "https://openalex.org/C89423630", "wikidata": "https://www.wikidata.org/wiki/Q7193", "display_name": "Microbiology", "level": 1, "score": 0.0}], "mesh": [{"descriptor_ui": "D019090", "descriptor_name": "Case Management", "qualifier_ui": "Q000592", "qualifier_name": "standards", "is_major_topic": true}, {"descriptor_ui": "D003453", "descriptor_name": "Cryptococcosis", "qualifier_ui": "Q000628", "qualifier_name": "therapy", "is_major_topic": true}, {"descriptor_ui": "D003453", "descriptor_name": "Cryptococcosis", "qualifier_ui": "Q000175", "qualifier_name": "diagnosis", "is_major_topic": true}, {"descriptor_ui": "D000935", "descriptor_name": "Antifungal Agents", "qualifier_ui": "Q000627", "qualifier_name": "therapeutic use", "is_major_topic": false}, {"descriptor_ui": "D000935", "descriptor_name": "Antifungal Agents", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D019090", "descriptor_name": "Case Management", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D002648", "descriptor_name": "Child", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D002675", "descriptor_name": "Child, Preschool", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D003453", "descriptor_name": "Cryptococcosis", "qualifier_ui": "Q000150", "qualifier_name": "complications", "is_major_topic": false}, {"descriptor_ui": "D003453", "descriptor_name": "Cryptococcosis", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D005260", "descriptor_name": "Female", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D006801", "descriptor_name": "Humans", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D019586", "descriptor_name": "Intracranial Hypertension", "qualifier_ui": "Q000601", "qualifier_name": "surgery", "is_major_topic": false}, {"descriptor_ui": "D019586", "descriptor_name": "Intracranial Hypertension", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D011247", "descriptor_name": "Pregnancy", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D014481", "descriptor_name": "United States", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}], "locations_count": 6, "locations": [{"is_oa": true, "landing_page_url": "https://doi.org/10.1086/649858", "pdf_url": "https://academic.oup.com/cid/article-pdf/50/3/291/34128197/50-3-291.pdf", "source": {"id": "https://openalex.org/S72350973", "display_name": "Clinical Infectious Diseases", "issn_l": "1058-4838", "issn": ["1058-4838", "1537-6591"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310311648", "host_organization_name": "Oxford University Press", "host_organization_lineage": ["https://openalex.org/P4310311647", "https://openalex.org/P4310311648"], "host_organization_lineage_names": ["University of Oxford", "Oxford University Press"], "type": "journal"}, "license": null, "version": "publishedVersion", "is_accepted": true, "is_published": true}, {"is_oa": true, "landing_page_url": "http://hdl.handle.net/10161/4137", "pdf_url": "https://dukespace.lib.duke.edu/dspace/bitstream/10161/4137/1/273500300001.pdf", "source": {"id": "https://openalex.org/S4306400687", "display_name": "DukeSpace (Duke University)", "issn_l": null, "issn": null, "is_oa": true, "is_in_doaj": false, "host_organization": "https://openalex.org/I170897317", "host_organization_name": "Duke University", "host_organization_lineage": ["https://openalex.org/I170897317"], "host_organization_lineage_names": ["Duke University"], "type": "repository"}, "license": null, "version": "publishedVersion", "is_accepted": true, "is_published": true}, {"is_oa": true, "landing_page_url": "https://europepmc.org/articles/pmc5826644", "pdf_url": "https://europepmc.org/articles/pmc5826644?pdf=render", "source": {"id": "https://openalex.org/S4306400806", "display_name": "Europe PMC (PubMed Central)", "issn_l": null, "issn": null, "is_oa": true, "is_in_doaj": false, "host_organization": "https://openalex.org/I1303153112", "host_organization_name": "European Bioinformatics Institute", "host_organization_lineage": ["https://openalex.org/I1303153112"], "host_organization_lineage_names": ["European Bioinformatics Institute"], "type": "repository"}, "license": null, "version": "acceptedVersion", "is_accepted": true, "is_published": false}, {"is_oa": true, "landing_page_url": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5826644", "pdf_url": null, "source": {"id": "https://openalex.org/S2764455111", "display_name": "PubMed Central", "issn_l": null, "issn": null, "is_oa": true, "is_in_doaj": false, "host_organization": "https://openalex.org/I1299303238", "host_organization_name": "National Institutes of Health", "host_organization_lineage": ["https://openalex.org/I1299303238"], "host_organization_lineage_names": ["National Institutes of Health"], "type": "repository"}, "license": null, "version": "acceptedVersion", "is_accepted": true, "is_published": false}, {"is_oa": true, "landing_page_url": "http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.477.491", "pdf_url": "http://dukespace.lib.duke.edu/dspace/bitstream/handle/10161/4137/273500300001.pdf?sequence=1", "source": {"id": "https://openalex.org/S4306400349", "display_name": "CiteSeer X (The Pennsylvania State University)", "issn_l": null, "issn": null, "is_oa": true, "is_in_doaj": false, "host_organization": "https://openalex.org/I130769515", "host_organization_name": "Pennsylvania State University", "host_organization_lineage": ["https://openalex.org/I130769515"], "host_organization_lineage_names": ["Pennsylvania State University"], "type": "repository"}, "license": null, "version": "submittedVersion", "is_accepted": false, "is_published": false}, {"is_oa": false, "landing_page_url": "https://pubmed.ncbi.nlm.nih.gov/20047480", "pdf_url": null, "source": {"id": "https://openalex.org/S4306525036", "display_name": "PubMed", "issn_l": null, "issn": null, "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/I1299303238", "host_organization_name": "National Institutes of Health", "host_organization_lineage": ["https://openalex.org/I1299303238"], "host_organization_lineage_names": ["National Institutes of Health"], "type": "repository"}, "license": null, "version": null, "is_accepted": false, "is_published": false}], "best_oa_location": {"is_oa": true, "landing_page_url": "https://doi.org/10.1086/649858", "pdf_url": "https://academic.oup.com/cid/article-pdf/50/3/291/34128197/50-3-291.pdf", "source": {"id": "https://openalex.org/S72350973", "display_name": "Clinical Infectious Diseases", "issn_l": "1058-4838", "issn": ["1058-4838", "1537-6591"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310311648", "host_organization_name": "Oxford University Press", "host_organization_lineage": ["https://openalex.org/P4310311647", "https://openalex.org/P4310311648"], "host_organization_lineage_names": ["University of Oxford", "Oxford University Press"], "type": "journal"}, "license": null, "version": "publishedVersion", "is_accepted": true, "is_published": true}, "sustainable_development_goals": [{"id": "https://metadata.un.org/sdg/3", "display_name": "Good health and well-being", "score": 0.8}], "grants": [], "referenced_works_count": 173, "referenced_works": ["https://openalex.org/W1499396787", "https://openalex.org/W1518879674", "https://openalex.org/W1963653528", "https://openalex.org/W1963966543", "https://openalex.org/W1964114235", "https://openalex.org/W1964188845", "https://openalex.org/W1964196564", "https://openalex.org/W1964340447", "https://openalex.org/W1965181237", "https://openalex.org/W1965953751", "https://openalex.org/W1966945089", "https://openalex.org/W1971844128", "https://openalex.org/W1972416376", "https://openalex.org/W1975776079", "https://openalex.org/W1976775531", "https://openalex.org/W1977238920", "https://openalex.org/W1979894181", "https://openalex.org/W1983734687", "https://openalex.org/W1983798558", "https://openalex.org/W1984472147", "https://openalex.org/W1985081494", "https://openalex.org/W1988626050", "https://openalex.org/W1988764370", "https://openalex.org/W1992472658", "https://openalex.org/W1994951476", "https://openalex.org/W1996714125", "https://openalex.org/W1997032499", "https://openalex.org/W1997545409", "https://openalex.org/W1998524987", "https://openalex.org/W1999897534", "https://openalex.org/W2001501762", "https://openalex.org/W2006515248", "https://openalex.org/W2009109638", "https://openalex.org/W2012369574", "https://openalex.org/W2014162283", "https://openalex.org/W2017312116", "https://openalex.org/W2017854924", "https://openalex.org/W2018552263", "https://openalex.org/W2019976746", "https://openalex.org/W2022352316", "https://openalex.org/W2023426779", "https://openalex.org/W2030000278", "https://openalex.org/W2030925037", "https://openalex.org/W2031605723", "https://openalex.org/W2032321007", "https://openalex.org/W2036149567", "https://openalex.org/W2037490208", "https://openalex.org/W2039576546", "https://openalex.org/W2043524178", "https://openalex.org/W2046033696", "https://openalex.org/W2049849256", "https://openalex.org/W2050100639", "https://openalex.org/W2050675715", "https://openalex.org/W2051185073", "https://openalex.org/W2052628378", "https://openalex.org/W2053323769", "https://openalex.org/W2053512987", "https://openalex.org/W2056104447", "https://openalex.org/W2058787796", "https://openalex.org/W2058895756", "https://openalex.org/W2059859053", "https://openalex.org/W2060815342", "https://openalex.org/W2062905473", "https://openalex.org/W2064009861", "https://openalex.org/W2068402403", "https://openalex.org/W2069403713", "https://openalex.org/W2069415770", "https://openalex.org/W2069625203", "https://openalex.org/W2072662158", "https://openalex.org/W2073263704", "https://openalex.org/W2073363331", "https://openalex.org/W2073489986", "https://openalex.org/W2075559398", "https://openalex.org/W2079513681", "https://openalex.org/W2082791803", "https://openalex.org/W2083620969", "https://openalex.org/W2084002265", "https://openalex.org/W2084004341", "https://openalex.org/W2084080689", "https://openalex.org/W2084308042", "https://openalex.org/W2086996616", "https://openalex.org/W2087548480", "https://openalex.org/W2088679003", "https://openalex.org/W2088741787", "https://openalex.org/W2088894189", "https://openalex.org/W2090411448", "https://openalex.org/W2090838556", "https://openalex.org/W2092024253", "https://openalex.org/W2093534814", "https://openalex.org/W2093995725", "https://openalex.org/W2099550951", "https://openalex.org/W2102638960", "https://openalex.org/W2103561828", "https://openalex.org/W2104376815", "https://openalex.org/W2104963769", "https://openalex.org/W2105740331", "https://openalex.org/W2107409415", "https://openalex.org/W2107864049", "https://openalex.org/W2110934408", "https://openalex.org/W2110976848", "https://openalex.org/W2112956661", "https://openalex.org/W2114551279", "https://openalex.org/W2114708714", "https://openalex.org/W2116333471", "https://openalex.org/W2116837564", "https://openalex.org/W2119079785", "https://openalex.org/W2119229753", "https://openalex.org/W2121087098", "https://openalex.org/W2121652735", "https://openalex.org/W2124554513", "https://openalex.org/W2124695690", "https://openalex.org/W2125105086", "https://openalex.org/W2125247264", "https://openalex.org/W2127436244", "https://openalex.org/W2128311713", "https://openalex.org/W2129833429", "https://openalex.org/W2130316654", "https://openalex.org/W2136584569", "https://openalex.org/W2136955446", "https://openalex.org/W2138275441", "https://openalex.org/W2138482144", "https://openalex.org/W2139368207", "https://openalex.org/W2139861350", "https://openalex.org/W2140554017", "https://openalex.org/W2140769127", "https://openalex.org/W2140847782", "https://openalex.org/W2141305342", "https://openalex.org/W2143126475", "https://openalex.org/W2143732246", "https://openalex.org/W2143769286", "https://openalex.org/W2146499128", "https://openalex.org/W2147038307", "https://openalex.org/W2147094739", "https://openalex.org/W2147720061", "https://openalex.org/W2148002357", "https://openalex.org/W2150467673", "https://openalex.org/W2150702714", "https://openalex.org/W2151045295", "https://openalex.org/W2151204504", "https://openalex.org/W2151531790", "https://openalex.org/W2151631204", "https://openalex.org/W2152260457", "https://openalex.org/W2152684781", "https://openalex.org/W2153871762", "https://openalex.org/W2154671988", "https://openalex.org/W2156629248", "https://openalex.org/W2157182877", "https://openalex.org/W2157408505", "https://openalex.org/W2160765050", "https://openalex.org/W2162742103", "https://openalex.org/W2162864885", "https://openalex.org/W2165882694", "https://openalex.org/W2167794954", "https://openalex.org/W2169466308", "https://openalex.org/W2171825111", "https://openalex.org/W2172207388", "https://openalex.org/W2216932747", "https://openalex.org/W2312660835", "https://openalex.org/W2313221814", "https://openalex.org/W2316588957", "https://openalex.org/W2317790888", "https://openalex.org/W2319879113", "https://openalex.org/W2328323126", "https://openalex.org/W2331249764", "https://openalex.org/W2332168013", "https://openalex.org/W2334134107", "https://openalex.org/W2339763729", "https://openalex.org/W2409401977", "https://openalex.org/W4211124143", "https://openalex.org/W4238081971", "https://openalex.org/W4241275127", "https://openalex.org/W4255021076", "https://openalex.org/W4256703099"], "related_works": ["https://openalex.org/W3011865304", "https://openalex.org/W2043524178", "https://openalex.org/W2120161557", "https://openalex.org/W2112675507", "https://openalex.org/W2982049191", "https://openalex.org/W4366492176", "https://openalex.org/W3029270694", "https://openalex.org/W3091083289", "https://openalex.org/W4214669534", "https://openalex.org/W2132760034"], "ngrams_url": "https://api.openalex.org/works/W2144543496/ngrams", "abstract_inverted_index": {"Cryptococcosis": [0, 184], "is": [1, 36, 203, 221], "a": [2, 37, 145, 186], "global": [3], "invasive": [4], "mycosis": [5], "associated": [6], "with": [7, 85, 181, 190], "significant": [8], "morbidity": [9], "and": [10, 31, 59, 62, 83, 126, 147, 160, 168, 216], "mortality.": [11], "These": [12], "guidelines": [13, 28], "for": [14, 69, 90, 100, 138], "its": [15], "management": [16, 41, 91, 111, 130, 188], "have": [17, 132], "been": [18, 105, 133], "built": [19], "on": [20, 107], "the": [21, 40, 170, 201, 210, 218, 230], "previous": [22], "Infectious": [23], "Diseases": [24], "Society": [25], "of": [26, 39, 42, 96, 112, 157, 162, 172, 175, 213, 233], "America": [27], "from": [29], "2000": [30], "include": [32, 93], "new": [33, 192], "sections.": [34], "There": [35, 65], "discussion": [38], "cryptococcal": [43, 113], "meningoencephalitis": [44, 139], "in": [45, 80, 110, 179, 229], "3": [46], "risk": [47, 72], "groups:": [48], "(1)": [49, 135], "human": [50], "immunodeficiency": [51], "virus": [52], "(HIV)-infected": [53], "individuals,": [54], "(2)": [55, 155], "organ": [56], "transplant": [57], "recipients,": [58], "(3)": [60, 169], "non-HIV-infected": [61], "nontransplant": [63], "hosts.": [64], "are": [66], "specific": [67], "recommendations": [68], "other": [70, 94], "unique": [71], "populations,": [73], "such": [74, 143], "as": [75, 144], "children,": [76], "pregnant": [77], "women,": [78], "persons": [79], "resource-limited": [81], "environments,": [82], "those": [84], "Cryptococcus": [86], "gattii": [87], "infection.": [88], "Recommendations": [89], "also": [92], "sites": [95], "infection,": [97, 114], "including": [98, 115], "strategies": [99], "pulmonary": [101], "cryptococcosis.": [102], "Emphasis": [103], "has": [104], "placed": [106], "potential": [108], "complications": [109], "increased": [116, 163], "intracranial": [117, 164], "pressure,": [118], "immune": [119], "reconstitution": [120], "inflammatory": [121], "syndrome": [122], "(IRIS),": [123], "drug": [124, 193], "resistance,": [125], "cryptococcomas.": [127], "Three": [128], "key": [129], "principles": [131, 212], "articulated:": [134], "induction": [136], "therapy": [137], "using": [140, 153], "fungicidal": [141], "regimens,": [142], "polyene": [146], "flucytosine,": [148], "followed": [149], "by": [150], "suppressive": [151], "regimens": [152, 178], "fluconazole;": [154], "importance": [156], "early": [158], "recognition": [159], "treatment": [161], "pressure": [165], "and/or": [166], "IRIS;": [167], "use": [171], "lipid": [173], "formulations": [174], "amphotericin": [176], "B": [177], "patients": [180], "renal": [182], "impairment.": [183], "remains": [185], "challenging": [187], "issue,": [189], "little": [191], "development": [194], "or": [195], "recent": [196], "definitive": [197], "studies.": [198], "However,": [199], "if": [200, 206, 217], "diagnosis": [202], "made": [204], "early,": [205], "clinicians": [207], "adhere": [208], "to": [209], "basic": [211], "these": [214], "guidelines,": [215], "underlying": [219], "disease": [220], "controlled,": [222], "then": [223], "cryptococcosis": [224], "can": [225], "be": [226], "managed": [227], "successfully": [228], "vast": [231], "majority": [232], "patients.": [234]}, "cited_by_api_url": "https://api.openalex.org/works?filter=cites:W2144543496", "counts_by_year": [{"year": 2023, "cited_by_count": 154}, {"year": 2022, "cited_by_count": 192}, {"year": 2021, "cited_by_count": 164}, {"year": 2020, "cited_by_count": 185}, {"year": 2019, "cited_by_count": 178}, {"year": 2018, "cited_by_count": 154}, {"year": 2017, "cited_by_count": 163}, {"year": 2016, "cited_by_count": 168}, {"year": 2015, "cited_by_count": 158}, {"year": 2014, "cited_by_count": 165}, {"year": 2013, "cited_by_count": 133}, {"year": 2012, "cited_by_count": 131}], "updated_date": "2023-12-05T01:32:54.055020", "created_date": "2016-06-24"} +{"id": "https://openalex.org/W2115169717", "doi": "https://doi.org/10.1016/s0140-6736(09)61965-6", "title": "Statins and risk of incident diabetes: a collaborative meta-analysis of randomised statin trials", "display_name": "Statins and risk of incident diabetes: a collaborative meta-analysis of randomised statin trials", "publication_year": 2010, "publication_date": "2010-02-01", "ids": {"openalex": "https://openalex.org/W2115169717", "doi": "https://doi.org/10.1016/s0140-6736(09)61965-6", "mag": "2115169717", "pmid": "https://pubmed.ncbi.nlm.nih.gov/20167359"}, "language": "en", "primary_location": {"is_oa": false, "landing_page_url": "https://doi.org/10.1016/s0140-6736(09)61965-6", "pdf_url": null, "source": {"id": "https://openalex.org/S49861241", "display_name": "The Lancet", "issn_l": "0140-6736", "issn": ["1474-547X", "0099-5355", "0140-6736"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310320990", "host_organization_name": "Elsevier BV", "host_organization_lineage": ["https://openalex.org/P4310320990"], "host_organization_lineage_names": ["Elsevier BV"], "type": "journal"}, "license": null, "version": null, "is_accepted": false, "is_published": false}, "type": "article", "type_crossref": "journal-article", "open_access": {"is_oa": false, "oa_status": "closed", "oa_url": null, "any_repository_has_fulltext": false}, "authorships": [{"author_position": "first", "author": {"id": "https://openalex.org/A5078498803", "display_name": "Naveed Sattar", "orcid": null}, "institutions": [{"id": "https://openalex.org/I32003884", "display_name": "British Heart Foundation", "ror": "https://ror.org/02wdwnk04", "country_code": "GB", "type": "nonprofit", "lineage": ["https://openalex.org/I32003884"]}, {"id": "https://openalex.org/I7882870", "display_name": "University of Glasgow", "ror": "https://ror.org/00vtgdb53", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I7882870"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Naveed Sattar", "raw_affiliation_string": "British Heart Foundation Glasgow Cardiovascular Research Centre; University of Glasgow; Glasgow UK", "raw_affiliation_strings": ["British Heart Foundation Glasgow Cardiovascular Research Centre; University of Glasgow; Glasgow UK"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5039985378", "display_name": "David Preiss", "orcid": "https://orcid.org/0000-0003-3139-1836"}, "institutions": [{"id": "https://openalex.org/I7882870", "display_name": "University of Glasgow", "ror": "https://ror.org/00vtgdb53", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I7882870"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "David Preiss", "raw_affiliation_string": "*University of Glasgow.", "raw_affiliation_strings": ["*University of Glasgow."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5005987955", "display_name": "Heather Murray", "orcid": null}, "institutions": [{"id": "https://openalex.org/I7882870", "display_name": "University of Glasgow", "ror": "https://ror.org/00vtgdb53", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I7882870"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Heather M Murray", "raw_affiliation_string": "*University of Glasgow.", "raw_affiliation_strings": ["*University of Glasgow."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5021225266", "display_name": "Paul Welsh", "orcid": "https://orcid.org/0000-0002-7970-3643"}, "institutions": [{"id": "https://openalex.org/I7882870", "display_name": "University of Glasgow", "ror": "https://ror.org/00vtgdb53", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I7882870"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Paul Welsh", "raw_affiliation_string": "*University of Glasgow.", "raw_affiliation_strings": ["*University of Glasgow."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5034827753", "display_name": "Brendan M. Buckley", "orcid": "https://orcid.org/0000-0003-1544-8003"}, "institutions": [{"id": "https://openalex.org/I2802396013", "display_name": "Cork University Hospital", "ror": "https://ror.org/04q107642", "country_code": "IE", "type": "healthcare", "lineage": ["https://openalex.org/I2802396013"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Brendan M Buckley", "raw_affiliation_string": "Department of Pharmacology and Therapeutics; Cork University Hospital; Cork Ireland", "raw_affiliation_strings": ["Department of Pharmacology and Therapeutics; Cork University Hospital; Cork Ireland"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5017836069", "display_name": "Anton J. M. de Craen", "orcid": null}, "institutions": [{"id": "https://openalex.org/I121797337", "display_name": "Leiden University", "ror": "https://ror.org/027bh9e22", "country_code": "NL", "type": "education", "lineage": ["https://openalex.org/I121797337"]}], "countries": ["NL"], "is_corresponding": false, "raw_author_name": "Anton J M de Craen", "raw_affiliation_string": "Leiden University", "raw_affiliation_strings": ["Leiden University"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5040241232", "display_name": "Sreenivasa Rao Kondapally Seshasai", "orcid": "https://orcid.org/0000-0002-5948-6522"}, "institutions": [{"id": "https://openalex.org/I241749", "display_name": "University of Cambridge", "ror": "https://ror.org/013meh722", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I241749"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Sreenivasa Rao Kondapally Seshasai", "raw_affiliation_string": "Univ. of Cambridge", "raw_affiliation_strings": ["Univ. of Cambridge"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5027213857", "display_name": "John J.V. McMurray", "orcid": "https://orcid.org/0000-0002-6317-3975"}, "institutions": [{"id": "https://openalex.org/I7882870", "display_name": "University of Glasgow", "ror": "https://ror.org/00vtgdb53", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I7882870"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "John J McMurray", "raw_affiliation_string": "*University of Glasgow.", "raw_affiliation_strings": ["*University of Glasgow."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5085003120", "display_name": "Dilys J. Freeman", "orcid": null}, "institutions": [{"id": "https://openalex.org/I7882870", "display_name": "University of Glasgow", "ror": "https://ror.org/00vtgdb53", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I7882870"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Dilys J Freeman", "raw_affiliation_string": "*University of Glasgow.", "raw_affiliation_strings": ["*University of Glasgow."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5005451442", "display_name": "J. Wouter Jukema", "orcid": "https://orcid.org/0000-0002-3246-8359"}, "institutions": [{"id": "https://openalex.org/I121797337", "display_name": "Leiden University", "ror": "https://ror.org/027bh9e22", "country_code": "NL", "type": "education", "lineage": ["https://openalex.org/I121797337"]}], "countries": ["NL"], "is_corresponding": false, "raw_author_name": "J Wouter Jukema", "raw_affiliation_string": "Leiden University", "raw_affiliation_strings": ["Leiden University"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5081878109", "display_name": "Peter W. Macfarlane", "orcid": "https://orcid.org/0000-0002-5390-1596"}, "institutions": [{"id": "https://openalex.org/I7882870", "display_name": "University of Glasgow", "ror": "https://ror.org/00vtgdb53", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I7882870"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Peter W Macfarlane", "raw_affiliation_string": "*University of Glasgow.", "raw_affiliation_strings": ["*University of Glasgow."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5003522491", "display_name": "Chris J. Packard", "orcid": "https://orcid.org/0000-0002-2386-9927"}, "institutions": [{"id": "https://openalex.org/I7882870", "display_name": "University of Glasgow", "ror": "https://ror.org/00vtgdb53", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I7882870"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Chris J Packard", "raw_affiliation_string": "*University of Glasgow.", "raw_affiliation_strings": ["*University of Glasgow."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5031245545", "display_name": "David J. Stott", "orcid": "https://orcid.org/0000-0002-3110-7746"}, "institutions": [{"id": "https://openalex.org/I7882870", "display_name": "University of Glasgow", "ror": "https://ror.org/00vtgdb53", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I7882870"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "David J Stott", "raw_affiliation_string": "*University of Glasgow.", "raw_affiliation_strings": ["*University of Glasgow."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5036896719", "display_name": "Rudi G.J. Westendorp", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210131551", "display_name": "Netherlands Consortium for Healthy Ageing", "ror": "https://ror.org/03wnqyy64", "country_code": "NL", "type": "healthcare", "lineage": ["https://openalex.org/I4210131551"]}], "countries": ["NL"], "is_corresponding": false, "raw_author_name": "Rudi G Westendorp", "raw_affiliation_string": "Netherlands Consortium for Healthy Ageing", "raw_affiliation_strings": ["Netherlands Consortium for Healthy Ageing"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5048047555", "display_name": "James Shepherd", "orcid": null}, "institutions": [{"id": "https://openalex.org/I7882870", "display_name": "University of Glasgow", "ror": "https://ror.org/00vtgdb53", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I7882870"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "James Shepherd", "raw_affiliation_string": "*University of Glasgow.", "raw_affiliation_strings": ["*University of Glasgow."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5042299275", "display_name": "Barry R. Davis", "orcid": "https://orcid.org/0000-0002-6943-5673"}, "institutions": [], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Barry R Davis", "raw_affiliation_string": "University of Texas, School of Public Health, TX, USA.", "raw_affiliation_strings": ["University of Texas, School of Public Health, TX, USA."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5072348600", "display_name": "Sara L. Pressel", "orcid": null}, "institutions": [], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Sara L Pressel", "raw_affiliation_string": "University of Texas, School of Public Health, TX, USA.", "raw_affiliation_strings": ["University of Texas, School of Public Health, TX, USA."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5090813987", "display_name": "Roberto Marchioli", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210110338", "display_name": "Mario Negri Sud Foundation", "ror": "https://ror.org/01qd3xc93", "country_code": "IT", "type": "nonprofit", "lineage": ["https://openalex.org/I4210110338"]}], "countries": ["IT"], "is_corresponding": false, "raw_author_name": "Roberto Marchioli", "raw_affiliation_string": "Consorzio Mario Negri Stud", "raw_affiliation_strings": ["Consorzio Mario Negri Stud"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5069819115", "display_name": "Rosa Maria Marfisi", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210110338", "display_name": "Mario Negri Sud Foundation", "ror": "https://ror.org/01qd3xc93", "country_code": "IT", "type": "nonprofit", "lineage": ["https://openalex.org/I4210110338"]}], "countries": ["IT"], "is_corresponding": false, "raw_author_name": "Rosa Maria Marfisi", "raw_affiliation_string": "Consorzio Mario Negri Stud", "raw_affiliation_strings": ["Consorzio Mario Negri Stud"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5088186128", "display_name": "Aldo P. Maggioni", "orcid": "https://orcid.org/0000-0003-2764-6779"}, "institutions": [{"id": "https://openalex.org/I4210095959", "display_name": "Associazione Nazionale Medici Cardiologi Ospedalieri", "ror": "https://ror.org/00pyc4352", "country_code": "IT", "type": "nonprofit", "lineage": ["https://openalex.org/I4210095959"]}], "countries": ["IT"], "is_corresponding": false, "raw_author_name": "Aldo P Maggioni", "raw_affiliation_string": "ANMCO Research Centre, Florence, Italy", "raw_affiliation_strings": ["ANMCO Research Centre, Florence, Italy"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5029606867", "display_name": "Luigi Tavazzi", "orcid": "https://orcid.org/0000-0003-4448-5209"}, "institutions": [{"id": "https://openalex.org/I2802469017", "display_name": "CARE Hospitals", "ror": "https://ror.org/01vka3a64", "country_code": "IN", "type": "healthcare", "lineage": ["https://openalex.org/I2802469017"]}], "countries": ["IN"], "is_corresponding": false, "raw_author_name": "Luigi Tavazzi", "raw_affiliation_string": "GVM Hospitals of Care and Research", "raw_affiliation_strings": ["GVM Hospitals of Care and Research"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5005350140", "display_name": "Gianni Tognoni", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210110338", "display_name": "Mario Negri Sud Foundation", "ror": "https://ror.org/01qd3xc93", "country_code": "IT", "type": "nonprofit", "lineage": ["https://openalex.org/I4210110338"]}], "countries": ["IT"], "is_corresponding": false, "raw_author_name": "Gianni Tognoni", "raw_affiliation_string": "Consorzio Mario Negri Stud", "raw_affiliation_strings": ["Consorzio Mario Negri Stud"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5025930905", "display_name": "John Kjekshus", "orcid": "https://orcid.org/0000-0003-4306-1244"}, "institutions": [{"id": "https://openalex.org/I1281400175", "display_name": "Oslo University Hospital", "ror": "https://ror.org/00j9c2840", "country_code": "NO", "type": "healthcare", "lineage": ["https://openalex.org/I1281400175"]}], "countries": ["NO"], "is_corresponding": false, "raw_author_name": "John Kjekshus", "raw_affiliation_string": "Department of Cardiology, Rikshospitalet University Hospital, Oslo, Norway", "raw_affiliation_strings": ["Department of Cardiology, Rikshospitalet University Hospital, Oslo, Norway"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5083340275", "display_name": "Terje R. Pedersen", "orcid": null}, "institutions": [{"id": "https://openalex.org/I1281400175", "display_name": "Oslo University Hospital", "ror": "https://ror.org/00j9c2840", "country_code": "NO", "type": "healthcare", "lineage": ["https://openalex.org/I1281400175"]}], "countries": ["NO"], "is_corresponding": false, "raw_author_name": "Terje R Pedersen", "raw_affiliation_string": "Centre for Preventative Medicine, Ulleval University Hospital, Oslo, Norway", "raw_affiliation_strings": ["Centre for Preventative Medicine, Ulleval University Hospital, Oslo, Norway"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5077547822", "display_name": "Thomas J. Cook", "orcid": "https://orcid.org/0009-0004-8785-0346"}, "institutions": [{"id": "https://openalex.org/I4210150308", "display_name": "Agile RF (United States)", "ror": "https://ror.org/049g0jw79", "country_code": "US", "type": "company", "lineage": ["https://openalex.org/I4210150308"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Thomas J Cook", "raw_affiliation_string": "Agile 1", "raw_affiliation_strings": ["Agile 1"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5068202304", "display_name": "Antonio M. Gotto", "orcid": "https://orcid.org/0000-0001-8076-6783"}, "institutions": [{"id": "https://openalex.org/I205783295", "display_name": "Cornell University", "ror": "https://ror.org/05bnh6r87", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I205783295"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Antonio M Gotto", "raw_affiliation_string": "[Weill Medical College, Cornell University, NY, USA]", "raw_affiliation_strings": ["[Weill Medical College, Cornell University, NY, USA]"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5036001636", "display_name": "Michael Clearfield", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210124038", "display_name": "Moscow University Touro", "ror": "https://ror.org/02pppmh23", "country_code": "RU", "type": "education", "lineage": ["https://openalex.org/I4210124038"]}], "countries": ["RU"], "is_corresponding": false, "raw_author_name": "Michael B Clearfield", "raw_affiliation_string": "TOURO UNIVERSITY", "raw_affiliation_strings": ["TOURO UNIVERSITY"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5056502629", "display_name": "John R. Downs", "orcid": null}, "institutions": [{"id": "https://openalex.org/I165951966", "display_name": "The University of Texas Health Science Center at San Antonio", "ror": "https://ror.org/02f6dcw23", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I16452829", "https://openalex.org/I165951966"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "John R Downs", "raw_affiliation_string": "Department of Medicine, University of Texas Health Science Centre, San Antonio, TX, USA", "raw_affiliation_strings": ["Department of Medicine, University of Texas Health Science Centre, San Antonio, TX, USA"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5023875153", "display_name": "Haruo Nakamura", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210159217", "display_name": "Mitsukoshi Health and Welfare Foundation", "ror": "https://ror.org/05wzgbw88", "country_code": "JP", "type": "other", "lineage": ["https://openalex.org/I4210159217"]}], "countries": ["JP"], "is_corresponding": false, "raw_author_name": "Haruo Nakamura", "raw_affiliation_string": "Mitsukoshi Health and Welfare Foundation", "raw_affiliation_strings": ["Mitsukoshi Health and Welfare Foundation"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5090955736", "display_name": "Yasuo Ohashi", "orcid": null}, "institutions": [{"id": "https://openalex.org/I74801974", "display_name": "The University of Tokyo", "ror": "https://ror.org/057zh3y96", "country_code": "JP", "type": "education", "lineage": ["https://openalex.org/I74801974"]}], "countries": ["JP"], "is_corresponding": false, "raw_author_name": "Yasuo Ohashi", "raw_affiliation_string": "Univ.\ of Tokyo", "raw_affiliation_strings": ["Univ.\ of Tokyo"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5024734422", "display_name": "Kyoichi Mizuno", "orcid": "https://orcid.org/0009-0003-9933-9513"}, "institutions": [{"id": "https://openalex.org/I80188885", "display_name": "Nippon Medical School", "ror": "https://ror.org/00krab219", "country_code": "JP", "type": "education", "lineage": ["https://openalex.org/I80188885"]}], "countries": ["JP"], "is_corresponding": false, "raw_author_name": "Kyoichi Mizuno", "raw_affiliation_string": "Nippon Medical School", "raw_affiliation_strings": ["Nippon Medical School"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5006206326", "display_name": "Kausik K. Ray", "orcid": "https://orcid.org/0000-0003-0508-0954"}, "institutions": [{"id": "https://openalex.org/I241749", "display_name": "University of Cambridge", "ror": "https://ror.org/013meh722", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I241749"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Kausik K Ray", "raw_affiliation_string": "Univ. of Cambridge", "raw_affiliation_strings": ["Univ. of Cambridge"]}, {"author_position": "last", "author": {"id": "https://openalex.org/A5016095791", "display_name": "Ian Ford", "orcid": "https://orcid.org/0000-0001-5927-1823"}, "institutions": [{"id": "https://openalex.org/I7882870", "display_name": "University of Glasgow", "ror": "https://ror.org/00vtgdb53", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I7882870"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Ian Ford", "raw_affiliation_string": "*University of Glasgow.", "raw_affiliation_strings": ["*University of Glasgow."]}], "countries_distinct_count": 9, "institutions_distinct_count": 17, "corresponding_author_ids": [], "corresponding_institution_ids": [], "apc_list": {"value": 6830, "currency": "USD", "value_usd": 6830, "provenance": "doaj"}, "apc_paid": {"value": 6830, "currency": "USD", "value_usd": 6830, "provenance": "doaj"}, "has_fulltext": true, "fulltext_origin": "ngrams", "cited_by_count": 2031, "cited_by_percentile_year": {"min": 99.9, "max": 100.0}, "biblio": {"volume": "375", "issue": "9716", "first_page": "735", "last_page": "742"}, "is_retracted": false, "is_paratext": false, "keywords": [{"keyword": "statins trials", "score": 0.7194}, {"keyword": "incident diabetes", "score": 0.4573}, {"keyword": "meta-analysis", "score": 0.25}], "concepts": [{"id": "https://openalex.org/C71924100", "wikidata": "https://www.wikidata.org/wiki/Q11190", "display_name": "Medicine", "level": 0, "score": 0.8956113}, {"id": "https://openalex.org/C126322002", "wikidata": "https://www.wikidata.org/wiki/Q11180", "display_name": "Internal medicine", "level": 1, "score": 0.7007866}, {"id": "https://openalex.org/C2776839432", "wikidata": "https://www.wikidata.org/wiki/Q954845", "display_name": "Statin", "level": 2, "score": 0.69842064}, {"id": "https://openalex.org/C555293320", "wikidata": "https://www.wikidata.org/wiki/Q12206", "display_name": "Diabetes mellitus", "level": 2, "score": 0.6833198}, {"id": "https://openalex.org/C82789193", "wikidata": "https://www.wikidata.org/wiki/Q2142611", "display_name": "Relative risk", "level": 3, "score": 0.5471921}, {"id": "https://openalex.org/C156957248", "wikidata": "https://www.wikidata.org/wiki/Q1862216", "display_name": "Odds ratio", "level": 2, "score": 0.54087865}, {"id": "https://openalex.org/C95190672", "wikidata": "https://www.wikidata.org/wiki/Q815382", "display_name": "Meta-analysis", "level": 2, "score": 0.5246632}, {"id": "https://openalex.org/C535046627", "wikidata": "https://www.wikidata.org/wiki/Q30612", "display_name": "Clinical trial", "level": 2, "score": 0.5221627}, {"id": "https://openalex.org/C168563851", "wikidata": "https://www.wikidata.org/wiki/Q1436668", "display_name": "Randomized controlled trial", "level": 2, "score": 0.47946703}, {"id": "https://openalex.org/C203092338", "wikidata": "https://www.wikidata.org/wiki/Q1340863", "display_name": "Clinical endpoint", "level": 3, "score": 0.4757145}, {"id": "https://openalex.org/C2777180221", "wikidata": "https://www.wikidata.org/wiki/Q3025883", "display_name": "Type 2 diabetes", "level": 3, "score": 0.4493629}, {"id": "https://openalex.org/C44249647", "wikidata": "https://www.wikidata.org/wiki/Q208498", "display_name": "Confidence interval", "level": 2, "score": 0.31917673}, {"id": "https://openalex.org/C134018914", "wikidata": "https://www.wikidata.org/wiki/Q162606", "display_name": "Endocrinology", "level": 1, "score": 0.13296235}], "mesh": [{"descriptor_ui": "D000924", "descriptor_name": "Anticholesteremic Agents", "qualifier_ui": "Q000009", "qualifier_name": "adverse effects", "is_major_topic": true}, {"descriptor_ui": "D002318", "descriptor_name": "Cardiovascular Diseases", "qualifier_ui": "Q000188", "qualifier_name": "drug therapy", "is_major_topic": true}, {"descriptor_ui": "D003924", "descriptor_name": "Diabetes Mellitus, Type 2", "qualifier_ui": "Q000139", "qualifier_name": "chemically induced", "is_major_topic": true}, {"descriptor_ui": "D019161", "descriptor_name": "Hydroxymethylglutaryl-CoA Reductase Inhibitors", "qualifier_ui": "Q000009", "qualifier_name": "adverse effects", "is_major_topic": true}, {"descriptor_ui": "D017677", "descriptor_name": "Age Distribution", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D000367", "descriptor_name": "Age Factors", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D000368", "descriptor_name": "Aged", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D000924", "descriptor_name": "Anticholesteremic Agents", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D002318", "descriptor_name": "Cardiovascular Diseases", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D003924", "descriptor_name": "Diabetes Mellitus, Type 2", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D003924", "descriptor_name": "Diabetes Mellitus, Type 2", "qualifier_ui": "Q000453", "qualifier_name": "epidemiology", "is_major_topic": false}, {"descriptor_ui": "D005260", "descriptor_name": "Female", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D006801", "descriptor_name": "Humans", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D019161", "descriptor_name": "Hydroxymethylglutaryl-CoA Reductase Inhibitors", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D008297", "descriptor_name": "Male", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D008875", "descriptor_name": "Middle Aged", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D016032", "descriptor_name": "Randomized Controlled Trials as Topic", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D012307", "descriptor_name": "Risk Factors", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D016896", "descriptor_name": "Treatment Outcome", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}], "locations_count": 2, "locations": [{"is_oa": false, "landing_page_url": "https://doi.org/10.1016/s0140-6736(09)61965-6", "pdf_url": null, "source": {"id": "https://openalex.org/S49861241", "display_name": "The Lancet", "issn_l": "0140-6736", "issn": ["1474-547X", "0099-5355", "0140-6736"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310320990", "host_organization_name": "Elsevier BV", "host_organization_lineage": ["https://openalex.org/P4310320990"], "host_organization_lineage_names": ["Elsevier BV"], "type": "journal"}, "license": null, "version": null, "is_accepted": false, "is_published": false}, {"is_oa": false, "landing_page_url": "https://pubmed.ncbi.nlm.nih.gov/20167359", "pdf_url": null, "source": {"id": "https://openalex.org/S4306525036", "display_name": "PubMed", "issn_l": null, "issn": null, "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/I1299303238", "host_organization_name": "National Institutes of Health", "host_organization_lineage": ["https://openalex.org/I1299303238"], "host_organization_lineage_names": ["National Institutes of Health"], "type": "repository"}, "license": null, "version": null, "is_accepted": false, "is_published": false}], "best_oa_location": null, "sustainable_development_goals": [{"id": "https://metadata.un.org/sdg/3", "display_name": "Good health and well-being", "score": 0.72}], "grants": [], "referenced_works_count": 33, "referenced_works": ["https://openalex.org/W137344628", "https://openalex.org/W1500963439", "https://openalex.org/W1502524639", "https://openalex.org/W1546258268", "https://openalex.org/W1588310170", "https://openalex.org/W1944558458", "https://openalex.org/W1976358806", "https://openalex.org/W1992022057", "https://openalex.org/W1997020689", "https://openalex.org/W2028948892", "https://openalex.org/W2031185570", "https://openalex.org/W2049776203", "https://openalex.org/W2061489988", "https://openalex.org/W2086139919", "https://openalex.org/W2089629277", "https://openalex.org/W2097264002", "https://openalex.org/W2097870088", "https://openalex.org/W2098783546", "https://openalex.org/W2116045728", "https://openalex.org/W2116946402", "https://openalex.org/W2118667643", "https://openalex.org/W2125435699", "https://openalex.org/W2126452437", "https://openalex.org/W2126678006", "https://openalex.org/W2129750583", "https://openalex.org/W2130636082", "https://openalex.org/W2135631433", "https://openalex.org/W2137983259", "https://openalex.org/W2157823046", "https://openalex.org/W2160390128", "https://openalex.org/W2165796078", "https://openalex.org/W2247997571", "https://openalex.org/W2322095705"], "related_works": ["https://openalex.org/W1539974851", "https://openalex.org/W3165215133", "https://openalex.org/W2611523470", "https://openalex.org/W3210678099", "https://openalex.org/W4246615163", "https://openalex.org/W4360943417", "https://openalex.org/W4386361997", "https://openalex.org/W2417314287", "https://openalex.org/W4200125571", "https://openalex.org/W2593300661"], "ngrams_url": "https://api.openalex.org/works/W2115169717/ngrams", "abstract_inverted_index": {"Trials": [0, 53], "of": [1, 11, 13, 27, 41, 51, 63, 82, 90, 127, 143, 175, 177, 205, 221, 231, 233], "statin": [2, 37, 121], "therapy": [3, 147, 223], "have": [4], "had": [5], "conflicting": [6], "findings": [7], "on": [8], "the": [9, 47, 101, 236, 248], "risk": [10, 111, 154, 174, 230, 237, 262], "development": [12, 40, 176, 232], "diabetes": [14, 115, 139, 157, 178], "mellitus": [15], "in": [16, 77, 183, 195, 202, 217, 241, 250, 255], "patients": [17, 91, 210, 256], "given": [18], "statins.": [19, 64], "We": [20, 65, 87, 99], "aimed": [21], "to": [22, 56, 104], "establish": [23], "by": [24], "a": [25, 141, 151, 227], "meta-analysis": [26], "published": [28], "and": [29, 39, 46, 80, 109, 133, 244], "unpublished": [30], "data": [31], "whether": [32], "any": [33], "relation": [34], "exists": [35], "between": [36, 107, 169], "use": [38], "diabetes.We": [42], "searched": [43], "Medline,": [44], "Embase,": [45], "Cochrane": [48], "Central": [49], "Register": [50], "Controlled": [52], "from": [54], "1994": [55], "2009,": [57], "for": [58, 113, 155, 199, 213], "randomised": [59], "controlled": [60], "endpoint": [61], "trials": [62, 68, 89, 108, 122, 184], "included": [66], "only": [67], "with": [69, 74, 92, 116, 123, 150, 165, 179, 185, 211, 226, 247, 257], "more": [70, 83], "than": [71, 84], "1000": [72], "patients,": [73], "identical": [75], "follow-up": [76], "both": [78, 240], "groups": [79], "duration": [81], "1": [85], "year.": [86], "excluded": [88], "organ": [93], "transplants": [94], "or": [95, 259, 263], "who": [96], "needed": [97], "haemodialysis.": [98], "used": [100], "I(2)": [102], "statistic": [103], "measure": [105], "heterogeneity": [106, 167], "calculated": [110], "estimates": [112], "incident": [114, 156], "random-effect": [117], "meta-analysis.We": [118], "identified": [119], "13": [120], "91": [124], "140": [125], "participants,": [126, 187], "whom": [128], "4278": [129], "(2226": [130], "assigned": [131, 135], "statins": [132, 180, 212], "2052": [134], "control": [136], "treatment)": [137], "developed": [138], "during": [140], "mean": [142], "4": [144, 214], "years.": [145], "Statin": [146], "was": [148, 181], "associated": [149, 225], "9%": [152], "increased": [153, 229], "(odds": [158], "ratio": [159], "[OR]": [160], "1.09;": [161], "95%": [162], "CI": [163, 208], "1.02-1.17),": [164], "little": [166], "(I(2)=11%)": [168], "trials.": [170], "Meta-regression": [171], "showed": [172], "that": [173], "highest": [182], "older": [186], "but": [188, 235], "neither": [189], "baseline": [190], "body-mass": [191], "index": [192], "nor": [193], "change": [194], "LDL-cholesterol": [196], "concentrations": [197], "accounted": [198], "residual": [200], "variation": [201], "risk.": [203], "Treatment": [204], "255": [206], "(95%": [207], "150-852)": [209], "years": [215], "resulted": [216], "one": [218], "extra": [219], "case": [220], "diabetes.Statin": [222], "is": [224, 238], "slightly": [228], "diabetes,": [234], "low": [239], "absolute": [242], "terms": [243], "when": [245], "compared": [246], "reduction": [249], "coronary": [251], "events.": [252], "Clinical": [253], "practice": [254], "moderate": [258], "high": [260], "cardiovascular": [261, 265], "existing": [264], "disease": [266], "should": [267], "not": [268], "change.None.": [269]}, "cited_by_api_url": "https://api.openalex.org/works?filter=cites:W2115169717", "counts_by_year": [{"year": 2023, "cited_by_count": 87}, {"year": 2022, "cited_by_count": 122}, {"year": 2021, "cited_by_count": 93}, {"year": 2020, "cited_by_count": 125}, {"year": 2019, "cited_by_count": 166}, {"year": 2018, "cited_by_count": 164}, {"year": 2017, "cited_by_count": 157}, {"year": 2016, "cited_by_count": 221}, {"year": 2015, "cited_by_count": 198}, {"year": 2014, "cited_by_count": 200}, {"year": 2013, "cited_by_count": 175}, {"year": 2012, "cited_by_count": 134}], "updated_date": "2023-11-29T15:25:34.068916", "created_date": "2016-06-24"} +{"id": "https://openalex.org/W2119378720", "doi": "https://doi.org/10.1038/nnano.2010.15", "title": "Nanowire transistors without junctions", "display_name": "Nanowire transistors without junctions", "publication_year": 2010, "publication_date": "2010-02-21", "ids": {"openalex": "https://openalex.org/W2119378720", "doi": "https://doi.org/10.1038/nnano.2010.15", "mag": "2119378720", "pmid": "https://pubmed.ncbi.nlm.nih.gov/20173755"}, "language": "en", "primary_location": {"is_oa": false, "landing_page_url": "https://doi.org/10.1038/nnano.2010.15", "pdf_url": null, "source": {"id": "https://openalex.org/S7822423", "display_name": "Nature Nanotechnology", "issn_l": "1748-3387", "issn": ["1748-3395", "1748-3387"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310319908", "host_organization_name": "Nature Portfolio", "host_organization_lineage": ["https://openalex.org/P4310319908", "https://openalex.org/P4310319965"], "host_organization_lineage_names": ["Nature Portfolio", "Springer Nature"], "type": "journal"}, "license": null, "version": null, "is_accepted": false, "is_published": false}, "type": "article", "type_crossref": "journal-article", "open_access": {"is_oa": false, "oa_status": "closed", "oa_url": null, "any_repository_has_fulltext": false}, "authorships": [{"author_position": "first", "author": {"id": "https://openalex.org/A5055173231", "display_name": "Jean–Pierre Colinge", "orcid": null}, "institutions": [{"id": "https://openalex.org/I27577105", "display_name": "University College Cork", "ror": "https://ror.org/03265fv13", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I181231927", "https://openalex.org/I27577105"]}], "countries": ["IE"], "is_corresponding": true, "raw_author_name": "Jean-Pierre Colinge", "raw_affiliation_string": "Tyndall National Institute, University College Cork, Cork, Ireland", "raw_affiliation_strings": ["Tyndall National Institute, University College Cork, Cork, Ireland"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5075793139", "display_name": "Chi‐Woo Lee", "orcid": null}, "institutions": [{"id": "https://openalex.org/I27577105", "display_name": "University College Cork", "ror": "https://ror.org/03265fv13", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I181231927", "https://openalex.org/I27577105"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Chi-Woo Lee", "raw_affiliation_string": "Tyndall National Institute, University College Cork, Cork, Ireland", "raw_affiliation_strings": ["Tyndall National Institute, University College Cork, Cork, Ireland"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5043132234", "display_name": "Aryan Afzalian", "orcid": "https://orcid.org/0000-0002-5260-0281"}, "institutions": [{"id": "https://openalex.org/I27577105", "display_name": "University College Cork", "ror": "https://ror.org/03265fv13", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I181231927", "https://openalex.org/I27577105"]}, {"id": "https://openalex.org/I95674353", "display_name": "Université Catholique de Louvain", "ror": "https://ror.org/02495e989", "country_code": "BE", "type": "education", "lineage": ["https://openalex.org/I95674353"]}], "countries": ["BE", "IE"], "is_corresponding": false, "raw_author_name": "Aryan Afzalian", "raw_affiliation_string": "Present address: Laboratoire de Microélectronique, Université Catholique de Louvain, Louvain-la-Neuve, Belgium,; Tyndall National Institute, University College Cork, Cork, Ireland", "raw_affiliation_strings": ["Present address: Laboratoire de Microélectronique, Université Catholique de Louvain, Louvain-la-Neuve, Belgium,", "Tyndall National Institute, University College Cork, Cork, Ireland"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5003149290", "display_name": "Nima Dehdashti Akhavan", "orcid": "https://orcid.org/0000-0003-1658-8323"}, "institutions": [{"id": "https://openalex.org/I27577105", "display_name": "University College Cork", "ror": "https://ror.org/03265fv13", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I181231927", "https://openalex.org/I27577105"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Nima Dehdashti Akhavan", "raw_affiliation_string": "Tyndall National Institute, University College Cork, Cork, Ireland", "raw_affiliation_strings": ["Tyndall National Institute, University College Cork, Cork, Ireland"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5041418760", "display_name": "Ran Yan", "orcid": "https://orcid.org/0000-0003-4400-8007"}, "institutions": [{"id": "https://openalex.org/I27577105", "display_name": "University College Cork", "ror": "https://ror.org/03265fv13", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I181231927", "https://openalex.org/I27577105"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Ran Yan", "raw_affiliation_string": "Tyndall National Institute, University College Cork, Cork, Ireland", "raw_affiliation_strings": ["Tyndall National Institute, University College Cork, Cork, Ireland"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5059409050", "display_name": "Isabelle Ferain", "orcid": null}, "institutions": [{"id": "https://openalex.org/I27577105", "display_name": "University College Cork", "ror": "https://ror.org/03265fv13", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I181231927", "https://openalex.org/I27577105"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Isabelle Ferain", "raw_affiliation_string": "Tyndall National Institute, University College Cork, Cork, Ireland", "raw_affiliation_strings": ["Tyndall National Institute, University College Cork, Cork, Ireland"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5029733167", "display_name": "Pedram Razavi", "orcid": "https://orcid.org/0000-0003-4236-0576"}, "institutions": [{"id": "https://openalex.org/I27577105", "display_name": "University College Cork", "ror": "https://ror.org/03265fv13", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I181231927", "https://openalex.org/I27577105"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Pedram Razavi", "raw_affiliation_string": "Tyndall National Institute, University College Cork, Cork, Ireland", "raw_affiliation_strings": ["Tyndall National Institute, University College Cork, Cork, Ireland"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5031519620", "display_name": "B. O'Neill", "orcid": null}, "institutions": [{"id": "https://openalex.org/I27577105", "display_name": "University College Cork", "ror": "https://ror.org/03265fv13", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I181231927", "https://openalex.org/I27577105"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Brendan O'Neill", "raw_affiliation_string": "Tyndall National Institute, University College Cork, Cork, Ireland", "raw_affiliation_strings": ["Tyndall National Institute, University College Cork, Cork, Ireland"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5004097220", "display_name": "Alan Blake", "orcid": "https://orcid.org/0000-0001-7961-4459"}, "institutions": [{"id": "https://openalex.org/I27577105", "display_name": "University College Cork", "ror": "https://ror.org/03265fv13", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I181231927", "https://openalex.org/I27577105"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Alan Blake", "raw_affiliation_string": "Tyndall National Institute, University College Cork, Cork, Ireland", "raw_affiliation_strings": ["Tyndall National Institute, University College Cork, Cork, Ireland"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5060577406", "display_name": "Mark H. White", "orcid": "https://orcid.org/0000-0003-4073-3519"}, "institutions": [{"id": "https://openalex.org/I27577105", "display_name": "University College Cork", "ror": "https://ror.org/03265fv13", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I181231927", "https://openalex.org/I27577105"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Mary White", "raw_affiliation_string": "Tyndall National Institute, University College Cork, Cork, Ireland", "raw_affiliation_strings": ["Tyndall National Institute, University College Cork, Cork, Ireland"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5034493709", "display_name": "Ann Kelleher", "orcid": null}, "institutions": [{"id": "https://openalex.org/I27577105", "display_name": "University College Cork", "ror": "https://ror.org/03265fv13", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I181231927", "https://openalex.org/I27577105"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Anne-Marie Kelleher", "raw_affiliation_string": "Tyndall National Institute, University College Cork, Cork, Ireland", "raw_affiliation_strings": ["Tyndall National Institute, University College Cork, Cork, Ireland"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5008227887", "display_name": "Brendan McCarthy", "orcid": null}, "institutions": [{"id": "https://openalex.org/I27577105", "display_name": "University College Cork", "ror": "https://ror.org/03265fv13", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I181231927", "https://openalex.org/I27577105"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Brendan McCarthy", "raw_affiliation_string": "Tyndall National Institute, University College Cork, Cork, Ireland", "raw_affiliation_strings": ["Tyndall National Institute, University College Cork, Cork, Ireland"]}, {"author_position": "last", "author": {"id": "https://openalex.org/A5079833472", "display_name": "Richard J. Murphy", "orcid": null}, "institutions": [{"id": "https://openalex.org/I27577105", "display_name": "University College Cork", "ror": "https://ror.org/03265fv13", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I181231927", "https://openalex.org/I27577105"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Richard Murphy", "raw_affiliation_string": "Tyndall National Institute, University College Cork, Cork, Ireland", "raw_affiliation_strings": ["Tyndall National Institute, University College Cork, Cork, Ireland"]}], "countries_distinct_count": 2, "institutions_distinct_count": 2, "corresponding_author_ids": ["https://openalex.org/A5055173231"], "corresponding_institution_ids": ["https://openalex.org/I27577105"], "apc_list": {"value": 9750, "currency": "EUR", "value_usd": 11690, "provenance": "doaj"}, "apc_paid": {"value": 9750, "currency": "EUR", "value_usd": 11690, "provenance": "doaj"}, "has_fulltext": true, "fulltext_origin": "ngrams", "cited_by_count": 1960, "cited_by_percentile_year": {"min": 99.9, "max": 100.0}, "biblio": {"volume": "5", "issue": "3", "first_page": "225", "last_page": "229"}, "is_retracted": false, "is_paratext": false, "keywords": [{"keyword": "transistors", "score": 0.6086}, {"keyword": "junctions", "score": 0.5641}], "concepts": [{"id": "https://openalex.org/C172385210", "wikidata": "https://www.wikidata.org/wiki/Q5339", "display_name": "Transistor", "level": 3, "score": 0.7854309}, {"id": "https://openalex.org/C192562407", "wikidata": "https://www.wikidata.org/wiki/Q228736", "display_name": "Materials science", "level": 0, "score": 0.75537133}, {"id": "https://openalex.org/C57863236", "wikidata": "https://www.wikidata.org/wiki/Q1130571", "display_name": "Doping", "level": 2, "score": 0.74234545}, {"id": "https://openalex.org/C74214498", "wikidata": "https://www.wikidata.org/wiki/Q631739", "display_name": "Nanowire", "level": 2, "score": 0.70526314}, {"id": "https://openalex.org/C49040817", "wikidata": "https://www.wikidata.org/wiki/Q193091", "display_name": "Optoelectronics", "level": 1, "score": 0.6840037}, {"id": "https://openalex.org/C156465305", "wikidata": "https://www.wikidata.org/wiki/Q1658601", "display_name": "Subthreshold conduction", "level": 4, "score": 0.6046566}, {"id": "https://openalex.org/C108225325", "wikidata": "https://www.wikidata.org/wiki/Q11456", "display_name": "Semiconductor", "level": 2, "score": 0.59717506}, {"id": "https://openalex.org/C191952053", "wikidata": "https://www.wikidata.org/wiki/Q15119237", "display_name": "Dopant", "level": 3, "score": 0.5878197}, {"id": "https://openalex.org/C46362747", "wikidata": "https://www.wikidata.org/wiki/Q173431", "display_name": "CMOS", "level": 2, "score": 0.5777692}, {"id": "https://openalex.org/C544956773", "wikidata": "https://www.wikidata.org/wiki/Q670", "display_name": "Silicon", "level": 2, "score": 0.5241428}, {"id": "https://openalex.org/C171250308", "wikidata": "https://www.wikidata.org/wiki/Q11468", "display_name": "Nanotechnology", "level": 1, "score": 0.4966994}, {"id": "https://openalex.org/C103566474", "wikidata": "https://www.wikidata.org/wiki/Q7632226", "display_name": "Subthreshold slope", "level": 5, "score": 0.47232193}, {"id": "https://openalex.org/C136525101", "wikidata": "https://www.wikidata.org/wiki/Q5428139", "display_name": "Fabrication", "level": 3, "score": 0.45515507}, {"id": "https://openalex.org/C195370968", "wikidata": "https://www.wikidata.org/wiki/Q1754002", "display_name": "Threshold voltage", "level": 4, "score": 0.4088757}, {"id": "https://openalex.org/C165801399", "wikidata": "https://www.wikidata.org/wiki/Q25428", "display_name": "Voltage", "level": 2, "score": 0.3053121}, {"id": "https://openalex.org/C119599485", "wikidata": "https://www.wikidata.org/wiki/Q43035", "display_name": "Electrical engineering", "level": 1, "score": 0.1837199}, {"id": "https://openalex.org/C71924100", "wikidata": "https://www.wikidata.org/wiki/Q11190", "display_name": "Medicine", "level": 0, "score": 0.0}, {"id": "https://openalex.org/C204787440", "wikidata": "https://www.wikidata.org/wiki/Q188504", "display_name": "Alternative medicine", "level": 2, "score": 0.0}, {"id": "https://openalex.org/C142724271", "wikidata": "https://www.wikidata.org/wiki/Q7208", "display_name": "Pathology", "level": 1, "score": 0.0}, {"id": "https://openalex.org/C127413603", "wikidata": "https://www.wikidata.org/wiki/Q11023", "display_name": "Engineering", "level": 0, "score": 0.0}], "mesh": [], "locations_count": 2, "locations": [{"is_oa": false, "landing_page_url": "https://doi.org/10.1038/nnano.2010.15", "pdf_url": null, "source": {"id": "https://openalex.org/S7822423", "display_name": "Nature Nanotechnology", "issn_l": "1748-3387", "issn": ["1748-3395", "1748-3387"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310319908", "host_organization_name": "Nature Portfolio", "host_organization_lineage": ["https://openalex.org/P4310319908", "https://openalex.org/P4310319965"], "host_organization_lineage_names": ["Nature Portfolio", "Springer Nature"], "type": "journal"}, "license": null, "version": null, "is_accepted": false, "is_published": false}, {"is_oa": false, "landing_page_url": "https://pubmed.ncbi.nlm.nih.gov/20173755", "pdf_url": null, "source": {"id": "https://openalex.org/S4306525036", "display_name": "PubMed", "issn_l": null, "issn": null, "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/I1299303238", "host_organization_name": "National Institutes of Health", "host_organization_lineage": ["https://openalex.org/I1299303238"], "host_organization_lineage_names": ["National Institutes of Health"], "type": "repository"}, "license": null, "version": null, "is_accepted": false, "is_published": false}], "best_oa_location": null, "sustainable_development_goals": [{"id": "https://metadata.un.org/sdg/7", "display_name": "Affordable and clean energy", "score": 0.78}], "grants": [], "referenced_works_count": 13, "referenced_works": ["https://openalex.org/W1971662529", "https://openalex.org/W1973711911", "https://openalex.org/W1976894262", "https://openalex.org/W2011148359", "https://openalex.org/W2014718851", "https://openalex.org/W2026399749", "https://openalex.org/W2038305545", "https://openalex.org/W2051721756", "https://openalex.org/W2092333682", "https://openalex.org/W2099168929", "https://openalex.org/W2116656110", "https://openalex.org/W2128762553", "https://openalex.org/W2154869256"], "related_works": ["https://openalex.org/W2000425643", "https://openalex.org/W2095078040", "https://openalex.org/W2062767191", "https://openalex.org/W2105853365", "https://openalex.org/W2117738807", "https://openalex.org/W1978942334", "https://openalex.org/W4231458110", "https://openalex.org/W2786811717", "https://openalex.org/W1186362247", "https://openalex.org/W1995720339"], "ngrams_url": "https://api.openalex.org/works/W2119378720/ngrams", "abstract_inverted_index": null, "cited_by_api_url": "https://api.openalex.org/works?filter=cites:W2119378720", "counts_by_year": [{"year": 2023, "cited_by_count": 97}, {"year": 2022, "cited_by_count": 125}, {"year": 2021, "cited_by_count": 176}, {"year": 2020, "cited_by_count": 180}, {"year": 2019, "cited_by_count": 182}, {"year": 2018, "cited_by_count": 156}, {"year": 2017, "cited_by_count": 169}, {"year": 2016, "cited_by_count": 179}, {"year": 2015, "cited_by_count": 142}, {"year": 2014, "cited_by_count": 158}, {"year": 2013, "cited_by_count": 173}, {"year": 2012, "cited_by_count": 113}], "updated_date": "2023-12-06T01:29:20.598863", "created_date": "2016-06-24"} +{"id": "https://openalex.org/W2140206763", "doi": "https://doi.org/10.1038/nature08900", "title": "Nuocytes represent a new innate effector leukocyte that mediates type-2 immunity", "display_name": "Nuocytes represent a new innate effector leukocyte that mediates type-2 immunity", "publication_year": 2010, "publication_date": "2010-04-01", "ids": {"openalex": "https://openalex.org/W2140206763", "doi": "https://doi.org/10.1038/nature08900", "mag": "2140206763", "pmid": "https://pubmed.ncbi.nlm.nih.gov/20200518", "pmcid": "https://www.ncbi.nlm.nih.gov/pmc/articles/2862165"}, "language": "en", "primary_location": {"is_oa": false, "landing_page_url": "https://doi.org/10.1038/nature08900", "pdf_url": null, "source": {"id": "https://openalex.org/S137773608", "display_name": "Nature", "issn_l": "0028-0836", "issn": ["1476-4687", "0028-0836"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310319908", "host_organization_name": "Nature Portfolio", "host_organization_lineage": ["https://openalex.org/P4310319908", "https://openalex.org/P4310319965"], "host_organization_lineage_names": ["Nature Portfolio", "Springer Nature"], "type": "journal"}, "license": null, "version": null, "is_accepted": false, "is_published": false}, "type": "article", "type_crossref": "journal-article", "open_access": {"is_oa": true, "oa_status": "green", "oa_url": "https://europepmc.org/articles/pmc2862165?pdf=render", "any_repository_has_fulltext": true}, "authorships": [{"author_position": "first", "author": {"id": "https://openalex.org/A5029038542", "display_name": "Daniel R. Neill", "orcid": "https://orcid.org/0000-0002-7911-8153"}, "institutions": [{"id": "https://openalex.org/I170203145", "display_name": "MRC Laboratory of Molecular Biology", "ror": "https://ror.org/00tw3jy02", "country_code": "GB", "type": "facility", "lineage": ["https://openalex.org/I170203145", "https://openalex.org/I90344618"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Daniel R. Neill", "raw_affiliation_string": "MRC Laboratory of Molecular Biology, Hills Road, Cambridge CB2 0QH, UK ,", "raw_affiliation_strings": ["MRC Laboratory of Molecular Biology, Hills Road, Cambridge CB2 0QH, UK ,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5001249567", "display_name": "See Heng Wong", "orcid": null}, "institutions": [{"id": "https://openalex.org/I170203145", "display_name": "MRC Laboratory of Molecular Biology", "ror": "https://ror.org/00tw3jy02", "country_code": "GB", "type": "facility", "lineage": ["https://openalex.org/I170203145", "https://openalex.org/I90344618"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "See Heng Wong", "raw_affiliation_string": "MRC Laboratory of Molecular Biology, Hills Road, Cambridge CB2 0QH, UK ,", "raw_affiliation_strings": ["MRC Laboratory of Molecular Biology, Hills Road, Cambridge CB2 0QH, UK ,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5064495627", "display_name": "Agustin Bellosi", "orcid": null}, "institutions": [{"id": "https://openalex.org/I170203145", "display_name": "MRC Laboratory of Molecular Biology", "ror": "https://ror.org/00tw3jy02", "country_code": "GB", "type": "facility", "lineage": ["https://openalex.org/I170203145", "https://openalex.org/I90344618"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Agustin Bellosi", "raw_affiliation_string": "MRC Laboratory of Molecular Biology, Hills Road, Cambridge CB2 0QH, UK ,", "raw_affiliation_strings": ["MRC Laboratory of Molecular Biology, Hills Road, Cambridge CB2 0QH, UK ,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5014773401", "display_name": "Robin J. Flynn", "orcid": "https://orcid.org/0000-0001-5304-3088"}, "institutions": [{"id": "https://openalex.org/I170203145", "display_name": "MRC Laboratory of Molecular Biology", "ror": "https://ror.org/00tw3jy02", "country_code": "GB", "type": "facility", "lineage": ["https://openalex.org/I170203145", "https://openalex.org/I90344618"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Robin J. Flynn", "raw_affiliation_string": "MRC Laboratory of Molecular Biology, Hills Road, Cambridge CB2 0QH, UK ,", "raw_affiliation_strings": ["MRC Laboratory of Molecular Biology, Hills Road, Cambridge CB2 0QH, UK ,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5061316505", "display_name": "Maria Daly", "orcid": null}, "institutions": [{"id": "https://openalex.org/I170203145", "display_name": "MRC Laboratory of Molecular Biology", "ror": "https://ror.org/00tw3jy02", "country_code": "GB", "type": "facility", "lineage": ["https://openalex.org/I170203145", "https://openalex.org/I90344618"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Maria Daly", "raw_affiliation_string": "MRC Laboratory of Molecular Biology, Hills Road, Cambridge CB2 0QH, UK ,", "raw_affiliation_strings": ["MRC Laboratory of Molecular Biology, Hills Road, Cambridge CB2 0QH, UK ,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5010686483", "display_name": "Theresa K. A. Langford", "orcid": null}, "institutions": [{"id": "https://openalex.org/I170203145", "display_name": "MRC Laboratory of Molecular Biology", "ror": "https://ror.org/00tw3jy02", "country_code": "GB", "type": "facility", "lineage": ["https://openalex.org/I170203145", "https://openalex.org/I90344618"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Theresa K. A. Langford", "raw_affiliation_string": "MRC Laboratory of Molecular Biology, Hills Road, Cambridge CB2 0QH, UK ,", "raw_affiliation_strings": ["MRC Laboratory of Molecular Biology, Hills Road, Cambridge CB2 0QH, UK ,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5063462454", "display_name": "Christine M. Bucks", "orcid": null}, "institutions": [], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Christine Bucks", "raw_affiliation_string": "Immunology Discovery Research, Centocor R&D Inc., 145 King of Prussia Road, Radnor, Pennsylvania 19087, USA ,", "raw_affiliation_strings": ["Immunology Discovery Research, Centocor R&D Inc., 145 King of Prussia Road, Radnor, Pennsylvania 19087, USA ,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5038258136", "display_name": "Colleen Kane", "orcid": null}, "institutions": [], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Colleen M. Kane", "raw_affiliation_string": "Immunology Discovery Research, Centocor R&D Inc., 145 King of Prussia Road, Radnor, Pennsylvania 19087, USA ,", "raw_affiliation_strings": ["Immunology Discovery Research, Centocor R&D Inc., 145 King of Prussia Road, Radnor, Pennsylvania 19087, USA ,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5076098159", "display_name": "Padraic G. Fallon", "orcid": "https://orcid.org/0000-0002-8401-7293"}, "institutions": [{"id": "https://openalex.org/I205274468", "display_name": "Trinity College Dublin", "ror": "https://ror.org/02tyrky19", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I205274468"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Padraic G. Fallon", "raw_affiliation_string": "Institute of Molecular Medicine, Trinity College Dublin, Dublin 8, Ireland", "raw_affiliation_strings": ["Institute of Molecular Medicine, Trinity College Dublin, Dublin 8, Ireland"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5069173472", "display_name": "Richard Pannell", "orcid": null}, "institutions": [{"id": "https://openalex.org/I170203145", "display_name": "MRC Laboratory of Molecular Biology", "ror": "https://ror.org/00tw3jy02", "country_code": "GB", "type": "facility", "lineage": ["https://openalex.org/I170203145", "https://openalex.org/I90344618"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Richard Pannell", "raw_affiliation_string": "MRC Laboratory of Molecular Biology, Hills Road, Cambridge CB2 0QH, UK ,", "raw_affiliation_strings": ["MRC Laboratory of Molecular Biology, Hills Road, Cambridge CB2 0QH, UK ,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5031918867", "display_name": "Helen E. Jolin", "orcid": null}, "institutions": [{"id": "https://openalex.org/I170203145", "display_name": "MRC Laboratory of Molecular Biology", "ror": "https://ror.org/00tw3jy02", "country_code": "GB", "type": "facility", "lineage": ["https://openalex.org/I170203145", "https://openalex.org/I90344618"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Helen E. Jolin", "raw_affiliation_string": "MRC Laboratory of Molecular Biology, Hills Road, Cambridge CB2 0QH, UK ,", "raw_affiliation_strings": ["MRC Laboratory of Molecular Biology, Hills Road, Cambridge CB2 0QH, UK ,"]}, {"author_position": "last", "author": {"id": "https://openalex.org/A5013123507", "display_name": "Andrew N. J. McKenzie", "orcid": "https://orcid.org/0000-0003-4018-4273"}, "institutions": [{"id": "https://openalex.org/I170203145", "display_name": "MRC Laboratory of Molecular Biology", "ror": "https://ror.org/00tw3jy02", "country_code": "GB", "type": "facility", "lineage": ["https://openalex.org/I170203145", "https://openalex.org/I90344618"]}], "countries": ["GB"], "is_corresponding": true, "raw_author_name": "Andrew N. J. McKenzie", "raw_affiliation_string": "MRC Laboratory of Molecular Biology, Hills Road, Cambridge CB2 0QH, UK ,", "raw_affiliation_strings": ["MRC Laboratory of Molecular Biology, Hills Road, Cambridge CB2 0QH, UK ,"]}], "countries_distinct_count": 3, "institutions_distinct_count": 2, "corresponding_author_ids": ["https://openalex.org/A5013123507"], "corresponding_institution_ids": ["https://openalex.org/I170203145"], "apc_list": {"value": 9750, "currency": "EUR", "value_usd": 11690, "provenance": "doaj"}, "apc_paid": {"value": 9750, "currency": "EUR", "value_usd": 11690, "provenance": "doaj"}, "has_fulltext": true, "fulltext_origin": "ngrams", "cited_by_count": 1797, "cited_by_percentile_year": {"min": 99.9, "max": 100.0}, "biblio": {"volume": "464", "issue": "7293", "first_page": "1367", "last_page": "1370"}, "is_retracted": false, "is_paratext": false, "keywords": [{"keyword": "new innate effector leukocyte", "score": 0.6539}, {"keyword": "nuocytes", "score": 0.559}, {"keyword": "immunity", "score": 0.4512}], "concepts": [{"id": "https://openalex.org/C203014093", "wikidata": "https://www.wikidata.org/wiki/Q101929", "display_name": "Immunology", "level": 1, "score": 0.75011593}, {"id": "https://openalex.org/C136449434", "wikidata": "https://www.wikidata.org/wiki/Q428253", "display_name": "Innate immune system", "level": 3, "score": 0.72261953}, {"id": "https://openalex.org/C86803240", "wikidata": "https://www.wikidata.org/wiki/Q420", "display_name": "Biology", "level": 0, "score": 0.6922859}, {"id": "https://openalex.org/C193419808", "wikidata": "https://www.wikidata.org/wiki/Q1645075", "display_name": "Acquired immune system", "level": 3, "score": 0.5698725}, {"id": "https://openalex.org/C2779341262", "wikidata": "https://www.wikidata.org/wiki/Q182581", "display_name": "Immunity", "level": 3, "score": 0.5462368}, {"id": "https://openalex.org/C47742525", "wikidata": "https://www.wikidata.org/wiki/Q13418826", "display_name": "Innate lymphoid cell", "level": 4, "score": 0.53756845}, {"id": "https://openalex.org/C8891405", "wikidata": "https://www.wikidata.org/wiki/Q1059", "display_name": "Immune system", "level": 2, "score": 0.52691215}, {"id": "https://openalex.org/C2908647359", "wikidata": "https://www.wikidata.org/wiki/Q2625603", "display_name": "Population", "level": 2, "score": 0.5237315}, {"id": "https://openalex.org/C2778690821", "wikidata": "https://www.wikidata.org/wiki/Q212354", "display_name": "Cytokine", "level": 2, "score": 0.4878931}, {"id": "https://openalex.org/C71924100", "wikidata": "https://www.wikidata.org/wiki/Q11190", "display_name": "Medicine", "level": 0, "score": 0.14353707}, {"id": "https://openalex.org/C99454951", "wikidata": "https://www.wikidata.org/wiki/Q932068", "display_name": "Environmental health", "level": 1, "score": 0.0}], "mesh": [{"descriptor_ui": "D007113", "descriptor_name": "Immunity, Innate", "qualifier_ui": "Q000276", "qualifier_name": "immunology", "is_major_topic": true}, {"descriptor_ui": "D007378", "descriptor_name": "Interleukins", "qualifier_ui": "Q000276", "qualifier_name": "immunology", "is_major_topic": true}, {"descriptor_ui": "D007962", "descriptor_name": "Leukocytes", "qualifier_ui": "Q000276", "qualifier_name": "immunology", "is_major_topic": true}, {"descriptor_ui": "D018418", "descriptor_name": "Th2 Cells", "qualifier_ui": "Q000276", "qualifier_name": "immunology", "is_major_topic": true}, {"descriptor_ui": "D019264", "descriptor_name": "Adoptive Transfer", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D000818", "descriptor_name": "Animals", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D002478", "descriptor_name": "Cells, Cultured", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D007113", "descriptor_name": "Immunity, Innate", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D018793", "descriptor_name": "Interleukin-13", "qualifier_ui": "Q000096", "qualifier_name": "biosynthesis", "is_major_topic": false}, {"descriptor_ui": "D018793", "descriptor_name": "Interleukin-13", "qualifier_ui": "Q000172", "qualifier_name": "deficiency", "is_major_topic": false}, {"descriptor_ui": "D018793", "descriptor_name": "Interleukin-13", "qualifier_ui": "Q000235", "qualifier_name": "genetics", "is_major_topic": false}, {"descriptor_ui": "D018793", "descriptor_name": "Interleukin-13", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D020381", "descriptor_name": "Interleukin-17", "qualifier_ui": "Q000172", "qualifier_name": "deficiency", "is_major_topic": false}, {"descriptor_ui": "D020381", "descriptor_name": "Interleukin-17", "qualifier_ui": "Q000235", "qualifier_name": "genetics", "is_major_topic": false}, {"descriptor_ui": "D020381", "descriptor_name": "Interleukin-17", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D007378", "descriptor_name": "Interleukins", "qualifier_ui": "Q000096", "qualifier_name": "biosynthesis", "is_major_topic": false}, {"descriptor_ui": "D007378", "descriptor_name": "Interleukins", "qualifier_ui": "Q000172", "qualifier_name": "deficiency", "is_major_topic": false}, {"descriptor_ui": "D007378", "descriptor_name": "Interleukins", "qualifier_ui": "Q000235", "qualifier_name": "genetics", "is_major_topic": false}, {"descriptor_ui": "D007378", "descriptor_name": "Interleukins", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D007962", "descriptor_name": "Leukocytes", "qualifier_ui": "Q000166", "qualifier_name": "cytology", "is_major_topic": false}, {"descriptor_ui": "D007962", "descriptor_name": "Leukocytes", "qualifier_ui": "Q000378", "qualifier_name": "metabolism", "is_major_topic": false}, {"descriptor_ui": "D007962", "descriptor_name": "Leukocytes", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D051379", "descriptor_name": "Mice", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D008807", "descriptor_name": "Mice, Inbred BALB C", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D008810", "descriptor_name": "Mice, Inbred C57BL", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D009559", "descriptor_name": "Nippostrongylus", "qualifier_ui": "Q000276", "qualifier_name": "immunology", "is_major_topic": false}, {"descriptor_ui": "D009559", "descriptor_name": "Nippostrongylus", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D017206", "descriptor_name": "Strongylida Infections", "qualifier_ui": "Q000276", "qualifier_name": "immunology", "is_major_topic": false}, {"descriptor_ui": "D017206", "descriptor_name": "Strongylida Infections", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D018418", "descriptor_name": "Th2 Cells", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}], "locations_count": 4, "locations": [{"is_oa": false, "landing_page_url": "https://doi.org/10.1038/nature08900", "pdf_url": null, "source": {"id": "https://openalex.org/S137773608", "display_name": "Nature", "issn_l": "0028-0836", "issn": ["1476-4687", "0028-0836"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310319908", "host_organization_name": "Nature Portfolio", "host_organization_lineage": ["https://openalex.org/P4310319908", "https://openalex.org/P4310319965"], "host_organization_lineage_names": ["Nature Portfolio", "Springer Nature"], "type": "journal"}, "license": null, "version": null, "is_accepted": false, "is_published": false}, {"is_oa": true, "landing_page_url": "https://europepmc.org/articles/pmc2862165", "pdf_url": "https://europepmc.org/articles/pmc2862165?pdf=render", "source": {"id": "https://openalex.org/S4306400806", "display_name": "Europe PMC (PubMed Central)", "issn_l": null, "issn": null, "is_oa": true, "is_in_doaj": false, "host_organization": "https://openalex.org/I1303153112", "host_organization_name": "European Bioinformatics Institute", "host_organization_lineage": ["https://openalex.org/I1303153112"], "host_organization_lineage_names": ["European Bioinformatics Institute"], "type": "repository"}, "license": "implied-oa", "version": "acceptedVersion", "is_accepted": true, "is_published": false}, {"is_oa": true, "landing_page_url": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2862165", "pdf_url": null, "source": {"id": "https://openalex.org/S2764455111", "display_name": "PubMed Central", "issn_l": null, "issn": null, "is_oa": true, "is_in_doaj": false, "host_organization": "https://openalex.org/I1299303238", "host_organization_name": "National Institutes of Health", "host_organization_lineage": ["https://openalex.org/I1299303238"], "host_organization_lineage_names": ["National Institutes of Health"], "type": "repository"}, "license": null, "version": "acceptedVersion", "is_accepted": true, "is_published": false}, {"is_oa": false, "landing_page_url": "https://pubmed.ncbi.nlm.nih.gov/20200518", "pdf_url": null, "source": {"id": "https://openalex.org/S4306525036", "display_name": "PubMed", "issn_l": null, "issn": null, "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/I1299303238", "host_organization_name": "National Institutes of Health", "host_organization_lineage": ["https://openalex.org/I1299303238"], "host_organization_lineage_names": ["National Institutes of Health"], "type": "repository"}, "license": null, "version": null, "is_accepted": false, "is_published": false}], "best_oa_location": {"is_oa": true, "landing_page_url": "https://europepmc.org/articles/pmc2862165", "pdf_url": "https://europepmc.org/articles/pmc2862165?pdf=render", "source": {"id": "https://openalex.org/S4306400806", "display_name": "Europe PMC (PubMed Central)", "issn_l": null, "issn": null, "is_oa": true, "is_in_doaj": false, "host_organization": "https://openalex.org/I1303153112", "host_organization_name": "European Bioinformatics Institute", "host_organization_lineage": ["https://openalex.org/I1303153112"], "host_organization_lineage_names": ["European Bioinformatics Institute"], "type": "repository"}, "license": "implied-oa", "version": "acceptedVersion", "is_accepted": true, "is_published": false}, "sustainable_development_goals": [{"id": "https://metadata.un.org/sdg/3", "display_name": "Good health and well-being", "score": 0.3}, {"id": "https://metadata.un.org/sdg/12", "display_name": "Responsible consumption and production", "score": 0.11}], "grants": [], "referenced_works_count": 24, "referenced_works": ["https://openalex.org/W1597600237", "https://openalex.org/W1980250352", "https://openalex.org/W1987699155", "https://openalex.org/W1992904489", "https://openalex.org/W1995529916", "https://openalex.org/W2019870550", "https://openalex.org/W2021454776", "https://openalex.org/W2052122855", "https://openalex.org/W2065708991", "https://openalex.org/W2066978721", "https://openalex.org/W2083207596", "https://openalex.org/W2086782209", "https://openalex.org/W2101500685", "https://openalex.org/W2125805192", "https://openalex.org/W2126464081", "https://openalex.org/W2129115323", "https://openalex.org/W2135751988", "https://openalex.org/W2140166442", "https://openalex.org/W2142899869", "https://openalex.org/W2143885071", "https://openalex.org/W2151214732", "https://openalex.org/W2154483825", "https://openalex.org/W2165933222", "https://openalex.org/W2376856463"], "related_works": ["https://openalex.org/W4234587088", "https://openalex.org/W4249576530", "https://openalex.org/W1551592253", "https://openalex.org/W2135436588", "https://openalex.org/W2050558417", "https://openalex.org/W4225914417", "https://openalex.org/W2087547757", "https://openalex.org/W3030144597", "https://openalex.org/W2498374562", "https://openalex.org/W4307359803"], "ngrams_url": "https://api.openalex.org/works/W2140206763/ngrams", "abstract_inverted_index": {"Type-2": [0, 214], "immunity,": [1], "the": [2, 14, 36, 77, 104, 141, 156, 172, 175, 196, 209, 224, 228, 237, 262, 273, 282, 299, 308, 316, 329, 353], "ancient": [3], "defence": [4, 200], "mechanism": [5], "that": [6, 21, 58, 72, 166, 178, 184, 295, 349], "provides": [7, 195, 205], "protection": [8, 189], "against": [9, 190, 201], "gastrointestinal": [10], "helminth": [11, 112, 191, 221, 323], "infections,": [12], "involves": [13], "recruitment": [15, 37], "of": [16, 38, 108, 143, 150, 174, 199, 211, 227, 230, 233, 254, 275, 287, 320, 332, 356], "T": [17, 248], "helper": [18], "(TH)": [19], "cells": [20, 71, 180, 249], "produce": [22], "immune": [23, 30, 218, 259, 292], "mediators": [24], "or": [25, 86], "cytokines": [26, 240, 256, 310], "to": [27, 76, 84, 220, 267, 307, 339], "coordinate": [28], "an": [29, 251], "response": [31, 75, 306], "involving": [32], "IgE": [33], "antibody": [34], "production,": [35], "eosinophils": [39], "and": [40, 81, 100, 126, 161, 169, 183, 204, 223, 244, 284, 312, 314, 334], "goblet": [41], "cell": [42, 154, 264, 373], "hyperplasia.": [43], "Two": [44], "groups": [45], "reporting": [46], "in": [47, 74, 129, 257, 303, 305, 342, 346, 357, 374], "this": [48], "issue": [49], "have": [50, 297], "characterized": [51], "innate": [52, 144, 151, 263, 290, 371], "type": [53, 149], "2": [54], "effector": [55, 152, 293, 372], "leukocyte": [56, 153, 294], "populations": [57], "promote": [59], "TH2": [60, 90], "cytokine": [61, 78], "responses.": [62], "Saenz": [63], "et": [64, 93], "al.": [65, 94], "describe": [66, 95], "multipotent": [67], "progenitor": [68], "type-2": [69, 239, 291, 375], "(MPPtype2)": [70], "accumulate": [73], "IL-25": [79], "(interleukin-25)": [80], "give": [82], "rise": [83], "macrophage": [85], "granulocyte": [87], "lineages": [88], "promoting": [89], "differentiation.": [91], "Neill": [92], "'nuocytes',": [96], "induced": [97], "by": [98, 236, 352], "IL25": [99, 311, 333], "IL33,": [101, 313], "which": [102], "are": [103, 186, 250], "predominant": [105, 317], "early": [106, 318], "source": [107, 253, 319], "IL13": [109, 245, 321], "during": [110, 322], "a": [111, 127, 147, 288, 343, 368], "infection.": [113, 192], "In": [114, 328], "News": [115], "&": [116], "Views,": [117], "Gérard": [118], "Eberl": [119], "discusses": [120], "how": [121], "these": [122, 179, 255], "two": [123], "papers": [124], "—": [125, 136, 155, 158], "third": [128], "Nature": [130], "Reviews": [131], "Immunology": [132], "(": [133], "http://go.nature.com/sJ9D77": [134], ")": [135], "influence": [137], "current": [138], "thinking": [139], "on": [140], "role": [142], "immunity.": [145, 213, 376], "Here,": [146, 271], "new": [148, 289], "nuocyte": [157, 176], "is": [159, 164, 350], "described": [160], "characterized.": [162], "It": [163], "shown": [165], "interleukin": [167, 241], "(IL)25": [168], "IL33": [170, 335], "drive": [171], "expansion": [173], "population,": [177], "secrete": [181], "IL13,": [182], "they": [185], "required": [187], "for": [188, 208, 216], "Innate": [193], "immunity": [194], "first": [197], "line": [198], "invading": [202], "pathogens": [203], "important": [206, 252, 370], "cues": [207], "development": [210], "adaptive": [212, 258], "immunity—responsible": [215], "protective": [217], "responses": [219, 234], "parasites1,2": [222], "underlying": [225], "cause": [226], "pathogenesis": [229], "allergic": [231], "asthma3,4—consists": [232], "dominated": [235], "cardinal": [238], "(IL)4,": [242], "IL5": [243], "(ref.": [246], "5).": [247], "responses,": [260], "but": [261, 361], "sources": [265], "remain": [266], "be": [268], "comprehensively": [269], "determined.": [270], "through": [272], "use": [274], "novel": [276], "Il13-eGFP": [277], "reporter": [278], "mice,": [279], "we": [280, 296], "present": [281], "identification": [283], "functional": [285], "characterization": [286], "named": [298], "nuocyte.": [300], "Nuocytes": [301], "expand": [302], "vivo": [304], "type-2-inducing": [309], "represent": [315, 367], "infection": [324], "with": [325], "Nippostrongylus": [326], "brasiliensis.": [327], "combined": [330], "absence": [331], "signalling,": [336], "nuocytes": [337, 366], "fail": [338], "expand,": [340], "resulting": [341], "severe": [344], "defect": [345], "worm": [347], "expulsion": [348], "rescued": [351], "adoptive": [354], "transfer": [355], "vitro": [358], "cultured": [359], "wild-type,": [360], "not": [362], "IL13-deficient,": [363], "nuocytes.": [364], "Thus,": [365], "critically": [369]}, "cited_by_api_url": "https://api.openalex.org/works?filter=cites:W2140206763", "counts_by_year": [{"year": 2023, "cited_by_count": 75}, {"year": 2022, "cited_by_count": 123}, {"year": 2021, "cited_by_count": 130}, {"year": 2020, "cited_by_count": 117}, {"year": 2019, "cited_by_count": 141}, {"year": 2018, "cited_by_count": 132}, {"year": 2017, "cited_by_count": 157}, {"year": 2016, "cited_by_count": 161}, {"year": 2015, "cited_by_count": 164}, {"year": 2014, "cited_by_count": 173}, {"year": 2013, "cited_by_count": 164}, {"year": 2012, "cited_by_count": 127}], "updated_date": "2023-12-03T12:59:28.615768", "created_date": "2016-06-24"} +{"id": "https://openalex.org/W2110374888", "doi": "https://doi.org/10.1038/nature09146", "title": "Functional impact of global rare copy number variation in autism spectrum disorders", "display_name": "Functional impact of global rare copy number variation in autism spectrum disorders", "publication_year": 2010, "publication_date": "2010-06-09", "ids": {"openalex": "https://openalex.org/W2110374888", "doi": "https://doi.org/10.1038/nature09146", "mag": "2110374888", "pmid": "https://pubmed.ncbi.nlm.nih.gov/20531469", "pmcid": "https://www.ncbi.nlm.nih.gov/pmc/articles/3021798"}, "language": "en", "primary_location": {"is_oa": true, "landing_page_url": "https://doi.org/10.1038/nature09146", "pdf_url": "https://www.nature.com/articles/nature09146.pdf", "source": {"id": "https://openalex.org/S137773608", "display_name": "Nature", "issn_l": "0028-0836", "issn": ["1476-4687", "0028-0836"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310319908", "host_organization_name": "Nature Portfolio", "host_organization_lineage": ["https://openalex.org/P4310319908", "https://openalex.org/P4310319965"], "host_organization_lineage_names": ["Nature Portfolio", "Springer Nature"], "type": "journal"}, "license": null, "version": "publishedVersion", "is_accepted": true, "is_published": true}, "type": "article", "type_crossref": "journal-article", "open_access": {"is_oa": true, "oa_status": "bronze", "oa_url": "https://www.nature.com/articles/nature09146.pdf", "any_repository_has_fulltext": true}, "authorships": [{"author_position": "first", "author": {"id": "https://openalex.org/A5088479227", "display_name": "Dalila Pinto", "orcid": "https://orcid.org/0000-0002-8769-0846"}, "institutions": [{"id": "https://openalex.org/I2801317318", "display_name": "Hospital for Sick Children", "ror": "https://ror.org/057q4rt57", "country_code": "CA", "type": "healthcare", "lineage": ["https://openalex.org/I2801317318"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "Dalila Pinto", "raw_affiliation_string": "The Centre for Applied Genomics and Program in Genetics and Genomic Biology, The Hospital for Sick Children, Toronto, Ontario M5G 1L7, Canada.,", "raw_affiliation_strings": ["The Centre for Applied Genomics and Program in Genetics and Genomic Biology, The Hospital for Sick Children, Toronto, Ontario M5G 1L7, Canada.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5052845737", "display_name": "Alistair T. Pagnamenta", "orcid": null}, "institutions": [{"id": "https://openalex.org/I1336263701", "display_name": "Wellcome Centre for Human Genetics", "ror": "https://ror.org/01rjnta51", "country_code": "GB", "type": "facility", "lineage": ["https://openalex.org/I1336263701", "https://openalex.org/I40120149", "https://openalex.org/I87048295"]}, {"id": "https://openalex.org/I40120149", "display_name": "University of Oxford", "ror": "https://ror.org/052gg0110", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I40120149"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Alistair T. Pagnamenta", "raw_affiliation_string": "Wellcome Trust Centre for Human Genetics, University of Oxford, Oxford OX3 7BN, UK.,", "raw_affiliation_strings": ["Wellcome Trust Centre for Human Genetics, University of Oxford, Oxford OX3 7BN, UK.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5068330330", "display_name": "Lambertus Klei", "orcid": null}, "institutions": [{"id": "https://openalex.org/I170201317", "display_name": "University of Pittsburgh", "ror": "https://ror.org/01an3r305", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I170201317"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Lambertus Klei", "raw_affiliation_string": "Department of Psychiatry, University of Pittsburgh School of Medicine, Pittsburgh, Pennsylvania 15213, USA.,", "raw_affiliation_strings": ["Department of Psychiatry, University of Pittsburgh School of Medicine, Pittsburgh, Pennsylvania 15213, USA.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5023294157", "display_name": "Richard Anney", "orcid": "https://orcid.org/0000-0002-6083-407X"}, "institutions": [{"id": "https://openalex.org/I205274468", "display_name": "Trinity College Dublin", "ror": "https://ror.org/02tyrky19", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I205274468"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Richard Anney", "raw_affiliation_string": "Department of Psychiatry, Autism Genetics Group, School of Medicine, Trinity College, Dublin 8, Ireland.,", "raw_affiliation_strings": ["Department of Psychiatry, Autism Genetics Group, School of Medicine, Trinity College, Dublin 8, Ireland.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5072404345", "display_name": "Daniele Merico", "orcid": "https://orcid.org/0000-0002-3728-4401"}, "institutions": [{"id": "https://openalex.org/I185261750", "display_name": "University of Toronto", "ror": "https://ror.org/03dbr7087", "country_code": "CA", "type": "education", "lineage": ["https://openalex.org/I185261750"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "Daniele Merico", "raw_affiliation_string": "Banting and Best Department of Medical Research, Terrence Donnelly Centre for Cellular and Biomolecular Research, University of Toronto, Toronto, Ontario M5S 3E1, Canada.,", "raw_affiliation_strings": ["Banting and Best Department of Medical Research, Terrence Donnelly Centre for Cellular and Biomolecular Research, University of Toronto, Toronto, Ontario M5S 3E1, Canada.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5013483449", "display_name": "Regina Regan", "orcid": null}, "institutions": [{"id": "https://openalex.org/I100930933", "display_name": "University College Dublin", "ror": "https://ror.org/05m7pjf47", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I100930933", "https://openalex.org/I181231927"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Regina Regan", "raw_affiliation_string": "School of Medicine and Medical Science University College, Dublin 4, Ireland.,", "raw_affiliation_strings": ["School of Medicine and Medical Science University College, Dublin 4, Ireland.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5053316308", "display_name": "Judith Conroy", "orcid": null}, "institutions": [{"id": "https://openalex.org/I100930933", "display_name": "University College Dublin", "ror": "https://ror.org/05m7pjf47", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I100930933", "https://openalex.org/I181231927"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Judith Conroy", "raw_affiliation_string": "School of Medicine and Medical Science University College, Dublin 4, Ireland.,", "raw_affiliation_strings": ["School of Medicine and Medical Science University College, Dublin 4, Ireland.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5053160751", "display_name": "Tiago R. Magalhães", "orcid": null}, "institutions": [{"id": "https://openalex.org/I48057805", "display_name": "Instituto Gulbenkian de Ciência", "ror": "https://ror.org/04b08hq31", "country_code": "PT", "type": "education", "lineage": ["https://openalex.org/I48057805"]}, {"id": "https://openalex.org/I4210142220", "display_name": "National Institute of Health Dr. Ricardo Jorge", "ror": "https://ror.org/03mx8d427", "country_code": "PT", "type": "other", "lineage": ["https://openalex.org/I4210142220", "https://openalex.org/I4210157859"]}, {"id": "https://openalex.org/I141596103", "display_name": "University of Lisbon", "ror": "https://ror.org/01c27hj86", "country_code": "PT", "type": "education", "lineage": ["https://openalex.org/I141596103"]}], "countries": ["PT"], "is_corresponding": false, "raw_author_name": "Tiago R. Magalhaes", "raw_affiliation_string": "BioFIG—Center for Biodiversity, Functional and Integrative Genomics, Campus da FCUL, C2.2.12, Campo Grande, 1749-016 Lisboa, Portugal.,; Instituto Nacional de Saude Dr Ricardo Jorge 1649-016 Lisbon and Instituto Gulbenkian de Cîencia, 2780-156 Oeiras, Portugal.,", "raw_affiliation_strings": ["BioFIG—Center for Biodiversity, Functional and Integrative Genomics, Campus da FCUL, C2.2.12, Campo Grande, 1749-016 Lisboa, Portugal.,", "Instituto Nacional de Saude Dr Ricardo Jorge 1649-016 Lisbon and Instituto Gulbenkian de Cîencia, 2780-156 Oeiras, Portugal.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5049220846", "display_name": "Catarina Correia", "orcid": null}, "institutions": [{"id": "https://openalex.org/I48057805", "display_name": "Instituto Gulbenkian de Ciência", "ror": "https://ror.org/04b08hq31", "country_code": "PT", "type": "education", "lineage": ["https://openalex.org/I48057805"]}, {"id": "https://openalex.org/I4210142220", "display_name": "National Institute of Health Dr. Ricardo Jorge", "ror": "https://ror.org/03mx8d427", "country_code": "PT", "type": "other", "lineage": ["https://openalex.org/I4210142220", "https://openalex.org/I4210157859"]}, {"id": "https://openalex.org/I141596103", "display_name": "University of Lisbon", "ror": "https://ror.org/01c27hj86", "country_code": "PT", "type": "education", "lineage": ["https://openalex.org/I141596103"]}], "countries": ["PT"], "is_corresponding": false, "raw_author_name": "Catarina Correia", "raw_affiliation_string": "BioFIG—Center for Biodiversity, Functional and Integrative Genomics, Campus da FCUL, C2.2.12, Campo Grande, 1749-016 Lisboa, Portugal.,; Instituto Nacional de Saude Dr Ricardo Jorge 1649-016 Lisbon and Instituto Gulbenkian de Cîencia, 2780-156 Oeiras, Portugal.,", "raw_affiliation_strings": ["BioFIG—Center for Biodiversity, Functional and Integrative Genomics, Campus da FCUL, C2.2.12, Campo Grande, 1749-016 Lisboa, Portugal.,", "Instituto Nacional de Saude Dr Ricardo Jorge 1649-016 Lisbon and Instituto Gulbenkian de Cîencia, 2780-156 Oeiras, Portugal.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5089411792", "display_name": "Brett S. Abrahams", "orcid": null}, "institutions": [{"id": "https://openalex.org/I94690351", "display_name": "Center for Autism and Related Disorders", "ror": "https://ror.org/00t7r5h51", "country_code": "US", "type": "other", "lineage": ["https://openalex.org/I94690351"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Brett S. Abrahams", "raw_affiliation_string": "Department of Neurology and Center for Autism Research and Treatment, Program in Neurogenetics, Semel Institute, David Geffen School of Medicine at UCLA, Los Angeles, California 90095, USA.,", "raw_affiliation_strings": ["Department of Neurology and Center for Autism Research and Treatment, Program in Neurogenetics, Semel Institute, David Geffen School of Medicine at UCLA, Los Angeles, California 90095, USA.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5014448106", "display_name": "Joana Almeida", "orcid": "https://orcid.org/0009-0007-1358-8623"}, "institutions": [], "countries": ["PT"], "is_corresponding": false, "raw_author_name": "Joana Almeida", "raw_affiliation_string": "Hospital Pediátrico de Coimbra, 3000 – 076 Coimbra, Portugal.,", "raw_affiliation_strings": ["Hospital Pediátrico de Coimbra, 3000 – 076 Coimbra, Portugal.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5005071352", "display_name": "Elena Bacchelli", "orcid": "https://orcid.org/0000-0001-8800-9568"}, "institutions": [{"id": "https://openalex.org/I9360294", "display_name": "University of Bologna", "ror": "https://ror.org/01111rn36", "country_code": "IT", "type": "education", "lineage": ["https://openalex.org/I9360294"]}], "countries": ["IT"], "is_corresponding": false, "raw_author_name": "Elena Bacchelli", "raw_affiliation_string": "Department of Biology, University of Bologna, 40126 Bologna, Italy.,", "raw_affiliation_strings": ["Department of Biology, University of Bologna, 40126 Bologna, Italy.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5010789029", "display_name": "Gary D. Bader", "orcid": "https://orcid.org/0000-0003-0185-8861"}, "institutions": [{"id": "https://openalex.org/I185261750", "display_name": "University of Toronto", "ror": "https://ror.org/03dbr7087", "country_code": "CA", "type": "education", "lineage": ["https://openalex.org/I185261750"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "Gary D. Bader", "raw_affiliation_string": "Banting and Best Department of Medical Research, Terrence Donnelly Centre for Cellular and Biomolecular Research, University of Toronto, Toronto, Ontario M5S 3E1, Canada.,", "raw_affiliation_strings": ["Banting and Best Department of Medical Research, Terrence Donnelly Centre for Cellular and Biomolecular Research, University of Toronto, Toronto, Ontario M5S 3E1, Canada.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5034194945", "display_name": "Anthony Bailey", "orcid": null}, "institutions": [{"id": "https://openalex.org/I2803066836", "display_name": "Warneford Hospital", "ror": "https://ror.org/03we1zb10", "country_code": "GB", "type": "healthcare", "lineage": ["https://openalex.org/I2802171185", "https://openalex.org/I2803066836"]}, {"id": "https://openalex.org/I40120149", "display_name": "University of Oxford", "ror": "https://ror.org/052gg0110", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I40120149"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Anthony J. Bailey", "raw_affiliation_string": "Department of Psychiatry, University of Oxford, Warneford Hospital, Headington, Oxford OX3 7JX, UK.,", "raw_affiliation_strings": ["Department of Psychiatry, University of Oxford, Warneford Hospital, Headington, Oxford OX3 7JX, UK.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5067210442", "display_name": "Gillian Baird", "orcid": "https://orcid.org/0000-0002-7601-7074"}, "institutions": [{"id": "https://openalex.org/I1298207432", "display_name": "Guy's Hospital", "ror": "https://ror.org/04r33pf22", "country_code": "GB", "type": "healthcare", "lineage": ["https://openalex.org/I1298207432", "https://openalex.org/I200166805", "https://openalex.org/I4210111135"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Gillian Baird", "raw_affiliation_string": "Newcomen Centre, Guy’s Hospital, London SE1 9RT, UK.,", "raw_affiliation_strings": ["Newcomen Centre, Guy’s Hospital, London SE1 9RT, UK.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5044044440", "display_name": "Agatino Battaglia", "orcid": "https://orcid.org/0000-0002-7128-7606"}, "institutions": [{"id": "https://openalex.org/I4210126460", "display_name": "Fondazione Stella Maris", "ror": "https://ror.org/02w8ez808", "country_code": "IT", "type": "healthcare", "lineage": ["https://openalex.org/I4210126460", "https://openalex.org/I4210153126"]}], "countries": ["IT"], "is_corresponding": false, "raw_author_name": "Agatino Battaglia", "raw_affiliation_string": "Stella Maris Institute for Child and Adolescent Neuropsychiatry, 56128 Calambrone (Pisa), Italy.,", "raw_affiliation_strings": ["Stella Maris Institute for Child and Adolescent Neuropsychiatry, 56128 Calambrone (Pisa), Italy.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5008688186", "display_name": "T. P. Berney", "orcid": null}, "institutions": [{"id": "https://openalex.org/I84884186", "display_name": "Newcastle University", "ror": "https://ror.org/01kj2bm70", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I84884186"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Tom Berney", "raw_affiliation_string": "Child and Adolescent Mental Health, University of Newcastle, Sir James Spence Institute, Newcastle upon Tyne NE1 4LP, UK.,", "raw_affiliation_strings": ["Child and Adolescent Mental Health, University of Newcastle, Sir James Spence Institute, Newcastle upon Tyne NE1 4LP, UK.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5016075120", "display_name": "Nadia Bolshakova", "orcid": "https://orcid.org/0000-0003-2173-6562"}, "institutions": [{"id": "https://openalex.org/I205274468", "display_name": "Trinity College Dublin", "ror": "https://ror.org/02tyrky19", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I205274468"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Nadia Bolshakova", "raw_affiliation_string": "Department of Psychiatry, Autism Genetics Group, School of Medicine, Trinity College, Dublin 8, Ireland.,", "raw_affiliation_strings": ["Department of Psychiatry, Autism Genetics Group, School of Medicine, Trinity College, Dublin 8, Ireland.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5054839489", "display_name": "Sven Bölte", "orcid": "https://orcid.org/0000-0002-4579-4970"}, "institutions": [{"id": "https://openalex.org/I114090438", "display_name": "Goethe University Frankfurt", "ror": "https://ror.org/04cvxnb49", "country_code": "DE", "type": "education", "lineage": ["https://openalex.org/I114090438"]}], "countries": ["DE"], "is_corresponding": false, "raw_author_name": "Sven Bölte", "raw_affiliation_string": "Department of Child and Adolescent Psychiatry, Psychosomatics and Psychotherapy, J.W. Goethe University Frankfurt, 60528 Frankfurt, Germany.,", "raw_affiliation_strings": ["Department of Child and Adolescent Psychiatry, Psychosomatics and Psychotherapy, J.W. Goethe University Frankfurt, 60528 Frankfurt, Germany.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5030755834", "display_name": "Patrick Bolton", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210114595", "display_name": "Psychiatry Research Trust", "ror": "https://ror.org/01ywejq18", "country_code": "GB", "type": "nonprofit", "lineage": ["https://openalex.org/I4210114595"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Patrick F. Bolton", "raw_affiliation_string": "Department of Child and Adolescent Psychiatry, Institute of Psychiatry, London SE5 8AF, UK.,", "raw_affiliation_strings": ["Department of Child and Adolescent Psychiatry, Institute of Psychiatry, London SE5 8AF, UK.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5084411567", "display_name": "Thomas Bourgeron", "orcid": "https://orcid.org/0000-0001-8164-9220"}, "institutions": [{"id": "https://openalex.org/I157536573", "display_name": "Institut Pasteur", "ror": "https://ror.org/0495fxg12", "country_code": "FR", "type": "nonprofit", "lineage": ["https://openalex.org/I157536573"]}, {"id": "https://openalex.org/I4210096842", "display_name": "Fondation FondaMental", "ror": "https://ror.org/00rrhf939", "country_code": "FR", "type": "other", "lineage": ["https://openalex.org/I4210096842"]}, {"id": "https://openalex.org/I204730241", "display_name": "Université Paris Cité", "ror": "https://ror.org/05f82e368", "country_code": "FR", "type": "education", "lineage": ["https://openalex.org/I204730241"]}, {"id": "https://openalex.org/I1294671590", "display_name": "French National Centre for Scientific Research", "ror": "https://ror.org/02feahw73", "country_code": "FR", "type": "government", "lineage": ["https://openalex.org/I1294671590"]}], "countries": ["FR"], "is_corresponding": false, "raw_author_name": "Thomas Bourgeron", "raw_affiliation_string": "Human Genetics and Cognitive Functions, Institut Pasteur,; University Paris Diderot-Paris 7, CNRS URA 2182, Fondation FondaMental, 75015 Paris, France.,", "raw_affiliation_strings": ["Human Genetics and Cognitive Functions, Institut Pasteur,", "University Paris Diderot-Paris 7, CNRS URA 2182, Fondation FondaMental, 75015 Paris, France.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5055823021", "display_name": "S. Brennan", "orcid": null}, "institutions": [{"id": "https://openalex.org/I205274468", "display_name": "Trinity College Dublin", "ror": "https://ror.org/02tyrky19", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I205274468"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Sean Brennan", "raw_affiliation_string": "Department of Psychiatry, Autism Genetics Group, School of Medicine, Trinity College, Dublin 8, Ireland.,", "raw_affiliation_strings": ["Department of Psychiatry, Autism Genetics Group, School of Medicine, Trinity College, Dublin 8, Ireland.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5029482914", "display_name": "Jessica Brian", "orcid": "https://orcid.org/0000-0001-8181-4353"}, "institutions": [{"id": "https://openalex.org/I2801420703", "display_name": "Holland Bloorview Kids Rehabilitation Hospital", "ror": "https://ror.org/03qea8398", "country_code": "CA", "type": "healthcare", "lineage": ["https://openalex.org/I2801420703"]}, {"id": "https://openalex.org/I185261750", "display_name": "University of Toronto", "ror": "https://ror.org/03dbr7087", "country_code": "CA", "type": "education", "lineage": ["https://openalex.org/I185261750"]}, {"id": "https://openalex.org/I2801317318", "display_name": "Hospital for Sick Children", "ror": "https://ror.org/057q4rt57", "country_code": "CA", "type": "healthcare", "lineage": ["https://openalex.org/I2801317318"]}, {"id": "https://openalex.org/I4210141030", "display_name": "SickKids Foundation", "ror": "https://ror.org/04374qe70", "country_code": "CA", "type": "nonprofit", "lineage": ["https://openalex.org/I4210141030"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "Jessica Brian", "raw_affiliation_string": "Autism Research Unit, The Hospital for Sick Children and Bloorview Kids Rehab, University of Toronto, Toronto, Ontario M5G 1X8, Canada.,", "raw_affiliation_strings": ["Autism Research Unit, The Hospital for Sick Children and Bloorview Kids Rehab, University of Toronto, Toronto, Ontario M5G 1X8, Canada.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5045154579", "display_name": "Susan E. Bryson", "orcid": null}, "institutions": [{"id": "https://openalex.org/I129902397", "display_name": "Dalhousie University", "ror": "https://ror.org/01e6qks80", "country_code": "CA", "type": "education", "lineage": ["https://openalex.org/I129902397"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "Susan E. Bryson", "raw_affiliation_string": "Department of Pediatrics and Psychology, Dalhousie University, Halifax, Nova Scotia B3K 6R8, Canada.,", "raw_affiliation_strings": ["Department of Pediatrics and Psychology, Dalhousie University, Halifax, Nova Scotia B3K 6R8, Canada.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5088381419", "display_name": "Andrew R. Carson", "orcid": null}, "institutions": [{"id": "https://openalex.org/I2801317318", "display_name": "Hospital for Sick Children", "ror": "https://ror.org/057q4rt57", "country_code": "CA", "type": "healthcare", "lineage": ["https://openalex.org/I2801317318"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "Andrew R. Carson", "raw_affiliation_string": "The Centre for Applied Genomics and Program in Genetics and Genomic Biology, The Hospital for Sick Children, Toronto, Ontario M5G 1L7, Canada.,", "raw_affiliation_strings": ["The Centre for Applied Genomics and Program in Genetics and Genomic Biology, The Hospital for Sick Children, Toronto, Ontario M5G 1L7, Canada.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5017587023", "display_name": "Guillermo Casallo", "orcid": null}, "institutions": [{"id": "https://openalex.org/I2801317318", "display_name": "Hospital for Sick Children", "ror": "https://ror.org/057q4rt57", "country_code": "CA", "type": "healthcare", "lineage": ["https://openalex.org/I2801317318"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "Guillermo Casallo", "raw_affiliation_string": "The Centre for Applied Genomics and Program in Genetics and Genomic Biology, The Hospital for Sick Children, Toronto, Ontario M5G 1L7, Canada.,", "raw_affiliation_strings": ["The Centre for Applied Genomics and Program in Genetics and Genomic Biology, The Hospital for Sick Children, Toronto, Ontario M5G 1L7, Canada.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5027846747", "display_name": "Jillian P. Casey", "orcid": null}, "institutions": [{"id": "https://openalex.org/I100930933", "display_name": "University College Dublin", "ror": "https://ror.org/05m7pjf47", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I100930933", "https://openalex.org/I181231927"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Jillian Casey", "raw_affiliation_string": "School of Medicine and Medical Science University College, Dublin 4, Ireland.,", "raw_affiliation_strings": ["School of Medicine and Medical Science University College, Dublin 4, Ireland.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5027881770", "display_name": "Brian Hon‐Yin Chung", "orcid": null}, "institutions": [{"id": "https://openalex.org/I2801317318", "display_name": "Hospital for Sick Children", "ror": "https://ror.org/057q4rt57", "country_code": "CA", "type": "healthcare", "lineage": ["https://openalex.org/I2801317318"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "Brian H.Y. Chung", "raw_affiliation_string": "The Centre for Applied Genomics and Program in Genetics and Genomic Biology, The Hospital for Sick Children, Toronto, Ontario M5G 1L7, Canada.,", "raw_affiliation_strings": ["The Centre for Applied Genomics and Program in Genetics and Genomic Biology, The Hospital for Sick Children, Toronto, Ontario M5G 1L7, Canada.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5081922755", "display_name": "Lynne Cochrane", "orcid": null}, "institutions": [{"id": "https://openalex.org/I205274468", "display_name": "Trinity College Dublin", "ror": "https://ror.org/02tyrky19", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I205274468"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Lynne Cochrane", "raw_affiliation_string": "Department of Psychiatry, Autism Genetics Group, School of Medicine, Trinity College, Dublin 8, Ireland.,", "raw_affiliation_strings": ["Department of Psychiatry, Autism Genetics Group, School of Medicine, Trinity College, Dublin 8, Ireland.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5027303675", "display_name": "Christina Corsello", "orcid": null}, "institutions": [{"id": "https://openalex.org/I27837315", "display_name": "University of Michigan–Ann Arbor", "ror": "https://ror.org/00jmfr291", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I27837315"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Christina Corsello", "raw_affiliation_string": "Autism and Communicative Disorders Centre, University of Michigan, Ann Arbor, Michigan 48109-2054, USA.,", "raw_affiliation_strings": ["Autism and Communicative Disorders Centre, University of Michigan, Ann Arbor, Michigan 48109-2054, USA.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5013222660", "display_name": "Emily L. Crawford", "orcid": null}, "institutions": [{"id": "https://openalex.org/I200719446", "display_name": "Vanderbilt University", "ror": "https://ror.org/02vm5rt34", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I200719446"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Emily L. Crawford", "raw_affiliation_string": "Department of Molecular Physiology and Biophysics, Vanderbilt Kennedy Center, and Centers for Human Genetics Research and Molecular Neuroscience, Vanderbilt University, Nashville, Tennessee 37232, USA.,", "raw_affiliation_strings": ["Department of Molecular Physiology and Biophysics, Vanderbilt Kennedy Center, and Centers for Human Genetics Research and Molecular Neuroscience, Vanderbilt University, Nashville, Tennessee 37232, USA.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5071221627", "display_name": "Andrew Crossett", "orcid": null}, "institutions": [{"id": "https://openalex.org/I74973139", "display_name": "Carnegie Mellon University", "ror": "https://ror.org/05x2bcf33", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I74973139"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Andrew Crossett", "raw_affiliation_string": "Department of Statistics, Carnegie Mellon University, Pittsburgh, Pennsylvania 15213, USA.,", "raw_affiliation_strings": ["Department of Statistics, Carnegie Mellon University, Pittsburgh, Pennsylvania 15213, USA.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5017430701", "display_name": "Cheryl Cytrynbaum", "orcid": "https://orcid.org/0000-0002-3742-1250"}, "institutions": [{"id": "https://openalex.org/I2801317318", "display_name": "Hospital for Sick Children", "ror": "https://ror.org/057q4rt57", "country_code": "CA", "type": "healthcare", "lineage": ["https://openalex.org/I2801317318"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "Cheryl Cytrynbaum", "raw_affiliation_string": "The Centre for Applied Genomics and Program in Genetics and Genomic Biology, The Hospital for Sick Children, Toronto, Ontario M5G 1L7, Canada.,", "raw_affiliation_strings": ["The Centre for Applied Genomics and Program in Genetics and Genomic Biology, The Hospital for Sick Children, Toronto, Ontario M5G 1L7, Canada.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5073522388", "display_name": "Geraldine Dawson", "orcid": "https://orcid.org/0000-0003-1410-2764"}, "institutions": [{"id": "https://openalex.org/I109266671", "display_name": "Autism Speaks", "ror": "https://ror.org/04bkad313", "country_code": "US", "type": "nonprofit", "lineage": ["https://openalex.org/I109266671"]}, {"id": "https://openalex.org/I114027177", "display_name": "University of North Carolina at Chapel Hill", "ror": "https://ror.org/0130frc33", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I114027177", "https://openalex.org/I4210158053"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Geraldine Dawson", "raw_affiliation_string": "Autism Speaks, New York 10016, USA.,; Department of Psychiatry, University of North Carolina, Chapel Hill, North Carolina 27599-3366, USA.,", "raw_affiliation_strings": ["Autism Speaks, New York 10016, USA.,", "Department of Psychiatry, University of North Carolina, Chapel Hill, North Carolina 27599-3366, USA.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5039670828", "display_name": "Maretha de Jonge", "orcid": null}, "institutions": [{"id": "https://openalex.org/I3018483916", "display_name": "University Medical Center Utrecht", "ror": "https://ror.org/0575yy874", "country_code": "NL", "type": "healthcare", "lineage": ["https://openalex.org/I3018483916"]}], "countries": ["NL"], "is_corresponding": false, "raw_author_name": "Maretha de Jonge", "raw_affiliation_string": "Department of Child Psychiatry, University Medical Center, Utrecht 3508 GA, The Netherlands.,", "raw_affiliation_strings": ["Department of Child Psychiatry, University Medical Center, Utrecht 3508 GA, The Netherlands.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5020448299", "display_name": "Richard Delorme", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210096842", "display_name": "Fondation FondaMental", "ror": "https://ror.org/00rrhf939", "country_code": "FR", "type": "other", "lineage": ["https://openalex.org/I4210096842"]}, {"id": "https://openalex.org/I154526488", "display_name": "Inserm", "ror": "https://ror.org/02vjkv261", "country_code": "FR", "type": "government", "lineage": ["https://openalex.org/I154526488"]}, {"id": "https://openalex.org/I4210126712", "display_name": "Hôpital Robert-Debré", "ror": "https://ror.org/02dcqy320", "country_code": "FR", "type": "healthcare", "lineage": ["https://openalex.org/I4210097159", "https://openalex.org/I4210126712"]}], "countries": ["FR"], "is_corresponding": false, "raw_author_name": "Richard Delorme", "raw_affiliation_string": "INSERM U 955, Fondation FondaMental, APHP, Hôpital Robert Debré, Child and Adolescent Psychiatry, 75019 Paris, France.,", "raw_affiliation_strings": ["INSERM U 955, Fondation FondaMental, APHP, Hôpital Robert Debré, Child and Adolescent Psychiatry, 75019 Paris, France.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5081523158", "display_name": "Irene Drmic", "orcid": "https://orcid.org/0000-0002-9212-4929"}, "institutions": [{"id": "https://openalex.org/I2801420703", "display_name": "Holland Bloorview Kids Rehabilitation Hospital", "ror": "https://ror.org/03qea8398", "country_code": "CA", "type": "healthcare", "lineage": ["https://openalex.org/I2801420703"]}, {"id": "https://openalex.org/I185261750", "display_name": "University of Toronto", "ror": "https://ror.org/03dbr7087", "country_code": "CA", "type": "education", "lineage": ["https://openalex.org/I185261750"]}, {"id": "https://openalex.org/I2801317318", "display_name": "Hospital for Sick Children", "ror": "https://ror.org/057q4rt57", "country_code": "CA", "type": "healthcare", "lineage": ["https://openalex.org/I2801317318"]}, {"id": "https://openalex.org/I4210141030", "display_name": "SickKids Foundation", "ror": "https://ror.org/04374qe70", "country_code": "CA", "type": "nonprofit", "lineage": ["https://openalex.org/I4210141030"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "Irene Drmic", "raw_affiliation_string": "Autism Research Unit, The Hospital for Sick Children and Bloorview Kids Rehab, University of Toronto, Toronto, Ontario M5G 1X8, Canada.,", "raw_affiliation_strings": ["Autism Research Unit, The Hospital for Sick Children and Bloorview Kids Rehab, University of Toronto, Toronto, Ontario M5G 1X8, Canada.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5065665765", "display_name": "Eftichia Duketis", "orcid": null}, "institutions": [{"id": "https://openalex.org/I114090438", "display_name": "Goethe University Frankfurt", "ror": "https://ror.org/04cvxnb49", "country_code": "DE", "type": "education", "lineage": ["https://openalex.org/I114090438"]}], "countries": ["DE"], "is_corresponding": false, "raw_author_name": "Eftichia Duketis", "raw_affiliation_string": "Department of Child and Adolescent Psychiatry, Psychosomatics and Psychotherapy, J.W. Goethe University Frankfurt, 60528 Frankfurt, Germany.,", "raw_affiliation_strings": ["Department of Child and Adolescent Psychiatry, Psychosomatics and Psychotherapy, J.W. Goethe University Frankfurt, 60528 Frankfurt, Germany.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5048358192", "display_name": "Frederico Duque", "orcid": null}, "institutions": [], "countries": ["PT"], "is_corresponding": false, "raw_author_name": "Frederico Duque", "raw_affiliation_string": "Hospital Pediátrico de Coimbra, 3000 – 076 Coimbra, Portugal.,", "raw_affiliation_strings": ["Hospital Pediátrico de Coimbra, 3000 – 076 Coimbra, Portugal.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5067974738", "display_name": "Annette Estes", "orcid": "https://orcid.org/0000-0003-2687-4155"}, "institutions": [{"id": "https://openalex.org/I201448701", "display_name": "University of Washington", "ror": "https://ror.org/00cvxb145", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I201448701"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Annette Estes", "raw_affiliation_string": "Department of Speech and Hearing Sciences, University of Washington, Seattle, Washington 98195, USA.,", "raw_affiliation_strings": ["Department of Speech and Hearing Sciences, University of Washington, Seattle, Washington 98195, USA.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5034071437", "display_name": "Penny Farrar", "orcid": null}, "institutions": [{"id": "https://openalex.org/I1336263701", "display_name": "Wellcome Centre for Human Genetics", "ror": "https://ror.org/01rjnta51", "country_code": "GB", "type": "facility", "lineage": ["https://openalex.org/I1336263701", "https://openalex.org/I40120149", "https://openalex.org/I87048295"]}, {"id": "https://openalex.org/I40120149", "display_name": "University of Oxford", "ror": "https://ror.org/052gg0110", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I40120149"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Penny Farrar", "raw_affiliation_string": "Wellcome Trust Centre for Human Genetics, University of Oxford, Oxford OX3 7BN, UK.,", "raw_affiliation_strings": ["Wellcome Trust Centre for Human Genetics, University of Oxford, Oxford OX3 7BN, UK.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5003632230", "display_name": "Bridget A. Fernandez", "orcid": "https://orcid.org/0000-0003-1265-8259"}, "institutions": [{"id": "https://openalex.org/I130438778", "display_name": "Memorial University of Newfoundland", "ror": "https://ror.org/04haebc03", "country_code": "CA", "type": "education", "lineage": ["https://openalex.org/I130438778"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "Bridget A. Fernandez", "raw_affiliation_string": "Disciplines of Genetics and Medicine, Memorial University of Newfoundland, St John’s Newfoundland A1B 3V6, Canada.,", "raw_affiliation_strings": ["Disciplines of Genetics and Medicine, Memorial University of Newfoundland, St John’s Newfoundland A1B 3V6, Canada.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5065212941", "display_name": "Susan E. Folstein", "orcid": null}, "institutions": [{"id": "https://openalex.org/I145608581", "display_name": "University of Miami", "ror": "https://ror.org/02dgjyy92", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I145608581"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Susan E. Folstein", "raw_affiliation_string": "The John P. Hussman Institute for Human Genomics, University of Miami, Miami, Florida 33101, USA.,", "raw_affiliation_strings": ["The John P. Hussman Institute for Human Genomics, University of Miami, Miami, Florida 33101, USA.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5009614153", "display_name": "Éric Fombonne", "orcid": "https://orcid.org/0000-0002-8605-3538"}, "institutions": [{"id": "https://openalex.org/I5023651", "display_name": "McGill University", "ror": "https://ror.org/01pxwe438", "country_code": "CA", "type": "education", "lineage": ["https://openalex.org/I5023651"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "Eric Fombonne", "raw_affiliation_string": "Division of Psychiatry, McGill University, Montreal, Quebec H3A 1A1, Canada.,", "raw_affiliation_strings": ["Division of Psychiatry, McGill University, Montreal, Quebec H3A 1A1, Canada.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5050223372", "display_name": "Christine M. Freitag", "orcid": "https://orcid.org/0000-0001-9676-4782"}, "institutions": [{"id": "https://openalex.org/I114090438", "display_name": "Goethe University Frankfurt", "ror": "https://ror.org/04cvxnb49", "country_code": "DE", "type": "education", "lineage": ["https://openalex.org/I114090438"]}], "countries": ["DE"], "is_corresponding": false, "raw_author_name": "Christine M. Freitag", "raw_affiliation_string": "Department of Child and Adolescent Psychiatry, Psychosomatics and Psychotherapy, J.W. Goethe University Frankfurt, 60528 Frankfurt, Germany.,", "raw_affiliation_strings": ["Department of Child and Adolescent Psychiatry, Psychosomatics and Psychotherapy, J.W. Goethe University Frankfurt, 60528 Frankfurt, Germany.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5076736767", "display_name": "John R. Gilbert", "orcid": null}, "institutions": [{"id": "https://openalex.org/I145608581", "display_name": "University of Miami", "ror": "https://ror.org/02dgjyy92", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I145608581"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "John Gilbert", "raw_affiliation_string": "The John P. Hussman Institute for Human Genomics, University of Miami, Miami, Florida 33101, USA.,", "raw_affiliation_strings": ["The John P. Hussman Institute for Human Genomics, University of Miami, Miami, Florida 33101, USA.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5028456703", "display_name": "Christopher Gillberg", "orcid": "https://orcid.org/0000-0001-8848-1934"}, "institutions": [{"id": "https://openalex.org/I881427289", "display_name": "University of Gothenburg", "ror": "https://ror.org/01tm6cn81", "country_code": "SE", "type": "education", "lineage": ["https://openalex.org/I881427289"]}], "countries": ["SE"], "is_corresponding": false, "raw_author_name": "Christopher Gillberg", "raw_affiliation_string": "Department of Child and Adolescent Psychiatry, Göteborg University, Göteborg S41345, Sweden.,", "raw_affiliation_strings": ["Department of Child and Adolescent Psychiatry, Göteborg University, Göteborg S41345, Sweden.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5066789855", "display_name": "Joseph T. Glessner", "orcid": "https://orcid.org/0000-0001-5131-2811"}, "institutions": [{"id": "https://openalex.org/I1335321130", "display_name": "Children's Hospital of Philadelphia", "ror": "https://ror.org/01z7r7q48", "country_code": "US", "type": "healthcare", "lineage": ["https://openalex.org/I1335321130"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Joseph T. Glessner", "raw_affiliation_string": "Division of Human Genetics, The Center for Applied Genomics, The Children’s Hospital of Philadelphia, Philadelphia, Pennsylvania 19104, USA.,", "raw_affiliation_strings": ["Division of Human Genetics, The Center for Applied Genomics, The Children’s Hospital of Philadelphia, Philadelphia, Pennsylvania 19104, USA.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5081908377", "display_name": "Jeremy Goldberg", "orcid": null}, "institutions": [{"id": "https://openalex.org/I98251732", "display_name": "McMaster University", "ror": "https://ror.org/02fa3aq29", "country_code": "CA", "type": "education", "lineage": ["https://openalex.org/I98251732"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "Jeremy Goldberg", "raw_affiliation_string": "Department of Psychiatry and Behavioural Neurosciences, McMaster University, Hamilton, Ontario L8N 3Z5, Canada.,", "raw_affiliation_strings": ["Department of Psychiatry and Behavioural Neurosciences, McMaster University, Hamilton, Ontario L8N 3Z5, Canada.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5000005180", "display_name": "Andrew Green", "orcid": "https://orcid.org/0000-0002-0488-5913"}, "institutions": [{"id": "https://openalex.org/I100930933", "display_name": "University College Dublin", "ror": "https://ror.org/05m7pjf47", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I100930933", "https://openalex.org/I181231927"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Andrew Green", "raw_affiliation_string": "School of Medicine and Medical Science University College, Dublin 4, Ireland.,", "raw_affiliation_strings": ["School of Medicine and Medical Science University College, Dublin 4, Ireland.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5076431905", "display_name": "Jonathan Green", "orcid": "https://orcid.org/0000-0002-0143-181X"}, "institutions": [], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Jonathan Green", "raw_affiliation_string": "Academic Department of Child Psychiatry, Booth Hall of Children’s Hospital, Blackley, Manchester M9 7AA, UK.,", "raw_affiliation_strings": ["Academic Department of Child Psychiatry, Booth Hall of Children’s Hospital, Blackley, Manchester M9 7AA, UK.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5067479223", "display_name": "Stephen J. Guter", "orcid": null}, "institutions": [{"id": "https://openalex.org/I39422238", "display_name": "University of Illinois at Chicago", "ror": "https://ror.org/02mpq6x41", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I2801919071", "https://openalex.org/I39422238"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Stephen J. Guter", "raw_affiliation_string": "Department of Psychiatry, Institute for Juvenile Research, University of Illinois at Chicago, Chicago, Illinois 60612, USA.,", "raw_affiliation_strings": ["Department of Psychiatry, Institute for Juvenile Research, University of Illinois at Chicago, Chicago, Illinois 60612, USA.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5088244425", "display_name": "Hákon Hákonarson", "orcid": "https://orcid.org/0000-0003-2814-7461"}, "institutions": [{"id": "https://openalex.org/I1335321130", "display_name": "Children's Hospital of Philadelphia", "ror": "https://ror.org/01z7r7q48", "country_code": "US", "type": "healthcare", "lineage": ["https://openalex.org/I1335321130"]}, {"id": "https://openalex.org/I79576946", "display_name": "University of Pennsylvania", "ror": "https://ror.org/00b30xv10", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I79576946"]}, {"id": "https://openalex.org/I922845939", "display_name": "Philadelphia University", "ror": "https://ror.org/03zzmyz63", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I922845939"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Hakon Hakonarson", "raw_affiliation_string": "Department of Pediatrics, Children’s Hospital of Philadelphia, University of Pennsylvania School of Medicine, Philadelphia, Pennsylvania 19104, USA.,; Division of Human Genetics, The Center for Applied Genomics, The Children’s Hospital of Philadelphia, Philadelphia, Pennsylvania 19104, USA.,", "raw_affiliation_strings": ["Department of Pediatrics, Children’s Hospital of Philadelphia, University of Pennsylvania School of Medicine, Philadelphia, Pennsylvania 19104, USA.,", "Division of Human Genetics, The Center for Applied Genomics, The Children’s Hospital of Philadelphia, Philadelphia, Pennsylvania 19104, USA.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5027468371", "display_name": "Elizabeth A. Heron", "orcid": "https://orcid.org/0000-0003-2219-4005"}, "institutions": [{"id": "https://openalex.org/I205274468", "display_name": "Trinity College Dublin", "ror": "https://ror.org/02tyrky19", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I205274468"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Elizabeth A. Heron", "raw_affiliation_string": "Department of Psychiatry, Autism Genetics Group, School of Medicine, Trinity College, Dublin 8, Ireland.,", "raw_affiliation_strings": ["Department of Psychiatry, Autism Genetics Group, School of Medicine, Trinity College, Dublin 8, Ireland.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5005795830", "display_name": "Matthew Hill", "orcid": null}, "institutions": [{"id": "https://openalex.org/I205274468", "display_name": "Trinity College Dublin", "ror": "https://ror.org/02tyrky19", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I205274468"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Matthew Hill", "raw_affiliation_string": "Department of Psychiatry, Autism Genetics Group, School of Medicine, Trinity College, Dublin 8, Ireland.,", "raw_affiliation_strings": ["Department of Psychiatry, Autism Genetics Group, School of Medicine, Trinity College, Dublin 8, Ireland.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5033914181", "display_name": "Richard Holt", "orcid": null}, "institutions": [{"id": "https://openalex.org/I1336263701", "display_name": "Wellcome Centre for Human Genetics", "ror": "https://ror.org/01rjnta51", "country_code": "GB", "type": "facility", "lineage": ["https://openalex.org/I1336263701", "https://openalex.org/I40120149", "https://openalex.org/I87048295"]}, {"id": "https://openalex.org/I40120149", "display_name": "University of Oxford", "ror": "https://ror.org/052gg0110", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I40120149"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Richard Holt", "raw_affiliation_string": "Wellcome Trust Centre for Human Genetics, University of Oxford, Oxford OX3 7BN, UK.,", "raw_affiliation_strings": ["Wellcome Trust Centre for Human Genetics, University of Oxford, Oxford OX3 7BN, UK.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5007448951", "display_name": "Jennifer Howe", "orcid": null}, "institutions": [{"id": "https://openalex.org/I2801317318", "display_name": "Hospital for Sick Children", "ror": "https://ror.org/057q4rt57", "country_code": "CA", "type": "healthcare", "lineage": ["https://openalex.org/I2801317318"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "Jennifer L. Howe", "raw_affiliation_string": "The Centre for Applied Genomics and Program in Genetics and Genomic Biology, The Hospital for Sick Children, Toronto, Ontario M5G 1L7, Canada.,", "raw_affiliation_strings": ["The Centre for Applied Genomics and Program in Genetics and Genomic Biology, The Hospital for Sick Children, Toronto, Ontario M5G 1L7, Canada.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5082898729", "display_name": "Gillian Hughes", "orcid": null}, "institutions": [{"id": "https://openalex.org/I205274468", "display_name": "Trinity College Dublin", "ror": "https://ror.org/02tyrky19", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I205274468"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Gillian Hughes", "raw_affiliation_string": "Department of Psychiatry, Autism Genetics Group, School of Medicine, Trinity College, Dublin 8, Ireland.,", "raw_affiliation_strings": ["Department of Psychiatry, Autism Genetics Group, School of Medicine, Trinity College, Dublin 8, Ireland.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5001696887", "display_name": "Vanessa Hus", "orcid": null}, "institutions": [{"id": "https://openalex.org/I27837315", "display_name": "University of Michigan–Ann Arbor", "ror": "https://ror.org/00jmfr291", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I27837315"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Vanessa Hus", "raw_affiliation_string": "Autism and Communicative Disorders Centre, University of Michigan, Ann Arbor, Michigan 48109-2054, USA.,", "raw_affiliation_strings": ["Autism and Communicative Disorders Centre, University of Michigan, Ann Arbor, Michigan 48109-2054, USA.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5046594518", "display_name": "Roberta Igliozzi", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210126460", "display_name": "Fondazione Stella Maris", "ror": "https://ror.org/02w8ez808", "country_code": "IT", "type": "healthcare", "lineage": ["https://openalex.org/I4210126460", "https://openalex.org/I4210153126"]}], "countries": ["IT"], "is_corresponding": false, "raw_author_name": "Roberta Igliozzi", "raw_affiliation_string": "Stella Maris Institute for Child and Adolescent Neuropsychiatry, 56128 Calambrone (Pisa), Italy.,", "raw_affiliation_strings": ["Stella Maris Institute for Child and Adolescent Neuropsychiatry, 56128 Calambrone (Pisa), Italy.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5000624898", "display_name": "Cecilia Kim", "orcid": null}, "institutions": [{"id": "https://openalex.org/I1335321130", "display_name": "Children's Hospital of Philadelphia", "ror": "https://ror.org/01z7r7q48", "country_code": "US", "type": "healthcare", "lineage": ["https://openalex.org/I1335321130"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Cecilia Kim", "raw_affiliation_string": "Division of Human Genetics, The Center for Applied Genomics, The Children’s Hospital of Philadelphia, Philadelphia, Pennsylvania 19104, USA.,", "raw_affiliation_strings": ["Division of Human Genetics, The Center for Applied Genomics, The Children’s Hospital of Philadelphia, Philadelphia, Pennsylvania 19104, USA.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5081738047", "display_name": "Sabine M. Klauck", "orcid": null}, "institutions": [{"id": "https://openalex.org/I17937529", "display_name": "German Cancer Research Center", "ror": "https://ror.org/04cdgtt98", "country_code": "DE", "type": "facility", "lineage": ["https://openalex.org/I1305996414", "https://openalex.org/I17937529"]}, {"id": "https://openalex.org/I223822909", "display_name": "Heidelberg University", "ror": "https://ror.org/038t36y30", "country_code": "DE", "type": "education", "lineage": ["https://openalex.org/I223822909"]}], "countries": ["DE"], "is_corresponding": false, "raw_author_name": "Sabine M. Klauck", "raw_affiliation_string": "Division of Molecular Genome Analysis, German Cancer Research Center (DKFZ), Heidelberg 69120, Germany.,", "raw_affiliation_strings": ["Division of Molecular Genome Analysis, German Cancer Research Center (DKFZ), Heidelberg 69120, Germany.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5016983099", "display_name": "Alexander Kolevzon", "orcid": "https://orcid.org/0000-0001-8129-2671"}, "institutions": [{"id": "https://openalex.org/I98704320", "display_name": "Icahn School of Medicine at Mount Sinai", "ror": "https://ror.org/04a9tmd77", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I1320796813", "https://openalex.org/I98704320"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Alexander Kolevzon", "raw_affiliation_string": "Department of Psychiatry, The Seaver Autism Center for Research and Treatment, Mount Sinai School of Medicine, New York 10029, USA.,", "raw_affiliation_strings": ["Department of Psychiatry, The Seaver Autism Center for Research and Treatment, Mount Sinai School of Medicine, New York 10029, USA.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5020224643", "display_name": "Olena Korvatska", "orcid": null}, "institutions": [{"id": "https://openalex.org/I201448701", "display_name": "University of Washington", "ror": "https://ror.org/00cvxb145", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I201448701"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Olena Korvatska", "raw_affiliation_string": "Department of Medicine, University of Washington, Seattle, Washington 98195, USA.,", "raw_affiliation_strings": ["Department of Medicine, University of Washington, Seattle, Washington 98195, USA.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5007111212", "display_name": "Vlad Kustanovich", "orcid": null}, "institutions": [{"id": "https://openalex.org/I109266671", "display_name": "Autism Speaks", "ror": "https://ror.org/04bkad313", "country_code": "US", "type": "nonprofit", "lineage": ["https://openalex.org/I109266671"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Vlad Kustanovich", "raw_affiliation_string": "Autism Genetic Resource Exchange, Autism Speaks, Los Angeles, California 90036-4234, USA.,", "raw_affiliation_strings": ["Autism Genetic Resource Exchange, Autism Speaks, Los Angeles, California 90036-4234, USA.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5065072190", "display_name": "Clara Lajonchere", "orcid": "https://orcid.org/0000-0002-3190-4606"}, "institutions": [{"id": "https://openalex.org/I109266671", "display_name": "Autism Speaks", "ror": "https://ror.org/04bkad313", "country_code": "US", "type": "nonprofit", "lineage": ["https://openalex.org/I109266671"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Clara M. Lajonchere", "raw_affiliation_string": "Autism Genetic Resource Exchange, Autism Speaks, Los Angeles, California 90036-4234, USA.,", "raw_affiliation_strings": ["Autism Genetic Resource Exchange, Autism Speaks, Los Angeles, California 90036-4234, USA.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5082992454", "display_name": "Janine Lamb", "orcid": null}, "institutions": [{"id": "https://openalex.org/I28407311", "display_name": "University of Manchester", "ror": "https://ror.org/027m9bs27", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I28407311"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Janine A. Lamb", "raw_affiliation_string": "Centre for Integrated Genomic Medical Research, University of Manchester, Manchester M13 9PT, UK.,", "raw_affiliation_strings": ["Centre for Integrated Genomic Medical Research, University of Manchester, Manchester M13 9PT, UK.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5020586330", "display_name": "Magdalena Laskawiec", "orcid": null}, "institutions": [{"id": "https://openalex.org/I2803066836", "display_name": "Warneford Hospital", "ror": "https://ror.org/03we1zb10", "country_code": "GB", "type": "healthcare", "lineage": ["https://openalex.org/I2802171185", "https://openalex.org/I2803066836"]}, {"id": "https://openalex.org/I40120149", "display_name": "University of Oxford", "ror": "https://ror.org/052gg0110", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I40120149"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Magdalena Laskawiec", "raw_affiliation_string": "Department of Psychiatry, University of Oxford, Warneford Hospital, Headington, Oxford OX3 7JX, UK.,", "raw_affiliation_strings": ["Department of Psychiatry, University of Oxford, Warneford Hospital, Headington, Oxford OX3 7JX, UK.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5043863759", "display_name": "Marion Leboyer", "orcid": "https://orcid.org/0000-0001-5473-3697"}, "institutions": [{"id": "https://openalex.org/I4210156254", "display_name": "Hôpital Albert-Chenevier", "ror": "https://ror.org/0511th722", "country_code": "FR", "type": "healthcare", "lineage": ["https://openalex.org/I4210097159", "https://openalex.org/I4210156254"]}, {"id": "https://openalex.org/I4210096842", "display_name": "Fondation FondaMental", "ror": "https://ror.org/00rrhf939", "country_code": "FR", "type": "other", "lineage": ["https://openalex.org/I4210096842"]}, {"id": "https://openalex.org/I197681013", "display_name": "Université Paris-Est Créteil", "ror": "https://ror.org/05ggc9x40", "country_code": "FR", "type": "education", "lineage": ["https://openalex.org/I197681013"]}, {"id": "https://openalex.org/I154526488", "display_name": "Inserm", "ror": "https://ror.org/02vjkv261", "country_code": "FR", "type": "government", "lineage": ["https://openalex.org/I154526488"]}], "countries": ["FR"], "is_corresponding": false, "raw_author_name": "Marion Leboyer", "raw_affiliation_string": "Department of Psychiatry, INSERM U995, Groupe Hospitalier Henri Mondor-Albert Chenevier, AP-HP,; University Paris 12, Fondation FondaMental, Créteil 94000, France.,", "raw_affiliation_strings": ["Department of Psychiatry, INSERM U995, Groupe Hospitalier Henri Mondor-Albert Chenevier, AP-HP,", "University Paris 12, Fondation FondaMental, Créteil 94000, France.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5067460528", "display_name": "Ann Le Couteur", "orcid": "https://orcid.org/0000-0001-9991-3608"}, "institutions": [{"id": "https://openalex.org/I84884186", "display_name": "Newcastle University", "ror": "https://ror.org/01kj2bm70", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I84884186"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Ann Le Couteur", "raw_affiliation_string": "Child and Adolescent Mental Health, University of Newcastle, Sir James Spence Institute, Newcastle upon Tyne NE1 4LP, UK.,", "raw_affiliation_strings": ["Child and Adolescent Mental Health, University of Newcastle, Sir James Spence Institute, Newcastle upon Tyne NE1 4LP, UK.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5063159746", "display_name": "Bennett L. Leventhal", "orcid": "https://orcid.org/0000-0001-6985-3691"}, "institutions": [{"id": "https://openalex.org/I125043107", "display_name": "Nathan Kline Institute for Psychiatric Research", "ror": "https://ror.org/01s434164", "country_code": "US", "type": "nonprofit", "lineage": ["https://openalex.org/I125043107"]}, {"id": "https://openalex.org/I57206974", "display_name": "New York University", "ror": "https://ror.org/0190ak572", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I57206974"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Bennett L. Leventhal", "raw_affiliation_string": "Department of Child and Adolescent Psychiatry, New York University and NYU Child Study Center, 550 First Avenue, New York, New York 10016, USA.,; Nathan Kline Institute for Psychiatric Research (NKI), 140 Old Orangeburg Road, Orangeburg, New York 10962, USA.,", "raw_affiliation_strings": ["Department of Child and Adolescent Psychiatry, New York University and NYU Child Study Center, 550 First Avenue, New York, New York 10016, USA.,", "Nathan Kline Institute for Psychiatric Research (NKI), 140 Old Orangeburg Road, Orangeburg, New York 10962, USA.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5077818820", "display_name": "Anath C. Lionel", "orcid": null}, "institutions": [{"id": "https://openalex.org/I2801317318", "display_name": "Hospital for Sick Children", "ror": "https://ror.org/057q4rt57", "country_code": "CA", "type": "healthcare", "lineage": ["https://openalex.org/I2801317318"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "Anath C. Lionel", "raw_affiliation_string": "The Centre for Applied Genomics and Program in Genetics and Genomic Biology, The Hospital for Sick Children, Toronto, Ontario M5G 1L7, Canada.,", "raw_affiliation_strings": ["The Centre for Applied Genomics and Program in Genetics and Genomic Biology, The Hospital for Sick Children, Toronto, Ontario M5G 1L7, Canada.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5044599087", "display_name": "Xiao Qing Liu", "orcid": "https://orcid.org/0000-0002-1767-5296"}, "institutions": [{"id": "https://openalex.org/I2801317318", "display_name": "Hospital for Sick Children", "ror": "https://ror.org/057q4rt57", "country_code": "CA", "type": "healthcare", "lineage": ["https://openalex.org/I2801317318"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "Xiao-Qing Liu", "raw_affiliation_string": "The Centre for Applied Genomics and Program in Genetics and Genomic Biology, The Hospital for Sick Children, Toronto, Ontario M5G 1L7, Canada.,", "raw_affiliation_strings": ["The Centre for Applied Genomics and Program in Genetics and Genomic Biology, The Hospital for Sick Children, Toronto, Ontario M5G 1L7, Canada.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5082294410", "display_name": "Catherine Lord", "orcid": "https://orcid.org/0000-0001-5633-1253"}, "institutions": [{"id": "https://openalex.org/I27837315", "display_name": "University of Michigan–Ann Arbor", "ror": "https://ror.org/00jmfr291", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I27837315"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Catherine Lord", "raw_affiliation_string": "Autism and Communicative Disorders Centre, University of Michigan, Ann Arbor, Michigan 48109-2054, USA.,", "raw_affiliation_strings": ["Autism and Communicative Disorders Centre, University of Michigan, Ann Arbor, Michigan 48109-2054, USA.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5087509488", "display_name": "Linda Lotspeich", "orcid": null}, "institutions": [{"id": "https://openalex.org/I97018004", "display_name": "Stanford University", "ror": "https://ror.org/00f54p054", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I97018004"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Linda Lotspeich", "raw_affiliation_string": "Department of Psychiatry, Division of Child and Adolescent Psychiatry and Child Development, Stanford University School of Medicine, Stanford, California 94304, USA.,", "raw_affiliation_strings": ["Department of Psychiatry, Division of Child and Adolescent Psychiatry and Child Development, Stanford University School of Medicine, Stanford, California 94304, USA.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5058596758", "display_name": "Sabata C. Lund", "orcid": null}, "institutions": [{"id": "https://openalex.org/I200719446", "display_name": "Vanderbilt University", "ror": "https://ror.org/02vm5rt34", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I200719446"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Sabata C. Lund", "raw_affiliation_string": "Department of Molecular Physiology and Biophysics, Vanderbilt Kennedy Center, and Centers for Human Genetics Research and Molecular Neuroscience, Vanderbilt University, Nashville, Tennessee 37232, USA.,", "raw_affiliation_strings": ["Department of Molecular Physiology and Biophysics, Vanderbilt Kennedy Center, and Centers for Human Genetics Research and Molecular Neuroscience, Vanderbilt University, Nashville, Tennessee 37232, USA.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5045533038", "display_name": "Elena Maestrini", "orcid": null}, "institutions": [{"id": "https://openalex.org/I9360294", "display_name": "University of Bologna", "ror": "https://ror.org/01111rn36", "country_code": "IT", "type": "education", "lineage": ["https://openalex.org/I9360294"]}], "countries": ["IT"], "is_corresponding": false, "raw_author_name": "Elena Maestrini", "raw_affiliation_string": "Department of Biology, University of Bologna, 40126 Bologna, Italy.,", "raw_affiliation_strings": ["Department of Biology, University of Bologna, 40126 Bologna, Italy.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5091869163", "display_name": "William J. Mahoney", "orcid": null}, "institutions": [{"id": "https://openalex.org/I98251732", "display_name": "McMaster University", "ror": "https://ror.org/02fa3aq29", "country_code": "CA", "type": "education", "lineage": ["https://openalex.org/I98251732"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "William Mahoney", "raw_affiliation_string": "Department of Pediatrics, McMaster University, Hamilton, Ontario L8N 3Z5, Canada.,", "raw_affiliation_strings": ["Department of Pediatrics, McMaster University, Hamilton, Ontario L8N 3Z5, Canada.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5085664426", "display_name": "Carine Mantoulan", "orcid": null}, "institutions": [], "countries": ["FR"], "is_corresponding": false, "raw_author_name": "Carine Mantoulan", "raw_affiliation_string": "Centre d’Eudes et de Recherches en Psychopathologie, University de Toulouse Le Mirail, Toulouse 31200, France.,", "raw_affiliation_strings": ["Centre d’Eudes et de Recherches en Psychopathologie, University de Toulouse Le Mirail, Toulouse 31200, France.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5055989261", "display_name": "Christian R. Marshall", "orcid": "https://orcid.org/0000-0002-4003-7671"}, "institutions": [{"id": "https://openalex.org/I2801317318", "display_name": "Hospital for Sick Children", "ror": "https://ror.org/057q4rt57", "country_code": "CA", "type": "healthcare", "lineage": ["https://openalex.org/I2801317318"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "Christian R. Marshall", "raw_affiliation_string": "The Centre for Applied Genomics and Program in Genetics and Genomic Biology, The Hospital for Sick Children, Toronto, Ontario M5G 1L7, Canada.,", "raw_affiliation_strings": ["The Centre for Applied Genomics and Program in Genetics and Genomic Biology, The Hospital for Sick Children, Toronto, Ontario M5G 1L7, Canada.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5078758013", "display_name": "Helen McConachie", "orcid": null}, "institutions": [{"id": "https://openalex.org/I84884186", "display_name": "Newcastle University", "ror": "https://ror.org/01kj2bm70", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I84884186"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Helen McConachie", "raw_affiliation_string": "Child and Adolescent Mental Health, University of Newcastle, Sir James Spence Institute, Newcastle upon Tyne NE1 4LP, UK.,", "raw_affiliation_strings": ["Child and Adolescent Mental Health, University of Newcastle, Sir James Spence Institute, Newcastle upon Tyne NE1 4LP, UK.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5019555383", "display_name": "Christopher J. McDougle", "orcid": "https://orcid.org/0000-0002-6229-9293"}, "institutions": [{"id": "https://openalex.org/I55769427", "display_name": "Indiana University – Purdue University Indianapolis", "ror": "https://ror.org/05gxnyn08", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I2801333002", "https://openalex.org/I55769427", "https://openalex.org/I592451"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Christopher J. McDougle", "raw_affiliation_string": "Department of Psychiatry, Indiana University School of Medicine, Indianapolis, Indiana 46202, USA.,", "raw_affiliation_strings": ["Department of Psychiatry, Indiana University School of Medicine, Indianapolis, Indiana 46202, USA.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5083956888", "display_name": "Jane McGrath", "orcid": "https://orcid.org/0000-0002-4894-4823"}, "institutions": [{"id": "https://openalex.org/I205274468", "display_name": "Trinity College Dublin", "ror": "https://ror.org/02tyrky19", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I205274468"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Jane McGrath", "raw_affiliation_string": "Department of Psychiatry, Autism Genetics Group, School of Medicine, Trinity College, Dublin 8, Ireland.,", "raw_affiliation_strings": ["Department of Psychiatry, Autism Genetics Group, School of Medicine, Trinity College, Dublin 8, Ireland.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5059720543", "display_name": "William M. McMahon", "orcid": null}, "institutions": [{"id": "https://openalex.org/I223532165", "display_name": "University of Utah", "ror": "https://ror.org/03r0ha626", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I223532165", "https://openalex.org/I2801365484"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "William M. McMahon", "raw_affiliation_string": "Psychiatry Department, University of Utah Medical School, Salt Lake City, Utah 84108, USA.,", "raw_affiliation_strings": ["Psychiatry Department, University of Utah Medical School, Salt Lake City, Utah 84108, USA.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5091281664", "display_name": "Alison Merikangas", "orcid": "https://orcid.org/0000-0003-2253-839X"}, "institutions": [{"id": "https://openalex.org/I205274468", "display_name": "Trinity College Dublin", "ror": "https://ror.org/02tyrky19", "country_code": "IE", "type": "education", "lineage": ["https://openalex.org/I205274468"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Alison Merikangas", "raw_affiliation_string": "Department of Psychiatry, Autism Genetics Group, School of Medicine, Trinity College, Dublin 8, Ireland.,", "raw_affiliation_strings": ["Department of Psychiatry, Autism Genetics Group, School of Medicine, Trinity College, Dublin 8, Ireland.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5009995418", "display_name": "Ohsuke Migita", "orcid": null}, "institutions": [{"id": "https://openalex.org/I2801317318", "display_name": "Hospital for Sick Children", "ror": "https://ror.org/057q4rt57", "country_code": "CA", "type": "healthcare", "lineage": ["https://openalex.org/I2801317318"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "Ohsuke Migita", "raw_affiliation_string": "The Centre for Applied Genomics and Program in Genetics and Genomic Biology, The Hospital for Sick Children, Toronto, Ontario M5G 1L7, Canada.,", "raw_affiliation_strings": ["The Centre for Applied Genomics and Program in Genetics and Genomic Biology, The Hospital for Sick Children, Toronto, Ontario M5G 1L7, Canada.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5010132043", "display_name": "Nancy J. Minshew", "orcid": null}, "institutions": [{"id": "https://openalex.org/I170201317", "display_name": "University of Pittsburgh", "ror": "https://ror.org/01an3r305", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I170201317"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Nancy J. Minshew", "raw_affiliation_string": "Departments of Psychiatry and Neurology, University of Pittsburgh School of Medicine, Pittsburgh, Pennsylvania 15213, USA.,", "raw_affiliation_strings": ["Departments of Psychiatry and Neurology, University of Pittsburgh School of Medicine, Pittsburgh, Pennsylvania 15213, USA.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5079047456", "display_name": "Ghazala Mirza", "orcid": null}, "institutions": [{"id": "https://openalex.org/I1336263701", "display_name": "Wellcome Centre for Human Genetics", "ror": "https://ror.org/01rjnta51", "country_code": "GB", "type": "facility", "lineage": ["https://openalex.org/I1336263701", "https://openalex.org/I40120149", "https://openalex.org/I87048295"]}, {"id": "https://openalex.org/I40120149", "display_name": "University of Oxford", "ror": "https://ror.org/052gg0110", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I40120149"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Ghazala K. Mirza", "raw_affiliation_string": "Wellcome Trust Centre for Human Genetics, University of Oxford, Oxford OX3 7BN, UK.,", "raw_affiliation_strings": ["Wellcome Trust Centre for Human Genetics, University of Oxford, Oxford OX3 7BN, UK.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5078933163", "display_name": "Jeff Munson", "orcid": null}, "institutions": [{"id": "https://openalex.org/I201448701", "display_name": "University of Washington", "ror": "https://ror.org/00cvxb145", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I201448701"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Jeff Munson", "raw_affiliation_string": "Department of Psychiatry and Behavioural Sciences, University of Washington, Seattle, Washington 98195, USA.,", "raw_affiliation_strings": ["Department of Psychiatry and Behavioural Sciences, University of Washington, Seattle, Washington 98195, USA.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5073896107", "display_name": "Stanley F. Nelson", "orcid": "https://orcid.org/0000-0002-2082-3114"}, "institutions": [{"id": "https://openalex.org/I161318765", "display_name": "University of California, Los Angeles", "ror": "https://ror.org/046rm7j60", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I161318765", "https://openalex.org/I2803209242"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Stanley F. Nelson", "raw_affiliation_string": "Department of Human Genetics, University of California—Los Angeles School of Medicine, Los Angeles, California 90095, USA.,", "raw_affiliation_strings": ["Department of Human Genetics, University of California—Los Angeles School of Medicine, Los Angeles, California 90095, USA.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5081166269", "display_name": "Carolyn Noakes", "orcid": null}, "institutions": [{"id": "https://openalex.org/I2801420703", "display_name": "Holland Bloorview Kids Rehabilitation Hospital", "ror": "https://ror.org/03qea8398", "country_code": "CA", "type": "healthcare", "lineage": ["https://openalex.org/I2801420703"]}, {"id": "https://openalex.org/I185261750", "display_name": "University of Toronto", "ror": "https://ror.org/03dbr7087", "country_code": "CA", "type": "education", "lineage": ["https://openalex.org/I185261750"]}, {"id": "https://openalex.org/I2801317318", "display_name": "Hospital for Sick Children", "ror": "https://ror.org/057q4rt57", "country_code": "CA", "type": "healthcare", "lineage": ["https://openalex.org/I2801317318"]}, {"id": "https://openalex.org/I4210141030", "display_name": "SickKids Foundation", "ror": "https://ror.org/04374qe70", "country_code": "CA", "type": "nonprofit", "lineage": ["https://openalex.org/I4210141030"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "Carolyn Noakes", "raw_affiliation_string": "Autism Research Unit, The Hospital for Sick Children and Bloorview Kids Rehab, University of Toronto, Toronto, Ontario M5G 1X8, Canada.,", "raw_affiliation_strings": ["Autism Research Unit, The Hospital for Sick Children and Bloorview Kids Rehab, University of Toronto, Toronto, Ontario M5G 1X8, Canada.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5057232191", "display_name": "Abdul Noor", "orcid": "https://orcid.org/0000-0002-4892-5876"}, "institutions": [{"id": "https://openalex.org/I1338135719", "display_name": "Centre for Addiction and Mental Health", "ror": "https://ror.org/03e71c577", "country_code": "CA", "type": "healthcare", "lineage": ["https://openalex.org/I1338135719"]}, {"id": "https://openalex.org/I185261750", "display_name": "University of Toronto", "ror": "https://ror.org/03dbr7087", "country_code": "CA", "type": "education", "lineage": ["https://openalex.org/I185261750"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "Abdul Noor", "raw_affiliation_string": "Clarke Institute and Department of Psychiatry, Centre for Addiction and Mental Health, University of Toronto, Toronto, Ontario M5G 1X8, Canada.,", "raw_affiliation_strings": ["Clarke Institute and Department of Psychiatry, Centre for Addiction and Mental Health, University of Toronto, Toronto, Ontario M5G 1X8, Canada.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5061989416", "display_name": "Gudrun Nygren", "orcid": null}, "institutions": [{"id": "https://openalex.org/I881427289", "display_name": "University of Gothenburg", "ror": "https://ror.org/01tm6cn81", "country_code": "SE", "type": "education", "lineage": ["https://openalex.org/I881427289"]}], "countries": ["SE"], "is_corresponding": false, "raw_author_name": "Gudrun Nygren", "raw_affiliation_string": "Department of Child and Adolescent Psychiatry, Göteborg University, Göteborg S41345, Sweden.,", "raw_affiliation_strings": ["Department of Child and Adolescent Psychiatry, Göteborg University, Göteborg S41345, Sweden.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5030071878", "display_name": "Guiomar Oliveira", "orcid": "https://orcid.org/0000-0002-7049-1277"}, "institutions": [], "countries": ["PT"], "is_corresponding": false, "raw_author_name": "Guiomar Oliveira", "raw_affiliation_string": "Hospital Pediátrico de Coimbra, 3000 – 076 Coimbra, Portugal.,", "raw_affiliation_strings": ["Hospital Pediátrico de Coimbra, 3000 – 076 Coimbra, Portugal.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5023320998", "display_name": "Κaterina Papanikolaou", "orcid": "https://orcid.org/0000-0002-5774-5375"}, "institutions": [{"id": "https://openalex.org/I4210120313", "display_name": "Children's Hospital Agia Sophia", "ror": "https://ror.org/0315ea826", "country_code": "GR", "type": "healthcare", "lineage": ["https://openalex.org/I4210120313"]}], "countries": ["GR"], "is_corresponding": false, "raw_author_name": "Katerina Papanikolaou", "raw_affiliation_string": "University Department of Child Psychiatry, Athens University, Medical School, Agia Sophia Children’s Hospital, 115 27 Athens, Greece.,", "raw_affiliation_strings": ["University Department of Child Psychiatry, Athens University, Medical School, Agia Sophia Children’s Hospital, 115 27 Athens, Greece.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5015104719", "display_name": "Jeremy Parr", "orcid": "https://orcid.org/0000-0002-2507-7878"}, "institutions": [{"id": "https://openalex.org/I84884186", "display_name": "Newcastle University", "ror": "https://ror.org/01kj2bm70", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I84884186"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Jeremy R. Parr", "raw_affiliation_string": "Insitutes of Neuroscience and Health and Society, Newcastle University, Newcastle Upon Tyne NE1 7RU, UK.,", "raw_affiliation_strings": ["Insitutes of Neuroscience and Health and Society, Newcastle University, Newcastle Upon Tyne NE1 7RU, UK.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5057931685", "display_name": "Barbara Parrini", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210126460", "display_name": "Fondazione Stella Maris", "ror": "https://ror.org/02w8ez808", "country_code": "IT", "type": "healthcare", "lineage": ["https://openalex.org/I4210126460", "https://openalex.org/I4210153126"]}], "countries": ["IT"], "is_corresponding": false, "raw_author_name": "Barbara Parrini", "raw_affiliation_string": "Stella Maris Institute for Child and Adolescent Neuropsychiatry, 56128 Calambrone (Pisa), Italy.,", "raw_affiliation_strings": ["Stella Maris Institute for Child and Adolescent Neuropsychiatry, 56128 Calambrone (Pisa), Italy.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5078094307", "display_name": "Tara Paton", "orcid": null}, "institutions": [{"id": "https://openalex.org/I2801317318", "display_name": "Hospital for Sick Children", "ror": "https://ror.org/057q4rt57", "country_code": "CA", "type": "healthcare", "lineage": ["https://openalex.org/I2801317318"]}], "countries": ["CA"], "is_corresponding": false, "raw_author_name": "Tara Paton", "raw_affiliation_string": "The Centre for Applied Genomics and Program in Genetics and Genomic Biology, The Hospital for Sick Children, Toronto, Ontario M5G 1L7, Canada.,", "raw_affiliation_strings": ["The Centre for Applied Genomics and Program in Genetics and Genomic Biology, The Hospital for Sick Children, Toronto, Ontario M5G 1L7, Canada.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5033846900", "display_name": "Andrew Pickles", "orcid": "https://orcid.org/0000-0003-1283-0346"}, "institutions": [{"id": "https://openalex.org/I28407311", "display_name": "University of Manchester", "ror": "https://ror.org/027m9bs27", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I28407311"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Andrew Pickles", "raw_affiliation_string": "Department of Medicine, School of Epidemiology and Health Science, University of Manchester, Manchester M13 9PT, UK.,", "raw_affiliation_strings": ["Department of Medicine, School of Epidemiology and Health Science, University of Manchester, Manchester M13 9PT, UK.,"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5023343529", "display_name": "Marion Pilorge", "orcid": null}, "institutions": [{"id": "https://openalex.org/I39804081", "display_name": "Sorbonne University", "ror": "https://ror.org/02en5vm52", "country_code": "FR", "type": "education", "lineage": ["https://openalex.org/I39804081"]}, {"id": "https://openalex.org/I154526488", "display_name": "Inserm", "ror": "https://ror.org/02vjkv261", "country_code": "FR", "type": "government", "lineage": ["https://openalex.org/I154526488"]}, {"id": "https://openalex.org/I1294671590", "display_name": "French National Centre for Scientific Research", "ror": "https://ror.org/02feahw73", "country_code": "FR", "type": "government", "lineage": ["https://openalex.org/I1294671590"]}], "countries": ["FR"], "is_corresponding": false, "raw_author_name": "Marion Pilorge", "raw_affiliation_string": "INSERM U952 and CNRS UMR 7224 and UPMC Univ Paris 06, Paris 75005, France.,", "raw_affiliation_strings": ["INSERM U952 and CNRS UMR 7224 and UPMC Univ Paris 06, Paris 75005, France.,"]}], "countries_distinct_count": 11, "institutions_distinct_count": 70, "corresponding_author_ids": ["https://openalex.org/A5049296994"], "corresponding_institution_ids": ["https://openalex.org/I2801317318", "https://openalex.org/I185261750"], "apc_list": {"value": 9750, "currency": "EUR", "value_usd": 11690, "provenance": "doaj"}, "apc_paid": {"value": 9750, "currency": "EUR", "value_usd": 11690, "provenance": "doaj"}, "is_authors_truncated": true, "has_fulltext": true, "fulltext_origin": "pdf", "cited_by_count": 1789, "cited_by_percentile_year": {"min": 99.9, "max": 100.0}, "biblio": {"volume": "466", "issue": "7304", "first_page": "368", "last_page": "372"}, "is_retracted": false, "is_paratext": false, "keywords": [{"keyword": "autism spectrum disorders", "score": 0.561}, {"keyword": "rare", "score": 0.2942}, {"keyword": "number", "score": 0.2594}], "concepts": [{"id": "https://openalex.org/C120821319", "wikidata": "https://www.wikidata.org/wiki/Q1501491", "display_name": "Copy-number variation", "level": 4, "score": 0.9137362}, {"id": "https://openalex.org/C54355233", "wikidata": "https://www.wikidata.org/wiki/Q7162", "display_name": "Genetics", "level": 1, "score": 0.6763087}, {"id": "https://openalex.org/C205778803", "wikidata": "https://www.wikidata.org/wiki/Q38404", "display_name": "Autism", "level": 2, "score": 0.66470337}, {"id": "https://openalex.org/C551499885", "wikidata": "https://www.wikidata.org/wiki/Q183560", "display_name": "Intellectual disability", "level": 2, "score": 0.648405}, {"id": "https://openalex.org/C86803240", "wikidata": "https://www.wikidata.org/wiki/Q420", "display_name": "Biology", "level": 0, "score": 0.5920409}, {"id": "https://openalex.org/C2778538070", "wikidata": "https://www.wikidata.org/wiki/Q1436063", "display_name": "Autism spectrum disorder", "level": 3, "score": 0.52668464}, {"id": "https://openalex.org/C106208931", "wikidata": "https://www.wikidata.org/wiki/Q1098876", "display_name": "Genome-wide association study", "level": 5, "score": 0.5093051}, {"id": "https://openalex.org/C108701171", "wikidata": "https://www.wikidata.org/wiki/Q3145036", "display_name": "Heritability of autism", "level": 4, "score": 0.5024488}, {"id": "https://openalex.org/C104317684", "wikidata": "https://www.wikidata.org/wiki/Q7187", "display_name": "Gene", "level": 2, "score": 0.4873384}, {"id": "https://openalex.org/C84597430", "wikidata": "https://www.wikidata.org/wiki/Q106227", "display_name": "Locus (genetics)", "level": 3, "score": 0.46522638}, {"id": "https://openalex.org/C141231307", "wikidata": "https://www.wikidata.org/wiki/Q7020", "display_name": "Genome", "level": 3, "score": 0.32067275}, {"id": "https://openalex.org/C127716648", "wikidata": "https://www.wikidata.org/wiki/Q104053", "display_name": "Phenotype", "level": 3, "score": 0.22953641}, {"id": "https://openalex.org/C153209595", "wikidata": "https://www.wikidata.org/wiki/Q501128", "display_name": "Single-nucleotide polymorphism", "level": 4, "score": 0.2290332}, {"id": "https://openalex.org/C135763542", "wikidata": "https://www.wikidata.org/wiki/Q106016", "display_name": "Genotype", "level": 3, "score": 0.19741482}, {"id": "https://openalex.org/C15744967", "wikidata": "https://www.wikidata.org/wiki/Q9418", "display_name": "Psychology", "level": 0, "score": 0.1837013}, {"id": "https://openalex.org/C138496976", "wikidata": "https://www.wikidata.org/wiki/Q175002", "display_name": "Developmental psychology", "level": 1, "score": 0.12833062}], "mesh": [{"descriptor_ui": "D002659", "descriptor_name": "Child Development Disorders, Pervasive", "qualifier_ui": "Q000235", "qualifier_name": "genetics", "is_major_topic": true}, {"descriptor_ui": "D002659", "descriptor_name": "Child Development Disorders, Pervasive", "qualifier_ui": "Q000503", "qualifier_name": "physiopathology", "is_major_topic": true}, {"descriptor_ui": "D056915", "descriptor_name": "DNA Copy Number Variations", "qualifier_ui": "Q000235", "qualifier_name": "genetics", "is_major_topic": true}, {"descriptor_ui": "D018628", "descriptor_name": "Gene Dosage", "qualifier_ui": "Q000235", "qualifier_name": "genetics", "is_major_topic": true}, {"descriptor_ui": "D020022", "descriptor_name": "Genetic Predisposition to Disease", "qualifier_ui": "Q000235", "qualifier_name": "genetics", "is_major_topic": true}, {"descriptor_ui": "D016022", "descriptor_name": "Case-Control Studies", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D002465", "descriptor_name": "Cell Movement", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D002648", "descriptor_name": "Child", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D002659", "descriptor_name": "Child Development Disorders, Pervasive", "qualifier_ui": "Q000473", "qualifier_name": "pathology", "is_major_topic": false}, {"descriptor_ui": "D002659", "descriptor_name": "Child Development Disorders, Pervasive", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D019610", "descriptor_name": "Cytoprotection", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D056915", "descriptor_name": "DNA Copy Number Variations", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D005060", "descriptor_name": "Europe", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D005060", "descriptor_name": "Europe", "qualifier_ui": "Q000208", "qualifier_name": "ethnology", "is_major_topic": false}, {"descriptor_ui": "D018628", "descriptor_name": "Gene Dosage", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D020022", "descriptor_name": "Genetic Predisposition to Disease", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D055106", "descriptor_name": "Genome-Wide Association Study", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D006801", "descriptor_name": "Humans", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D015398", "descriptor_name": "Signal Transduction", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D012919", "descriptor_name": "Social Behavior", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}], "locations_count": 11, "locations": [{"is_oa": true, "landing_page_url": "https://doi.org/10.1038/nature09146", "pdf_url": "https://www.nature.com/articles/nature09146.pdf", "source": {"id": "https://openalex.org/S137773608", "display_name": "Nature", "issn_l": "0028-0836", "issn": ["1476-4687", "0028-0836"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310319908", "host_organization_name": "Nature Portfolio", "host_organization_lineage": ["https://openalex.org/P4310319908", "https://openalex.org/P4310319965"], "host_organization_lineage_names": ["Nature Portfolio", "Springer Nature"], "type": "journal"}, "license": null, "version": "publishedVersion", "is_accepted": true, "is_published": true}, {"is_oa": true, "landing_page_url": "https://www.research.manchester.ac.uk/portal/en/publications/functional-impact-of-global-rare-copy-number-variation-in-autism-spectrum-disorders(a89151d4-350e-4d7c-89e2-0df4e211b1ba).html", "pdf_url": "https://research.manchester.ac.uk/files/30298262/POST-PEER-REVIEW-PUBLISHERS-DOCUMENT.PDF", "source": {"id": "https://openalex.org/S4306400662", "display_name": "Research Explorer (The University of Manchester)", "issn_l": null, "issn": null, "is_oa": true, "is_in_doaj": false, "host_organization": "https://openalex.org/I28407311", "host_organization_name": "University of Manchester", "host_organization_lineage": ["https://openalex.org/I28407311"], "host_organization_lineage_names": ["University of Manchester"], "type": "repository"}, "license": null, "version": "publishedVersion", "is_accepted": true, "is_published": true}, {"is_oa": true, "landing_page_url": "https://pure.manchester.ac.uk/ws/files/30298262/POST-PEER-REVIEW-PUBLISHERS-DOCUMENT.PDF", "pdf_url": "https://pure.manchester.ac.uk/ws/files/30298262/POST-PEER-REVIEW-PUBLISHERS-DOCUMENT.PDF", "source": {"id": "https://openalex.org/S4306400662", "display_name": "Research Explorer (The University of Manchester)", "issn_l": null, "issn": null, "is_oa": true, "is_in_doaj": false, "host_organization": "https://openalex.org/I28407311", "host_organization_name": "University of Manchester", "host_organization_lineage": ["https://openalex.org/I28407311"], "host_organization_lineage_names": ["University of Manchester"], "type": "repository"}, "license": null, "version": "publishedVersion", "is_accepted": true, "is_published": true}, {"is_oa": true, "landing_page_url": "http://hdl.handle.net/10400.18/214", "pdf_url": "http://repositorio.insa.pt/bitstream/10400.18/214/1/Functional%20impact%20of%20global%20rare%20copy%20number%20variation%20in%20autism%20spectrum%20disorders.pdf", "source": {"id": "https://openalex.org/S4306402433", "display_name": "Portuguese National Funding Agency for Science, Research and Technology (RCAAP Project by FCT)", "issn_l": null, "issn": null, "is_oa": true, "is_in_doaj": false, "host_organization": null, "host_organization_name": null, "host_organization_lineage": [], "host_organization_lineage_names": [], "type": "repository"}, "license": null, "version": "publishedVersion", "is_accepted": true, "is_published": true}, {"is_oa": true, "landing_page_url": "https://europepmc.org/articles/pmc3021798", "pdf_url": "https://europepmc.org/articles/pmc3021798?pdf=render", "source": {"id": "https://openalex.org/S4306400806", "display_name": "Europe PMC (PubMed Central)", "issn_l": null, "issn": null, "is_oa": true, "is_in_doaj": false, "host_organization": "https://openalex.org/I1303153112", "host_organization_name": "European Bioinformatics Institute", "host_organization_lineage": ["https://openalex.org/I1303153112"], "host_organization_lineage_names": ["European Bioinformatics Institute"], "type": "repository"}, "license": null, "version": "acceptedVersion", "is_accepted": true, "is_published": false}, {"is_oa": true, "landing_page_url": "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3021798", "pdf_url": null, "source": {"id": "https://openalex.org/S2764455111", "display_name": "PubMed Central", "issn_l": null, "issn": null, "is_oa": true, "is_in_doaj": false, "host_organization": "https://openalex.org/I1299303238", "host_organization_name": "National Institutes of Health", "host_organization_lineage": ["https://openalex.org/I1299303238"], "host_organization_lineage_names": ["National Institutes of Health"], "type": "repository"}, "license": null, "version": "acceptedVersion", "is_accepted": true, "is_published": false}, {"is_oa": true, "landing_page_url": "http://hdl.handle.net/10197/4381", "pdf_url": "http://researchrepository.ucd.ie/bitstreams/bd73528d-640c-49b3-a5d2-0e9275c7c9d6/download", "source": {"id": "https://openalex.org/S4306402280", "display_name": "Research Repository UCD (University College Dublin)", "issn_l": null, "issn": null, "is_oa": true, "is_in_doaj": false, "host_organization": "https://openalex.org/I100930933", "host_organization_name": "University College Dublin", "host_organization_lineage": ["https://openalex.org/I100930933"], "host_organization_lineage_names": ["University College Dublin"], "type": "repository"}, "license": "cc-by-nc-nd", "version": "submittedVersion", "is_accepted": false, "is_published": false}, {"is_oa": true, "landing_page_url": "https://www.hal.inserm.fr/inserm-00521387/file/Pinto_AGP_CNV_Nature_2010.pdf", "pdf_url": "https://www.hal.inserm.fr/inserm-00521387/file/Pinto_AGP_CNV_Nature_2010.pdf", "source": {"id": "https://openalex.org/S4306402512", "display_name": "HAL (Le Centre pour la Communication Scientifique Directe)", "issn_l": null, "issn": null, "is_oa": true, "is_in_doaj": false, "host_organization": "https://openalex.org/I1294671590", "host_organization_name": "French National Centre for Scientific Research", "host_organization_lineage": ["https://openalex.org/I1294671590"], "host_organization_lineage_names": ["French National Centre for Scientific Research"], "type": "repository"}, "license": null, "version": "submittedVersion", "is_accepted": false, "is_published": false}, {"is_oa": true, "landing_page_url": "https://www.hal.inserm.fr/inserm-00521387", "pdf_url": "https://inserm.hal.science/inserm-00521387/document", "source": {"id": "https://openalex.org/S4306402512", "display_name": "HAL (Le Centre pour la Communication Scientifique Directe)", "issn_l": null, "issn": null, "is_oa": true, "is_in_doaj": false, "host_organization": "https://openalex.org/I1294671590", "host_organization_name": "French National Centre for Scientific Research", "host_organization_lineage": ["https://openalex.org/I1294671590"], "host_organization_lineage_names": ["French National Centre for Scientific Research"], "type": "repository"}, "license": null, "version": "submittedVersion", "is_accepted": false, "is_published": false}, {"is_oa": true, "landing_page_url": "https://www.hal.inserm.fr/inserm-00521387/document", "pdf_url": "https://www.hal.inserm.fr/inserm-00521387/document", "source": {"id": "https://openalex.org/S4306402512", "display_name": "HAL (Le Centre pour la Communication Scientifique Directe)", "issn_l": null, "issn": null, "is_oa": true, "is_in_doaj": false, "host_organization": "https://openalex.org/I1294671590", "host_organization_name": "French National Centre for Scientific Research", "host_organization_lineage": ["https://openalex.org/I1294671590"], "host_organization_lineage_names": ["French National Centre for Scientific Research"], "type": "repository"}, "license": null, "version": "submittedVersion", "is_accepted": false, "is_published": false}, {"is_oa": false, "landing_page_url": "https://pubmed.ncbi.nlm.nih.gov/20531469", "pdf_url": null, "source": {"id": "https://openalex.org/S4306525036", "display_name": "PubMed", "issn_l": null, "issn": null, "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/I1299303238", "host_organization_name": "National Institutes of Health", "host_organization_lineage": ["https://openalex.org/I1299303238"], "host_organization_lineage_names": ["National Institutes of Health"], "type": "repository"}, "license": null, "version": null, "is_accepted": false, "is_published": false}], "best_oa_location": {"is_oa": true, "landing_page_url": "https://doi.org/10.1038/nature09146", "pdf_url": "https://www.nature.com/articles/nature09146.pdf", "source": {"id": "https://openalex.org/S137773608", "display_name": "Nature", "issn_l": "0028-0836", "issn": ["1476-4687", "0028-0836"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310319908", "host_organization_name": "Nature Portfolio", "host_organization_lineage": ["https://openalex.org/P4310319908", "https://openalex.org/P4310319965"], "host_organization_lineage_names": ["Nature Portfolio", "Springer Nature"], "type": "journal"}, "license": null, "version": "publishedVersion", "is_accepted": true, "is_published": true}, "sustainable_development_goals": [{"id": "https://metadata.un.org/sdg/10", "display_name": "Reduced inequalities", "score": 0.5}, {"id": "https://metadata.un.org/sdg/4", "display_name": "Quality Education", "score": 0.32}], "grants": [], "referenced_works_count": 30, "referenced_works": ["https://openalex.org/W1971321176", "https://openalex.org/W1979803024", "https://openalex.org/W1983194657", "https://openalex.org/W1985400815", "https://openalex.org/W1989879415", "https://openalex.org/W2021714239", "https://openalex.org/W2025142360", "https://openalex.org/W2037995604", "https://openalex.org/W2046683077", "https://openalex.org/W2050161506", "https://openalex.org/W2073062083", "https://openalex.org/W2074579392", "https://openalex.org/W2094379124", "https://openalex.org/W2109529720", "https://openalex.org/W2117154339", "https://openalex.org/W2119907069", "https://openalex.org/W2123739801", "https://openalex.org/W2125147395", "https://openalex.org/W2130410032", "https://openalex.org/W2137531873", "https://openalex.org/W2141704631", "https://openalex.org/W2142452151", "https://openalex.org/W2143947527", "https://openalex.org/W2144601878", "https://openalex.org/W2149681218", "https://openalex.org/W2151566815", "https://openalex.org/W2156676902", "https://openalex.org/W2158778721", "https://openalex.org/W2161633633", "https://openalex.org/W2162066558"], "related_works": ["https://openalex.org/W2781225729", "https://openalex.org/W2915855127", "https://openalex.org/W3215405001", "https://openalex.org/W2013286117", "https://openalex.org/W2106428414", "https://openalex.org/W2118617229", "https://openalex.org/W2525895979", "https://openalex.org/W2088224929", "https://openalex.org/W1483729743", "https://openalex.org/W3205273894"], "ngrams_url": "https://api.openalex.org/works/W2110374888/ngrams", "abstract_inverted_index": {"The": [0], "autism": [1], "spectrum": [2], "disorders": [3], "(ASDs)": [4], "are": [5, 47, 60], "a": [6, 100, 150], "group": [7], "of": [8, 22, 70, 88, 104, 173], "conditions": [9], "characterized": [10], "by": [11], "impairments": [12], "in": [13, 33, 77, 122, 147, 149, 180, 198], "reciprocal": [14], "social": [15], "interaction": [16], "and": [17, 19, 24, 143, 163, 184, 186, 195], "communication,": [18], "the": [20, 56, 67, 136, 164], "presence": [21], "restricted": [23], "repetitive": [25], "behaviours.": [26], "Individuals": [27], "with": [28], "an": [29, 171], "ASD": [30, 78, 86, 124, 156, 199], "vary": [31], "greatly": [32], "cognitive": [34], "development,": [35], "which": [36], "can": [37], "range": [38], "from": [39], "above": [40], "average": [41], "to": [42, 49, 91, 98, 203], "intellectual": [43, 126], "disability.": [44], "Although": [45], "ASDs": [46], "known": [48], "be": [50], "highly": [51], "heritable": [52], "(": [53], "approximately": [54], "90%),": [55], "underlying": [57], "genetic": [58, 194], "determinants": [59], "still": [61], "largely": [62], "unknown.": [63], "Here": [64], "we": [65], "analysed": [66], "genome-wide": [68], "characteristics": [69], "rare": [71], "(<1%": [72], "frequency)": [73], "copy": [74, 107], "number": [75, 108], "variation": [76], "using": [79], "dense": [80], "genotyping": [81], "arrays.": [82], "When": [83], "comparing": [84], "996": [85], "individuals": [87], "European": [89], "ancestry": [90], "1,287": [92], "matched": [93], "controls,": [94], "cases": [95], "were": [96, 139], "found": [97], "carry": [99], "higher": [101], "global": [102], "burden": [103], "rare,": [105], "genic": [106], "variants": [109], "(CNVs)": [110], "(1.19": [111], "fold,": [112, 129], "P": [113, 130], "=": [114, 131], "0.012),": [115], "especially": [116], "so": [117], "for": [118], "loci": [119], "previously": [120], "implicated": [121], "either": [123], "and/or": [125], "disability": [127], "(1.69": [128], "3.4": [132], "x": [133], "10(-4)).": [134], "Among": [135], "CNVs": [137, 174], "there": [138], "numerous": [140], "de": [141], "novo": [142], "inherited": [144], "events,": [145], "sometimes": [146], "combination": [148], "given": [151], "family,": [152], "implicating": [153], "many": [154, 192], "novel": [155], "genes": [157], "such": [158], "as": [159], "SHANK2,": [160], "SYNGAP1,": [161], "DLGAP2": [162], "X-linked": [165], "DDX53-PTCHD1": [166], "locus.": [167], "We": [168], "also": [169], "discovered": [170], "enrichment": [172], "disrupting": [175], "functional": [176, 196], "gene": [177], "sets": [178], "involved": [179], "cellular": [181], "proliferation,": [182], "projection": [183], "motility,": [185], "GTPase/Ras": [187], "signalling.": [188], "Our": [189], "results": [190], "reveal": [191], "new": [193], "targets": [197], "that": [200], "may": [201], "lead": [202], "final": [204], "connected": [205], "pathways.": [206]}, "cited_by_api_url": "https://api.openalex.org/works?filter=cites:W2110374888", "counts_by_year": [{"year": 2023, "cited_by_count": 56}, {"year": 2022, "cited_by_count": 88}, {"year": 2021, "cited_by_count": 93}, {"year": 2020, "cited_by_count": 98}, {"year": 2019, "cited_by_count": 96}, {"year": 2018, "cited_by_count": 108}, {"year": 2017, "cited_by_count": 116}, {"year": 2016, "cited_by_count": 128}, {"year": 2015, "cited_by_count": 174}, {"year": 2014, "cited_by_count": 192}, {"year": 2013, "cited_by_count": 200}, {"year": 2012, "cited_by_count": 200}], "updated_date": "2023-12-07T09:19:47.159076", "created_date": "2016-06-24"} \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/webcrawl/part-00002 b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/webcrawl/part-00002 new file mode 100644 index 000000000..f64206167 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/webcrawl/part-00002 @@ -0,0 +1 @@ +{"id": "https://openalex.org/W2115169717", "doi": "https://doi.org/10.1016/s0140-6736(09)61965-6", "title": "Statins and risk of incident diabetes: a collaborative meta-analysis of randomised statin trials", "display_name": "Statins and risk of incident diabetes: a collaborative meta-analysis of randomised statin trials", "publication_year": 2010, "publication_date": "2010-02-01", "ids": {"openalex": "https://openalex.org/W2115169717", "doi": "https://doi.org/10.1016/s0140-6736(09)61965-6", "mag": "2115169717", "pmid": "https://pubmed.ncbi.nlm.nih.gov/20167359"}, "language": "en", "primary_location": {"is_oa": false, "landing_page_url": "https://doi.org/10.1016/s0140-6736(09)61965-6", "pdf_url": null, "source": {"id": "https://openalex.org/S49861241", "display_name": "The Lancet", "issn_l": "0140-6736", "issn": ["1474-547X", "0099-5355", "0140-6736"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310320990", "host_organization_name": "Elsevier BV", "host_organization_lineage": ["https://openalex.org/P4310320990"], "host_organization_lineage_names": ["Elsevier BV"], "type": "journal"}, "license": null, "version": null, "is_accepted": false, "is_published": false}, "type": "article", "type_crossref": "journal-article", "open_access": {"is_oa": false, "oa_status": "closed", "oa_url": null, "any_repository_has_fulltext": false}, "authorships": [{"author_position": "first", "author": {"id": "https://openalex.org/A5078498803", "display_name": "Naveed Sattar", "orcid": null}, "institutions": [{"id": "https://openalex.org/I32003884", "display_name": "British Heart Foundation", "ror": "https://ror.org/02wdwnk04", "country_code": "GB", "type": "nonprofit", "lineage": ["https://openalex.org/I32003884"]}, {"id": "https://openalex.org/I7882870", "display_name": "University of Glasgow", "ror": "https://ror.org/00vtgdb53", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I7882870"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Naveed Sattar", "raw_affiliation_string": "British Heart Foundation Glasgow Cardiovascular Research Centre; University of Glasgow; Glasgow UK", "raw_affiliation_strings": ["British Heart Foundation Glasgow Cardiovascular Research Centre; University of Glasgow; Glasgow UK"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5039985378", "display_name": "David Preiss", "orcid": "https://orcid.org/0000-0003-3139-1836"}, "institutions": [{"id": "https://openalex.org/I7882870", "display_name": "University of Glasgow", "ror": "https://ror.org/00vtgdb53", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I7882870"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "David Preiss", "raw_affiliation_string": "*University of Glasgow.", "raw_affiliation_strings": ["*University of Glasgow."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5005987955", "display_name": "Heather Murray", "orcid": null}, "institutions": [{"id": "https://openalex.org/I7882870", "display_name": "University of Glasgow", "ror": "https://ror.org/00vtgdb53", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I7882870"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Heather M Murray", "raw_affiliation_string": "*University of Glasgow.", "raw_affiliation_strings": ["*University of Glasgow."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5021225266", "display_name": "Paul Welsh", "orcid": "https://orcid.org/0000-0002-7970-3643"}, "institutions": [{"id": "https://openalex.org/I7882870", "display_name": "University of Glasgow", "ror": "https://ror.org/00vtgdb53", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I7882870"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Paul Welsh", "raw_affiliation_string": "*University of Glasgow.", "raw_affiliation_strings": ["*University of Glasgow."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5034827753", "display_name": "Brendan M. Buckley", "orcid": "https://orcid.org/0000-0003-1544-8003"}, "institutions": [{"id": "https://openalex.org/I2802396013", "display_name": "Cork University Hospital", "ror": "https://ror.org/04q107642", "country_code": "IE", "type": "healthcare", "lineage": ["https://openalex.org/I2802396013"]}], "countries": ["IE"], "is_corresponding": false, "raw_author_name": "Brendan M Buckley", "raw_affiliation_string": "Department of Pharmacology and Therapeutics; Cork University Hospital; Cork Ireland", "raw_affiliation_strings": ["Department of Pharmacology and Therapeutics; Cork University Hospital; Cork Ireland"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5017836069", "display_name": "Anton J. M. de Craen", "orcid": null}, "institutions": [{"id": "https://openalex.org/I121797337", "display_name": "Leiden University", "ror": "https://ror.org/027bh9e22", "country_code": "NL", "type": "education", "lineage": ["https://openalex.org/I121797337"]}], "countries": ["NL"], "is_corresponding": false, "raw_author_name": "Anton J M de Craen", "raw_affiliation_string": "Leiden University", "raw_affiliation_strings": ["Leiden University"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5040241232", "display_name": "Sreenivasa Rao Kondapally Seshasai", "orcid": "https://orcid.org/0000-0002-5948-6522"}, "institutions": [{"id": "https://openalex.org/I241749", "display_name": "University of Cambridge", "ror": "https://ror.org/013meh722", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I241749"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Sreenivasa Rao Kondapally Seshasai", "raw_affiliation_string": "Univ. of Cambridge", "raw_affiliation_strings": ["Univ. of Cambridge"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5027213857", "display_name": "John J.V. McMurray", "orcid": "https://orcid.org/0000-0002-6317-3975"}, "institutions": [{"id": "https://openalex.org/I7882870", "display_name": "University of Glasgow", "ror": "https://ror.org/00vtgdb53", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I7882870"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "John J McMurray", "raw_affiliation_string": "*University of Glasgow.", "raw_affiliation_strings": ["*University of Glasgow."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5085003120", "display_name": "Dilys J. Freeman", "orcid": null}, "institutions": [{"id": "https://openalex.org/I7882870", "display_name": "University of Glasgow", "ror": "https://ror.org/00vtgdb53", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I7882870"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Dilys J Freeman", "raw_affiliation_string": "*University of Glasgow.", "raw_affiliation_strings": ["*University of Glasgow."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5005451442", "display_name": "J. Wouter Jukema", "orcid": "https://orcid.org/0000-0002-3246-8359"}, "institutions": [{"id": "https://openalex.org/I121797337", "display_name": "Leiden University", "ror": "https://ror.org/027bh9e22", "country_code": "NL", "type": "education", "lineage": ["https://openalex.org/I121797337"]}], "countries": ["NL"], "is_corresponding": false, "raw_author_name": "J Wouter Jukema", "raw_affiliation_string": "Leiden University", "raw_affiliation_strings": ["Leiden University"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5081878109", "display_name": "Peter W. Macfarlane", "orcid": "https://orcid.org/0000-0002-5390-1596"}, "institutions": [{"id": "https://openalex.org/I7882870", "display_name": "University of Glasgow", "ror": "https://ror.org/00vtgdb53", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I7882870"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Peter W Macfarlane", "raw_affiliation_string": "*University of Glasgow.", "raw_affiliation_strings": ["*University of Glasgow."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5003522491", "display_name": "Chris J. Packard", "orcid": "https://orcid.org/0000-0002-2386-9927"}, "institutions": [{"id": "https://openalex.org/I7882870", "display_name": "University of Glasgow", "ror": "https://ror.org/00vtgdb53", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I7882870"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Chris J Packard", "raw_affiliation_string": "*University of Glasgow.", "raw_affiliation_strings": ["*University of Glasgow."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5031245545", "display_name": "David J. Stott", "orcid": "https://orcid.org/0000-0002-3110-7746"}, "institutions": [{"id": "https://openalex.org/I7882870", "display_name": "University of Glasgow", "ror": "https://ror.org/00vtgdb53", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I7882870"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "David J Stott", "raw_affiliation_string": "*University of Glasgow.", "raw_affiliation_strings": ["*University of Glasgow."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5036896719", "display_name": "Rudi G.J. Westendorp", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210131551", "display_name": "Netherlands Consortium for Healthy Ageing", "ror": "https://ror.org/03wnqyy64", "country_code": "NL", "type": "healthcare", "lineage": ["https://openalex.org/I4210131551"]}], "countries": ["NL"], "is_corresponding": false, "raw_author_name": "Rudi G Westendorp", "raw_affiliation_string": "Netherlands Consortium for Healthy Ageing", "raw_affiliation_strings": ["Netherlands Consortium for Healthy Ageing"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5048047555", "display_name": "James Shepherd", "orcid": null}, "institutions": [{"id": "https://openalex.org/I7882870", "display_name": "University of Glasgow", "ror": "https://ror.org/00vtgdb53", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I7882870"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "James Shepherd", "raw_affiliation_string": "*University of Glasgow.", "raw_affiliation_strings": ["*University of Glasgow."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5042299275", "display_name": "Barry R. Davis", "orcid": "https://orcid.org/0000-0002-6943-5673"}, "institutions": [], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Barry R Davis", "raw_affiliation_string": "University of Texas, School of Public Health, TX, USA.", "raw_affiliation_strings": ["University of Texas, School of Public Health, TX, USA."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5072348600", "display_name": "Sara L. Pressel", "orcid": null}, "institutions": [], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Sara L Pressel", "raw_affiliation_string": "University of Texas, School of Public Health, TX, USA.", "raw_affiliation_strings": ["University of Texas, School of Public Health, TX, USA."]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5090813987", "display_name": "Roberto Marchioli", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210110338", "display_name": "Mario Negri Sud Foundation", "ror": "https://ror.org/01qd3xc93", "country_code": "IT", "type": "nonprofit", "lineage": ["https://openalex.org/I4210110338"]}], "countries": ["IT"], "is_corresponding": false, "raw_author_name": "Roberto Marchioli", "raw_affiliation_string": "Consorzio Mario Negri Stud", "raw_affiliation_strings": ["Consorzio Mario Negri Stud"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5069819115", "display_name": "Rosa Maria Marfisi", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210110338", "display_name": "Mario Negri Sud Foundation", "ror": "https://ror.org/01qd3xc93", "country_code": "IT", "type": "nonprofit", "lineage": ["https://openalex.org/I4210110338"]}], "countries": ["IT"], "is_corresponding": false, "raw_author_name": "Rosa Maria Marfisi", "raw_affiliation_string": "Consorzio Mario Negri Stud", "raw_affiliation_strings": ["Consorzio Mario Negri Stud"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5088186128", "display_name": "Aldo P. Maggioni", "orcid": "https://orcid.org/0000-0003-2764-6779"}, "institutions": [{"id": "https://openalex.org/I4210095959", "display_name": "Associazione Nazionale Medici Cardiologi Ospedalieri", "ror": "https://ror.org/00pyc4352", "country_code": "IT", "type": "nonprofit", "lineage": ["https://openalex.org/I4210095959"]}], "countries": ["IT"], "is_corresponding": false, "raw_author_name": "Aldo P Maggioni", "raw_affiliation_string": "ANMCO Research Centre, Florence, Italy", "raw_affiliation_strings": ["ANMCO Research Centre, Florence, Italy"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5029606867", "display_name": "Luigi Tavazzi", "orcid": "https://orcid.org/0000-0003-4448-5209"}, "institutions": [{"id": "https://openalex.org/I2802469017", "display_name": "CARE Hospitals", "ror": "https://ror.org/01vka3a64", "country_code": "IN", "type": "healthcare", "lineage": ["https://openalex.org/I2802469017"]}], "countries": ["IN"], "is_corresponding": false, "raw_author_name": "Luigi Tavazzi", "raw_affiliation_string": "GVM Hospitals of Care and Research", "raw_affiliation_strings": ["GVM Hospitals of Care and Research"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5005350140", "display_name": "Gianni Tognoni", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210110338", "display_name": "Mario Negri Sud Foundation", "ror": "https://ror.org/01qd3xc93", "country_code": "IT", "type": "nonprofit", "lineage": ["https://openalex.org/I4210110338"]}], "countries": ["IT"], "is_corresponding": false, "raw_author_name": "Gianni Tognoni", "raw_affiliation_string": "Consorzio Mario Negri Stud", "raw_affiliation_strings": ["Consorzio Mario Negri Stud"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5025930905", "display_name": "John Kjekshus", "orcid": "https://orcid.org/0000-0003-4306-1244"}, "institutions": [{"id": "https://openalex.org/I1281400175", "display_name": "Oslo University Hospital", "ror": "https://ror.org/00j9c2840", "country_code": "NO", "type": "healthcare", "lineage": ["https://openalex.org/I1281400175"]}], "countries": ["NO"], "is_corresponding": false, "raw_author_name": "John Kjekshus", "raw_affiliation_string": "Department of Cardiology, Rikshospitalet University Hospital, Oslo, Norway", "raw_affiliation_strings": ["Department of Cardiology, Rikshospitalet University Hospital, Oslo, Norway"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5083340275", "display_name": "Terje R. Pedersen", "orcid": null}, "institutions": [{"id": "https://openalex.org/I1281400175", "display_name": "Oslo University Hospital", "ror": "https://ror.org/00j9c2840", "country_code": "NO", "type": "healthcare", "lineage": ["https://openalex.org/I1281400175"]}], "countries": ["NO"], "is_corresponding": false, "raw_author_name": "Terje R Pedersen", "raw_affiliation_string": "Centre for Preventative Medicine, Ulleval University Hospital, Oslo, Norway", "raw_affiliation_strings": ["Centre for Preventative Medicine, Ulleval University Hospital, Oslo, Norway"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5077547822", "display_name": "Thomas J. Cook", "orcid": "https://orcid.org/0009-0004-8785-0346"}, "institutions": [{"id": "https://openalex.org/I4210150308", "display_name": "Agile RF (United States)", "ror": "https://ror.org/049g0jw79", "country_code": "US", "type": "company", "lineage": ["https://openalex.org/I4210150308"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Thomas J Cook", "raw_affiliation_string": "Agile 1", "raw_affiliation_strings": ["Agile 1"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5068202304", "display_name": "Antonio M. Gotto", "orcid": "https://orcid.org/0000-0001-8076-6783"}, "institutions": [{"id": "https://openalex.org/I205783295", "display_name": "Cornell University", "ror": "https://ror.org/05bnh6r87", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I205783295"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "Antonio M Gotto", "raw_affiliation_string": "[Weill Medical College, Cornell University, NY, USA]", "raw_affiliation_strings": ["[Weill Medical College, Cornell University, NY, USA]"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5036001636", "display_name": "Michael Clearfield", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210124038", "display_name": "Moscow University Touro", "ror": "https://ror.org/02pppmh23", "country_code": "RU", "type": "education", "lineage": ["https://openalex.org/I4210124038"]}], "countries": ["RU"], "is_corresponding": false, "raw_author_name": "Michael B Clearfield", "raw_affiliation_string": "TOURO UNIVERSITY", "raw_affiliation_strings": ["TOURO UNIVERSITY"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5056502629", "display_name": "John R. Downs", "orcid": null}, "institutions": [{"id": "https://openalex.org/I165951966", "display_name": "The University of Texas Health Science Center at San Antonio", "ror": "https://ror.org/02f6dcw23", "country_code": "US", "type": "education", "lineage": ["https://openalex.org/I16452829", "https://openalex.org/I165951966"]}], "countries": ["US"], "is_corresponding": false, "raw_author_name": "John R Downs", "raw_affiliation_string": "Department of Medicine, University of Texas Health Science Centre, San Antonio, TX, USA", "raw_affiliation_strings": ["Department of Medicine, University of Texas Health Science Centre, San Antonio, TX, USA"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5023875153", "display_name": "Haruo Nakamura", "orcid": null}, "institutions": [{"id": "https://openalex.org/I4210159217", "display_name": "Mitsukoshi Health and Welfare Foundation", "ror": "https://ror.org/05wzgbw88", "country_code": "JP", "type": "other", "lineage": ["https://openalex.org/I4210159217"]}], "countries": ["JP"], "is_corresponding": false, "raw_author_name": "Haruo Nakamura", "raw_affiliation_string": "Mitsukoshi Health and Welfare Foundation", "raw_affiliation_strings": ["Mitsukoshi Health and Welfare Foundation"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5090955736", "display_name": "Yasuo Ohashi", "orcid": null}, "institutions": [{"id": "https://openalex.org/I74801974", "display_name": "The University of Tokyo", "ror": "https://ror.org/057zh3y96", "country_code": "JP", "type": "education", "lineage": ["https://openalex.org/I74801974"]}], "countries": ["JP"], "is_corresponding": false, "raw_author_name": "Yasuo Ohashi", "raw_affiliation_string": "Univ. of Tokyo", "raw_affiliation_strings": ["Univ. of Tokyo"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5024734422", "display_name": "Kyoichi Mizuno", "orcid": "https://orcid.org/0009-0003-9933-9513"}, "institutions": [{"id": "https://openalex.org/I80188885", "display_name": "Nippon Medical School", "ror": "https://ror.org/00krab219", "country_code": "JP", "type": "education", "lineage": ["https://openalex.org/I80188885"]}], "countries": ["JP"], "is_corresponding": false, "raw_author_name": "Kyoichi Mizuno", "raw_affiliation_string": "Nippon Medical School", "raw_affiliation_strings": ["Nippon Medical School"]}, {"author_position": "middle", "author": {"id": "https://openalex.org/A5006206326", "display_name": "Kausik K. Ray", "orcid": "https://orcid.org/0000-0003-0508-0954"}, "institutions": [{"id": "https://openalex.org/I241749", "display_name": "University of Cambridge", "ror": "https://ror.org/013meh722", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I241749"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Kausik K Ray", "raw_affiliation_string": "Univ. of Cambridge", "raw_affiliation_strings": ["Univ. of Cambridge"]}, {"author_position": "last", "author": {"id": "https://openalex.org/A5016095791", "display_name": "Ian Ford", "orcid": "https://orcid.org/0000-0001-5927-1823"}, "institutions": [{"id": "https://openalex.org/I7882870", "display_name": "University of Glasgow", "ror": "https://ror.org/00vtgdb53", "country_code": "GB", "type": "education", "lineage": ["https://openalex.org/I7882870"]}], "countries": ["GB"], "is_corresponding": false, "raw_author_name": "Ian Ford", "raw_affiliation_string": "*University of Glasgow.", "raw_affiliation_strings": ["*University of Glasgow."]}], "countries_distinct_count": 9, "institutions_distinct_count": 17, "corresponding_author_ids": [], "corresponding_institution_ids": [], "apc_list": {"value": 6830, "currency": "USD", "value_usd": 6830, "provenance": "doaj"}, "apc_paid": {"value": 6830, "currency": "USD", "value_usd": 6830, "provenance": "doaj"}, "has_fulltext": true, "fulltext_origin": "ngrams", "cited_by_count": 2031, "cited_by_percentile_year": {"min": 99.9, "max": 100.0}, "biblio": {"volume": "375", "issue": "9716", "first_page": "735", "last_page": "742"}, "is_retracted": false, "is_paratext": false, "keywords": [{"keyword": "statins trials", "score": 0.7194}, {"keyword": "incident diabetes", "score": 0.4573}, {"keyword": "meta-analysis", "score": 0.25}], "concepts": [{"id": "https://openalex.org/C71924100", "wikidata": "https://www.wikidata.org/wiki/Q11190", "display_name": "Medicine", "level": 0, "score": 0.8956113}, {"id": "https://openalex.org/C126322002", "wikidata": "https://www.wikidata.org/wiki/Q11180", "display_name": "Internal medicine", "level": 1, "score": 0.7007866}, {"id": "https://openalex.org/C2776839432", "wikidata": "https://www.wikidata.org/wiki/Q954845", "display_name": "Statin", "level": 2, "score": 0.69842064}, {"id": "https://openalex.org/C555293320", "wikidata": "https://www.wikidata.org/wiki/Q12206", "display_name": "Diabetes mellitus", "level": 2, "score": 0.6833198}, {"id": "https://openalex.org/C82789193", "wikidata": "https://www.wikidata.org/wiki/Q2142611", "display_name": "Relative risk", "level": 3, "score": 0.5471921}, {"id": "https://openalex.org/C156957248", "wikidata": "https://www.wikidata.org/wiki/Q1862216", "display_name": "Odds ratio", "level": 2, "score": 0.54087865}, {"id": "https://openalex.org/C95190672", "wikidata": "https://www.wikidata.org/wiki/Q815382", "display_name": "Meta-analysis", "level": 2, "score": 0.5246632}, {"id": "https://openalex.org/C535046627", "wikidata": "https://www.wikidata.org/wiki/Q30612", "display_name": "Clinical trial", "level": 2, "score": 0.5221627}, {"id": "https://openalex.org/C168563851", "wikidata": "https://www.wikidata.org/wiki/Q1436668", "display_name": "Randomized controlled trial", "level": 2, "score": 0.47946703}, {"id": "https://openalex.org/C203092338", "wikidata": "https://www.wikidata.org/wiki/Q1340863", "display_name": "Clinical endpoint", "level": 3, "score": 0.4757145}, {"id": "https://openalex.org/C2777180221", "wikidata": "https://www.wikidata.org/wiki/Q3025883", "display_name": "Type 2 diabetes", "level": 3, "score": 0.4493629}, {"id": "https://openalex.org/C44249647", "wikidata": "https://www.wikidata.org/wiki/Q208498", "display_name": "Confidence interval", "level": 2, "score": 0.31917673}, {"id": "https://openalex.org/C134018914", "wikidata": "https://www.wikidata.org/wiki/Q162606", "display_name": "Endocrinology", "level": 1, "score": 0.13296235}], "mesh": [{"descriptor_ui": "D000924", "descriptor_name": "Anticholesteremic Agents", "qualifier_ui": "Q000009", "qualifier_name": "adverse effects", "is_major_topic": true}, {"descriptor_ui": "D002318", "descriptor_name": "Cardiovascular Diseases", "qualifier_ui": "Q000188", "qualifier_name": "drug therapy", "is_major_topic": true}, {"descriptor_ui": "D003924", "descriptor_name": "Diabetes Mellitus, Type 2", "qualifier_ui": "Q000139", "qualifier_name": "chemically induced", "is_major_topic": true}, {"descriptor_ui": "D019161", "descriptor_name": "Hydroxymethylglutaryl-CoA Reductase Inhibitors", "qualifier_ui": "Q000009", "qualifier_name": "adverse effects", "is_major_topic": true}, {"descriptor_ui": "D017677", "descriptor_name": "Age Distribution", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D000367", "descriptor_name": "Age Factors", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D000368", "descriptor_name": "Aged", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D000924", "descriptor_name": "Anticholesteremic Agents", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D002318", "descriptor_name": "Cardiovascular Diseases", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D003924", "descriptor_name": "Diabetes Mellitus, Type 2", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D003924", "descriptor_name": "Diabetes Mellitus, Type 2", "qualifier_ui": "Q000453", "qualifier_name": "epidemiology", "is_major_topic": false}, {"descriptor_ui": "D005260", "descriptor_name": "Female", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D006801", "descriptor_name": "Humans", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D019161", "descriptor_name": "Hydroxymethylglutaryl-CoA Reductase Inhibitors", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D008297", "descriptor_name": "Male", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D008875", "descriptor_name": "Middle Aged", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D016032", "descriptor_name": "Randomized Controlled Trials as Topic", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D012307", "descriptor_name": "Risk Factors", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}, {"descriptor_ui": "D016896", "descriptor_name": "Treatment Outcome", "qualifier_ui": "", "qualifier_name": null, "is_major_topic": false}], "locations_count": 2, "locations": [{"is_oa": false, "landing_page_url": "https://doi.org/10.1016/s0140-6736(09)61965-6", "pdf_url": null, "source": {"id": "https://openalex.org/S49861241", "display_name": "The Lancet", "issn_l": "0140-6736", "issn": ["1474-547X", "0099-5355", "0140-6736"], "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/P4310320990", "host_organization_name": "Elsevier BV", "host_organization_lineage": ["https://openalex.org/P4310320990"], "host_organization_lineage_names": ["Elsevier BV"], "type": "journal"}, "license": null, "version": null, "is_accepted": false, "is_published": false}, {"is_oa": false, "landing_page_url": "https://pubmed.ncbi.nlm.nih.gov/20167359", "pdf_url": null, "source": {"id": "https://openalex.org/S4306525036", "display_name": "PubMed", "issn_l": null, "issn": null, "is_oa": false, "is_in_doaj": false, "host_organization": "https://openalex.org/I1299303238", "host_organization_name": "National Institutes of Health", "host_organization_lineage": ["https://openalex.org/I1299303238"], "host_organization_lineage_names": ["National Institutes of Health"], "type": "repository"}, "license": null, "version": null, "is_accepted": false, "is_published": false}], "best_oa_location": null, "sustainable_development_goals": [{"id": "https://metadata.un.org/sdg/3", "display_name": "Good health and well-being", "score": 0.72}], "grants": [], "referenced_works_count": 33, "referenced_works": ["https://openalex.org/W137344628", "https://openalex.org/W1500963439", "https://openalex.org/W1502524639", "https://openalex.org/W1546258268", "https://openalex.org/W1588310170", "https://openalex.org/W1944558458", "https://openalex.org/W1976358806", "https://openalex.org/W1992022057", "https://openalex.org/W1997020689", "https://openalex.org/W2028948892", "https://openalex.org/W2031185570", "https://openalex.org/W2049776203", "https://openalex.org/W2061489988", "https://openalex.org/W2086139919", "https://openalex.org/W2089629277", "https://openalex.org/W2097264002", "https://openalex.org/W2097870088", "https://openalex.org/W2098783546", "https://openalex.org/W2116045728", "https://openalex.org/W2116946402", "https://openalex.org/W2118667643", "https://openalex.org/W2125435699", "https://openalex.org/W2126452437", "https://openalex.org/W2126678006", "https://openalex.org/W2129750583", "https://openalex.org/W2130636082", "https://openalex.org/W2135631433", "https://openalex.org/W2137983259", "https://openalex.org/W2157823046", "https://openalex.org/W2160390128", "https://openalex.org/W2165796078", "https://openalex.org/W2247997571", "https://openalex.org/W2322095705"], "related_works": ["https://openalex.org/W1539974851", "https://openalex.org/W3165215133", "https://openalex.org/W2611523470", "https://openalex.org/W3210678099", "https://openalex.org/W4246615163", "https://openalex.org/W4360943417", "https://openalex.org/W4386361997", "https://openalex.org/W2417314287", "https://openalex.org/W4200125571", "https://openalex.org/W2593300661"], "ngrams_url": "https://api.openalex.org/works/W2115169717/ngrams", "abstract_inverted_index": {"Trials": [0, 53], "of": [1, 11, 13, 27, 41, 51, 63, 82, 90, 127, 143, 175, 177, 205, 221, 231, 233], "statin": [2, 37, 121], "therapy": [3, 147, 223], "have": [4], "had": [5], "conflicting": [6], "findings": [7], "on": [8], "the": [9, 47, 101, 236, 248], "risk": [10, 111, 154, 174, 230, 237, 262], "development": [12, 40, 176, 232], "diabetes": [14, 115, 139, 157, 178], "mellitus": [15], "in": [16, 77, 183, 195, 202, 217, 241, 250, 255], "patients": [17, 91, 210, 256], "given": [18], "statins.": [19, 64], "We": [20, 65, 87, 99], "aimed": [21], "to": [22, 56, 104], "establish": [23], "by": [24], "a": [25, 141, 151, 227], "meta-analysis": [26], "published": [28], "and": [29, 39, 46, 80, 109, 133, 244], "unpublished": [30], "data": [31], "whether": [32], "any": [33], "relation": [34], "exists": [35], "between": [36, 107, 169], "use": [38], "diabetes.We": [42], "searched": [43], "Medline,": [44], "Embase,": [45], "Cochrane": [48], "Central": [49], "Register": [50], "Controlled": [52], "from": [54], "1994": [55], "2009,": [57], "for": [58, 113, 155, 199, 213], "randomised": [59], "controlled": [60], "endpoint": [61], "trials": [62, 68, 89, 108, 122, 184], "included": [66], "only": [67], "with": [69, 74, 92, 116, 123, 150, 165, 179, 185, 211, 226, 247, 257], "more": [70, 83], "than": [71, 84], "1000": [72], "patients,": [73], "identical": [75], "follow-up": [76], "both": [78, 240], "groups": [79], "duration": [81], "1": [85], "year.": [86], "excluded": [88], "organ": [93], "transplants": [94], "or": [95, 259, 263], "who": [96], "needed": [97], "haemodialysis.": [98], "used": [100], "I(2)": [102], "statistic": [103], "measure": [105], "heterogeneity": [106, 167], "calculated": [110], "estimates": [112], "incident": [114, 156], "random-effect": [117], "meta-analysis.We": [118], "identified": [119], "13": [120], "91": [124], "140": [125], "participants,": [126, 187], "whom": [128], "4278": [129], "(2226": [130], "assigned": [131, 135], "statins": [132, 180, 212], "2052": [134], "control": [136], "treatment)": [137], "developed": [138], "during": [140], "mean": [142], "4": [144, 214], "years.": [145], "Statin": [146], "was": [148, 181], "associated": [149, 225], "9%": [152], "increased": [153, 229], "(odds": [158], "ratio": [159], "[OR]": [160], "1.09;": [161], "95%": [162], "CI": [163, 208], "1.02-1.17),": [164], "little": [166], "(I(2)=11%)": [168], "trials.": [170], "Meta-regression": [171], "showed": [172], "that": [173], "highest": [182], "older": [186], "but": [188, 235], "neither": [189], "baseline": [190], "body-mass": [191], "index": [192], "nor": [193], "change": [194], "LDL-cholesterol": [196], "concentrations": [197], "accounted": [198], "residual": [200], "variation": [201], "risk.": [203], "Treatment": [204], "255": [206], "(95%": [207], "150-852)": [209], "years": [215], "resulted": [216], "one": [218], "extra": [219], "case": [220], "diabetes.Statin": [222], "is": [224, 238], "slightly": [228], "diabetes,": [234], "low": [239], "absolute": [242], "terms": [243], "when": [245], "compared": [246], "reduction": [249], "coronary": [251], "events.": [252], "Clinical": [253], "practice": [254], "moderate": [258], "high": [260], "cardiovascular": [261, 265], "existing": [264], "disease": [266], "should": [267], "not": [268], "change.None.": [269]}, "cited_by_api_url": "https://api.openalex.org/works?filter=cites:W2115169717", "counts_by_year": [{"year": 2023, "cited_by_count": 87}, {"year": 2022, "cited_by_count": 122}, {"year": 2021, "cited_by_count": 93}, {"year": 2020, "cited_by_count": 125}, {"year": 2019, "cited_by_count": 166}, {"year": 2018, "cited_by_count": 164}, {"year": 2017, "cited_by_count": 157}, {"year": 2016, "cited_by_count": 221}, {"year": 2015, "cited_by_count": 198}, {"year": 2014, "cited_by_count": 200}, {"year": 2013, "cited_by_count": 175}, {"year": 2012, "cited_by_count": 134}], "updated_date": "2023-11-29T15:25:34.068916", "created_date": "2016-06-24"} \ No newline at end of file From 073f320c6a2735bda5d51e7bd7766f01f791651d Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Mon, 22 Apr 2024 11:32:31 +0200 Subject: [PATCH 16/97] Added module containing all the dependencies, useful for spark deploy on k8. --- .../eu/dnetlib/pace/model/SparkModel.scala | 4 +- dhp-shade-package/pom.xml | 169 ++++++++++++++++++ .../dhp/oa/dedup/SparkCreateMergeRels.java | 4 +- .../dhp/oa/dedup/SparkPropagateRelation.java | 1 - pom.xml | 1 + 5 files changed, 174 insertions(+), 5 deletions(-) create mode 100644 dhp-shade-package/pom.xml diff --git a/dhp-pace-core/src/main/java/eu/dnetlib/pace/model/SparkModel.scala b/dhp-pace-core/src/main/java/eu/dnetlib/pace/model/SparkModel.scala index aa04188da..e6a1c4ccc 100644 --- a/dhp-pace-core/src/main/java/eu/dnetlib/pace/model/SparkModel.scala +++ b/dhp-pace-core/src/main/java/eu/dnetlib/pace/model/SparkModel.scala @@ -3,7 +3,7 @@ package eu.dnetlib.pace.model import com.jayway.jsonpath.{Configuration, JsonPath} import eu.dnetlib.pace.common.AbstractPaceFunctions import eu.dnetlib.pace.config.{DedupConfig, Type} -import eu.dnetlib.pace.util.MapDocumentUtil +import eu.dnetlib.pace.util.{MapDocumentUtil, SparkCompatUtils} import org.apache.commons.lang3.StringUtils import org.apache.spark.sql.catalyst.encoders.RowEncoder import org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema @@ -52,7 +52,7 @@ case class SparkModel(conf: DedupConfig) { val orderingFieldPosition: Int = schema.fieldIndex(orderingFieldName) val parseJsonDataset: (Dataset[String] => Dataset[Row]) = df => { - df.map(r => rowFromJson(r))(RowEncoder(schema)) + df.map(r => rowFromJson(r))(SparkCompatUtils.encoderFor(schema)) } def rowFromJson(json: String): Row = { diff --git a/dhp-shade-package/pom.xml b/dhp-shade-package/pom.xml new file mode 100644 index 000000000..128a57116 --- /dev/null +++ b/dhp-shade-package/pom.xml @@ -0,0 +1,169 @@ + + + 4.0.0 + + eu.dnetlib.dhp + dhp + 1.2.5-SNAPSHOT + ../pom.xml + + + + dhp-shade-package + jar + + + + DHPSite + ${dhp.site.stage.path}/dhp-common + + + + This module create a jar of all module dependencies + + + + + + eu.dnetlib.dhp + dhp-actionmanager + ${project.version} + + + eu.dnetlib.dhp + dhp-aggregation + ${project.version} + + + eu.dnetlib.dhp + dhp-blacklist + ${project.version} + + + eu.dnetlib.dhp + dhp-broker-events + ${project.version} + + + eu.dnetlib.dhp + dhp-dedup-openaire + ${project.version} + + + eu.dnetlib.dhp + dhp-enrichment + ${project.version} + + + eu.dnetlib.dhp + dhp-graph-mapper + ${project.version} + + + eu.dnetlib.dhp + dhp-graph-provision + ${project.version} + + + eu.dnetlib.dhp + dhp-impact-indicators + ${project.version} + + + eu.dnetlib.dhp + dhp-stats-actionsets + ${project.version} + + + eu.dnetlib.dhp + dhp-stats-hist-snaps + ${project.version} + + + eu.dnetlib.dhp + dhp-stats-monitor-irish + ${project.version} + + + eu.dnetlib.dhp + dhp-stats-promote + ${project.version} + + + eu.dnetlib.dhp + dhp-stats-update + ${project.version} + + + eu.dnetlib.dhp + dhp-swh + ${project.version} + + + eu.dnetlib.dhp + dhp-usage-raw-data-update + ${project.version} + + + eu.dnetlib.dhp + dhp-usage-stats-build + ${project.version} + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + + + eu.dnetlib.dhp.oa.dedup.SparkCreateSimRels + + + + + META-INF/cxf/bus-extensions.txt + + + + + *:* + + META-INF/maven/** + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + com + repackaged.com.google.common + + com.google.common.** + + + + + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateMergeRels.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateMergeRels.java index 59626c141..d48351c48 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateMergeRels.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateMergeRels.java @@ -42,6 +42,7 @@ import eu.dnetlib.dhp.utils.ISLookupClientFactory; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; import eu.dnetlib.pace.config.DedupConfig; +import eu.dnetlib.pace.util.SparkCompatUtils; import scala.Tuple3; import scala.collection.JavaConversions; @@ -148,8 +149,7 @@ public class SparkCreateMergeRels extends AbstractSparkAction { Dataset pivotHistory = spark .createDataset( Collections.emptyList(), - RowEncoder - .apply(StructType.fromDDL("id STRING, lastUsage STRING"))); + SparkCompatUtils.encoderFor(StructType.fromDDL("id STRING, lastUsage STRING"))); if (StringUtils.isNotBlank(pivotHistoryDatabase)) { pivotHistory = spark diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java index c64fbe4a4..c7efce4d7 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java @@ -8,7 +8,6 @@ import org.apache.spark.SparkConf; import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.api.java.function.ReduceFunction; import org.apache.spark.sql.*; -import org.apache.spark.sql.catalyst.encoders.RowEncoder; import org.apache.spark.sql.types.StructType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/pom.xml b/pom.xml index 06e4ba9d4..fc68a666d 100644 --- a/pom.xml +++ b/pom.xml @@ -23,6 +23,7 @@ dhp-pace-core dhp-common dhp-workflows + dhp-shade-package From 24a83fc24f438e6dd928495864f4d32897133612 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Mon, 22 Apr 2024 11:39:44 +0200 Subject: [PATCH 17/97] avoid NPEs in common Oaf merge utilities --- .../java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils.java index c95c31c51..62b9731dc 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils.java @@ -270,8 +270,9 @@ public class MergeUtils { } HashMap values = new HashMap<>(); - left.forEach(kv -> values.put(kv.getKey(), kv)); - right.forEach(kv -> values.putIfAbsent(kv.getKey(), kv)); + + Optional.ofNullable(left).ifPresent(l -> l.forEach(kv -> values.put(kv.getKey(), kv))); + Optional.ofNullable(right).ifPresent(r -> r.forEach(kv -> values.putIfAbsent(kv.getKey(), kv))); return new ArrayList<>(values.values()); } From 7de114bda03bcf7b1e2a77e1b185bf61fd126995 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Mon, 22 Apr 2024 13:52:50 +0200 Subject: [PATCH 18/97] [WebCrawl] addressing comments from PR --- .../CreateActionSetFromWebEntries.java | 354 +++++++------- .../actionmanager/webcrawl/CreateASTest.java | 436 +++++++++--------- 2 files changed, 376 insertions(+), 414 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateActionSetFromWebEntries.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateActionSetFromWebEntries.java index 5a0be98d3..4035eb33a 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateActionSetFromWebEntries.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateActionSetFromWebEntries.java @@ -35,238 +35,198 @@ import scala.Tuple2; * @Date 18/04/24 */ public class CreateActionSetFromWebEntries implements Serializable { - private static final Logger log = LoggerFactory.getLogger(CreateActionSetFromWebEntries.class); - private static final String DOI_PREFIX = "50|doi_________::"; + private static final Logger log = LoggerFactory.getLogger(CreateActionSetFromWebEntries.class); + private static final String DOI_PREFIX = "50|doi_________::"; - private static final String ROR_PREFIX = "20|ror_________::"; + private static final String ROR_PREFIX = "20|ror_________::"; - private static final String PMID_PREFIX = "50|pmid________::"; + private static final String PMID_PREFIX = "50|pmid________::"; - private static final String PMCID_PREFIX = "50|pmc_________::"; - private static final String WEB_CRAWL_ID = "10|openaire____::fb98a192f6a055ba495ef414c330834b"; - private static final String WEB_CRAWL_NAME = "Web Crawl"; - public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final String PMCID_PREFIX = "50|pmc_________::"; + private static final String WEB_CRAWL_ID = "10|openaire____::fb98a192f6a055ba495ef414c330834b"; + private static final String WEB_CRAWL_NAME = "Web Crawl"; + public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - public static void main(String[] args) throws Exception { - String jsonConfiguration = IOUtils - .toString( - CreateActionSetFromWebEntries.class - .getResourceAsStream( - "/eu/dnetlib/dhp/actionmanager/webcrawl/as_parameters.json")); + public static void main(String[] args) throws Exception { + String jsonConfiguration = IOUtils + .toString( + CreateActionSetFromWebEntries.class + .getResourceAsStream( + "/eu/dnetlib/dhp/actionmanager/webcrawl/as_parameters.json")); - final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); - parser.parseArgument(args); + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + parser.parseArgument(args); - Boolean isSparkSessionManaged = Optional - .ofNullable(parser.get("isSparkSessionManaged")) - .map(Boolean::valueOf) - .orElse(Boolean.TRUE); + Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); - log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); - final String inputPath = parser.get("sourcePath"); - log.info("inputPath: {}", inputPath); + final String inputPath = parser.get("sourcePath"); + log.info("inputPath: {}", inputPath); - final String outputPath = parser.get("outputPath"); - log.info("outputPath: {}", outputPath); + final String outputPath = parser.get("outputPath"); + log.info("outputPath: {}", outputPath); - SparkConf conf = new SparkConf(); + SparkConf conf = new SparkConf(); - runWithSparkSession( - conf, - isSparkSessionManaged, - spark -> { + runWithSparkSession( + conf, + isSparkSessionManaged, + spark -> { - createActionSet(spark, inputPath, outputPath + "actionSet"); - createPlainRelations(spark, inputPath, outputPath + "relations"); - }); - } + createActionSet(spark, inputPath, outputPath ); - private static void createPlainRelations(SparkSession spark, String inputPath, String outputPath) { - final Dataset dataset = readWebCrawl(spark, inputPath); + }); + } - dataset.flatMap((FlatMapFunction>) row -> { - List> ret = new ArrayList<>(); + public static void createActionSet(SparkSession spark, String inputPath, + String outputPath) { - final String ror = row.getAs("ror"); - ret.addAll(createAffiliationRelationPairDOI(row.getAs("publication_year"), row.getAs("doi"), ror)); - ret.addAll(createAffiliationRelationPairPMID(row.getAs("publication_year"), row.getAs("pmid"), ror)); - ret.addAll(createAffiliationRelationPairPMCID(row.getAs("publication_year"), row.getAs("pmcid"), ror)); + final Dataset dataset = readWebCrawl(spark, inputPath) + .filter("publication_year <= 2020 or country_code=='IE'") + .drop("publication_year"); - return ret - .iterator(); - }, Encoders.tuple(Encoders.STRING(), Encoders.bean(Relation.class))) - .write() - .mode(SaveMode.Overwrite) - .option("compression", "gzip") - .json(outputPath); - } + dataset.flatMap((FlatMapFunction) row -> { + List ret = new ArrayList<>(); + final String ror = ROR_PREFIX + + IdentifierFactory.md5(PidCleaner.normalizePidValue("ROR", row.getAs("ror"))); + ret.addAll(createAffiliationRelationPairDOI(row.getAs("doi"), ror)); + ret.addAll(createAffiliationRelationPairPMID(row.getAs("pmid"), ror)); + ret.addAll(createAffiliationRelationPairPMCID(row.getAs("pmcid"), ror)); - private static Collection> createAffiliationRelationPairPMCID( - String publication_year, String pmcid, String ror) { - if (pmcid == null) - return new ArrayList<>(); + return ret + .iterator(); + }, Encoders.bean(Relation.class)) + .toJavaRDD() + .map(p -> new AtomicAction(p.getClass(), p)) + .mapToPair( + aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()), + new Text(OBJECT_MAPPER.writeValueAsString(aa)))) + .saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, GzipCodec.class); - return createAffiliatioRelationPair("PMC" + pmcid, ror) - .stream() - .map(r -> new Tuple2(publication_year, r)) - .collect(Collectors.toList()); - } + } - private static Collection> createAffiliationRelationPairPMID( - String publication_year, String pmid, String ror) { - if (pmid == null) - return new ArrayList<>(); + private static Dataset readWebCrawl(SparkSession spark, String inputPath) { + StructType webInfo = StructType + .fromDDL( + "`id` STRING , `doi` STRING, `ids` STRUCT<`pmid` :STRING, `pmcid`: STRING >, `publication_year` STRING, " + + + "`authorships` ARRAY>>>"); - return createAffiliatioRelationPair(pmid, ror) - .stream() - .map(r -> new Tuple2(publication_year, r)) - .collect(Collectors.toList()); - } + return spark + .read() + .schema(webInfo) + .json(inputPath) + .withColumn( + "authors", functions + .explode( + functions.col("authorships"))) + .selectExpr("id", "doi", "ids", "publication_year", "authors.institutions as institutions") + .withColumn( + "institution", functions + .explode( + functions.col("institutions"))) + .selectExpr( + "id", "doi", "ids.pmcid as pmcid", "ids.pmid as pmid", "institution.ror as ror", + "institution.country_code as country_code", "publication_year") + .distinct(); - private static Collection> createAffiliationRelationPairDOI( - String publication_year, String doi, String ror) { - if (doi == null) - return new ArrayList<>(); + } - return createAffiliatioRelationPair(doi, ror) - .stream() - .map(r -> new Tuple2(publication_year, r)) - .collect(Collectors.toList()); - } + private static List createAffiliationRelationPairPMCID(String pmcid, String ror) { + if (pmcid == null) + return new ArrayList<>(); - public static void createActionSet(SparkSession spark, String inputPath, - String outputPath) { + return createAffiliatioRelationPair( + PMCID_PREFIX + + IdentifierFactory + .md5(PidCleaner.normalizePidValue(PidType.pmc.toString(), removeResolver("PMC" , pmcid))), + ror); + } - final Dataset dataset = readWebCrawl(spark, inputPath) - .filter("publication_year <= 2020 or country_code=='IE'") - .drop("publication_year"); + private static List createAffiliationRelationPairPMID(String pmid, String ror) { + if (pmid == null) + return new ArrayList<>(); - dataset.flatMap((FlatMapFunction) row -> { - List ret = new ArrayList<>(); - final String ror = ROR_PREFIX - + IdentifierFactory.md5(PidCleaner.normalizePidValue("ROR", row.getAs("ror"))); - ret.addAll(createAffiliationRelationPairDOI(row.getAs("doi"), ror)); - ret.addAll(createAffiliationRelationPairPMID(row.getAs("pmid"), ror)); - ret.addAll(createAffiliationRelationPairPMCID(row.getAs("pmcid"), ror)); + return createAffiliatioRelationPair( + PMID_PREFIX + + IdentifierFactory + .md5(PidCleaner.normalizePidValue(PidType.pmid.toString(), removeResolver("PMID", pmid))), + ror); + } - return ret - .iterator(); - }, Encoders.bean(Relation.class)) - .toJavaRDD() - .map(p -> new AtomicAction(p.getClass(), p)) - .mapToPair( - aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()), - new Text(OBJECT_MAPPER.writeValueAsString(aa)))) - .saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, GzipCodec.class); + private static String removeResolver(String pidType, String pid) { + switch (pidType){ + case "PMID": + return pid.substring(33); + case "PMC": + return "PMC" + pid.substring(43); + case "DOI": + return pid.substring(16); + } - } + throw new RuntimeException(); - private static Dataset readWebCrawl(SparkSession spark, String inputPath) { - StructType webInfo = StructType - .fromDDL( - "`id` STRING , `doi` STRING, `ids` STRUCT<`pmid` :STRING, `pmcid`: STRING >, `publication_year` STRING, " - + - "`authorships` ARRAY>>>"); + } - return spark - .read() - .schema(webInfo) - .json(inputPath) - .withColumn( - "authors", functions - .explode( - functions.col("authorships"))) - .selectExpr("id", "doi", "ids", "publication_year", "authors.institutions as institutions") - .withColumn( - "institution", functions - .explode( - functions.col("institutions"))) - .selectExpr( - "id", "doi", "ids.pmcid as pmcid", "ids.pmid as pmid", "institution.ror as ror", - "institution.country_code as country_code", "publication_year") - // .where("country_code == 'IE'") - .distinct(); + private static List createAffiliationRelationPairDOI(String doi, String ror) { + if (doi == null) + return new ArrayList<>(); - } + return createAffiliatioRelationPair( + DOI_PREFIX + + IdentifierFactory + .md5(PidCleaner.normalizePidValue(PidType.doi.toString(), removeResolver("DOI" ,doi))), + ror); - private static List createAffiliationRelationPairPMCID(String pmcid, String ror) { - if (pmcid == null) - return new ArrayList<>(); + } - return createAffiliatioRelationPair( - PMCID_PREFIX - + IdentifierFactory - .md5(PidCleaner.normalizePidValue(PidType.pmc.toString(), "PMC" + pmcid.substring(43))), - ror); - } + private static List createAffiliatioRelationPair(String resultId, String orgId) { + ArrayList newRelations = new ArrayList(); - private static List createAffiliationRelationPairPMID(String pmid, String ror) { - if (pmid == null) - return new ArrayList<>(); + newRelations + .add( + OafMapperUtils + .getRelation( + orgId, resultId, ModelConstants.RESULT_ORGANIZATION, ModelConstants.AFFILIATION, + ModelConstants.IS_AUTHOR_INSTITUTION_OF, + Arrays + .asList( + OafMapperUtils.keyValue(WEB_CRAWL_ID, WEB_CRAWL_NAME)), + OafMapperUtils + .dataInfo( + false, null, false, false, + OafMapperUtils + .qualifier( + "sysimport:crasswalk:webcrawl", "Imported from Webcrawl", + ModelConstants.DNET_PROVENANCE_ACTIONS, ModelConstants.DNET_PROVENANCE_ACTIONS), + "0.9"), + null)); - return createAffiliatioRelationPair( - PMID_PREFIX - + IdentifierFactory - .md5(PidCleaner.normalizePidValue(PidType.pmid.toString(), pmid.substring(33))), - ror); - } + newRelations + .add( + OafMapperUtils + .getRelation( + resultId, orgId, ModelConstants.RESULT_ORGANIZATION, ModelConstants.AFFILIATION, + ModelConstants.HAS_AUTHOR_INSTITUTION, + Arrays + .asList( + OafMapperUtils.keyValue(WEB_CRAWL_ID, WEB_CRAWL_NAME)), + OafMapperUtils + .dataInfo( + false, null, false, false, + OafMapperUtils + .qualifier( + "sysimport:crasswalk:webcrawl", "Imported from Webcrawl", + ModelConstants.DNET_PROVENANCE_ACTIONS, ModelConstants.DNET_PROVENANCE_ACTIONS), + "0.9"), + null)); - private static List createAffiliationRelationPairDOI(String doi, String ror) { - if (doi == null) - return new ArrayList<>(); + return newRelations; - return createAffiliatioRelationPair( - DOI_PREFIX - + IdentifierFactory - .md5(PidCleaner.normalizePidValue(PidType.doi.toString(), doi.substring(16))), - ror); - - } - - private static List createAffiliatioRelationPair(String resultId, String orgId) { - ArrayList newRelations = new ArrayList(); - - newRelations - .add( - OafMapperUtils - .getRelation( - orgId, resultId, ModelConstants.RESULT_ORGANIZATION, ModelConstants.AFFILIATION, - ModelConstants.IS_AUTHOR_INSTITUTION_OF, - Arrays - .asList( - OafMapperUtils.keyValue(WEB_CRAWL_ID, WEB_CRAWL_NAME)), - OafMapperUtils - .dataInfo( - false, null, false, false, - OafMapperUtils - .qualifier( - "sysimport:crasswalk:webcrawl", "Imported from Webcrawl", - ModelConstants.DNET_PROVENANCE_ACTIONS, ModelConstants.DNET_PROVENANCE_ACTIONS), - "0.9"), - null)); - - newRelations - .add( - OafMapperUtils - .getRelation( - resultId, orgId, ModelConstants.RESULT_ORGANIZATION, ModelConstants.AFFILIATION, - ModelConstants.HAS_AUTHOR_INSTITUTION, - Arrays - .asList( - OafMapperUtils.keyValue(WEB_CRAWL_ID, WEB_CRAWL_NAME)), - OafMapperUtils - .dataInfo( - false, null, false, false, - OafMapperUtils - .qualifier( - "sysimport:crasswalk:webcrawl", "Imported from Webcrawl", - ModelConstants.DNET_PROVENANCE_ACTIONS, ModelConstants.DNET_PROVENANCE_ACTIONS), - "0.9"), - null)); - - return newRelations; - - } + } } diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateASTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateASTest.java index a1cd69dcc..402f07d4d 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateASTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateASTest.java @@ -1,13 +1,12 @@ + package eu.dnetlib.dhp.actionmanager.webcrawl; + import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; -import eu.dnetlib.dhp.schema.oaf.utils.PidCleaner; -import eu.dnetlib.dhp.schema.oaf.utils.PidType; import org.apache.commons.io.FileUtils; import org.apache.hadoop.io.Text; import org.apache.spark.SparkConf; @@ -25,261 +24,264 @@ import com.fasterxml.jackson.databind.ObjectMapper; import eu.dnetlib.dhp.schema.action.AtomicAction; import eu.dnetlib.dhp.schema.oaf.Relation; +import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; +import eu.dnetlib.dhp.schema.oaf.utils.PidCleaner; +import eu.dnetlib.dhp.schema.oaf.utils.PidType; /** * @author miriam.baglioni * @Date 22/04/24 */ public class CreateASTest { - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private static SparkSession spark; + private static SparkSession spark; - private static Path workingDir; - private static final Logger log = LoggerFactory - .getLogger(CreateASTest.class); + private static Path workingDir; + private static final Logger log = LoggerFactory + .getLogger(CreateASTest.class); - @BeforeAll - public static void beforeAll() throws IOException { - workingDir = Files - .createTempDirectory(CreateASTest.class.getSimpleName()); - log.info("using work dir {}", workingDir); + @BeforeAll + public static void beforeAll() throws IOException { + workingDir = Files + .createTempDirectory(CreateASTest.class.getSimpleName()); + log.info("using work dir {}", workingDir); - SparkConf conf = new SparkConf(); - conf.setAppName(CreateASTest.class.getSimpleName()); + SparkConf conf = new SparkConf(); + conf.setAppName(CreateASTest.class.getSimpleName()); - conf.setMaster("local[*]"); - conf.set("spark.driver.host", "localhost"); - conf.set("hive.metastore.local", "true"); - conf.set("spark.ui.enabled", "false"); - conf.set("spark.sql.warehouse.dir", workingDir.toString()); - conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString()); + conf.setMaster("local[*]"); + conf.set("spark.driver.host", "localhost"); + conf.set("hive.metastore.local", "true"); + conf.set("spark.ui.enabled", "false"); + conf.set("spark.sql.warehouse.dir", workingDir.toString()); + conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString()); - spark = SparkSession - .builder() - .appName(CreateASTest.class.getSimpleName()) - .config(conf) - .getOrCreate(); - } + spark = SparkSession + .builder() + .appName(CreateASTest.class.getSimpleName()) + .config(conf) + .getOrCreate(); + } - @AfterAll - public static void afterAll() throws IOException { - FileUtils.deleteDirectory(workingDir.toFile()); - spark.stop(); - } - @Test - void testNumberofRelations() throws Exception { + @AfterAll + public static void afterAll() throws IOException { + FileUtils.deleteDirectory(workingDir.toFile()); + spark.stop(); + } - String inputPath = getClass() - .getResource( - "/eu/dnetlib/dhp/actionmanager/webcrawl/") - .getPath(); + @Test + void testNumberofRelations() throws Exception { - CreateActionSetFromWebEntries - .main( - new String[] { - "-isSparkSessionManaged", - Boolean.FALSE.toString(), - "-sourcePath", - inputPath, - "-outputPath", - workingDir.toString() + "/actionSet1" - }); + String inputPath = getClass() + .getResource( + "/eu/dnetlib/dhp/actionmanager/webcrawl/") + .getPath(); - final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); + CreateActionSetFromWebEntries + .main( + new String[] { + "-isSparkSessionManaged", + Boolean.FALSE.toString(), + "-sourcePath", + inputPath, + "-outputPath", + workingDir.toString() + "/actionSet1" + }); - JavaRDD tmp = sc - .sequenceFile(workingDir.toString() + "/actionSet1", Text.class, Text.class) - .map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class)) - .map(aa -> ((Relation) aa.getPayload())); + final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); - Assertions.assertEquals(64, tmp.count()); + JavaRDD tmp = sc + .sequenceFile(workingDir.toString() + "/actionSet1", Text.class, Text.class) + .map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class)) + .map(aa -> ((Relation) aa.getPayload())); - } - @Test - void testRelations() throws Exception { + Assertions.assertEquals(64, tmp.count()); + + } + + @Test + void testRelations() throws Exception { // , "doi":"https://doi.org/10.1126/science.1188021", "pmid":"https://pubmed.ncbi.nlm.nih.gov/20448178", https://www.ncbi.nlm.nih.gov/pmc/articles/5100745 - String inputPath = getClass() - .getResource( - "/eu/dnetlib/dhp/actionmanager/webcrawl/") - .getPath(); + String inputPath = getClass() + .getResource( + "/eu/dnetlib/dhp/actionmanager/webcrawl/") + .getPath(); - CreateActionSetFromWebEntries - .main( - new String[] { - "-isSparkSessionManaged", - Boolean.FALSE.toString(), - "-sourcePath", - inputPath, - "-outputPath", - workingDir.toString() + "/actionSet1" - }); + CreateActionSetFromWebEntries + .main( + new String[] { + "-isSparkSessionManaged", + Boolean.FALSE.toString(), + "-sourcePath", + inputPath, + "-outputPath", + workingDir.toString() + "/actionSet1" + }); - final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); + final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); - JavaRDD tmp = sc - .sequenceFile(workingDir.toString() + "/actionSet1", Text.class, Text.class) - .map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class)) - .map(aa -> ((Relation) aa.getPayload())); + JavaRDD tmp = sc + .sequenceFile(workingDir.toString() + "/actionSet1", Text.class, Text.class) + .map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class)) + .map(aa -> ((Relation) aa.getPayload())); - tmp.foreach(r -> System.out.println(new ObjectMapper().writeValueAsString(r))); + tmp.foreach(r -> System.out.println(new ObjectMapper().writeValueAsString(r))); - Assertions - .assertEquals( - 1, tmp - .filter( - r -> r - .getSource() - .equals( - "50|doi_________::" + IdentifierFactory - .md5( - PidCleaner - .normalizePidValue(PidType.doi.toString(), "10.1098/rstl.1684.0023")))) - .count()); + Assertions + .assertEquals( + 1, tmp + .filter( + r -> r + .getSource() + .equals( + "50|doi_________::" + IdentifierFactory + .md5( + PidCleaner + .normalizePidValue(PidType.doi.toString(), "10.1098/rstl.1684.0023")))) + .count()); - Assertions - .assertEquals( - 1, tmp - .filter( - r -> r - .getTarget() - .equals( - "50|doi_________::" + IdentifierFactory - .md5( - PidCleaner - .normalizePidValue(PidType.doi.toString(), "10.1098/rstl.1684.0023")))) - .count()); + Assertions + .assertEquals( + 1, tmp + .filter( + r -> r + .getTarget() + .equals( + "50|doi_________::" + IdentifierFactory + .md5( + PidCleaner + .normalizePidValue(PidType.doi.toString(), "10.1098/rstl.1684.0023")))) + .count()); - Assertions - .assertEquals( - 1, tmp - .filter( - r -> r - .getSource() - .equals( - "20|ror_________::" + IdentifierFactory - .md5( - PidCleaner - .normalizePidValue("ROR", "https://ror.org/03argrj65")))) - .count()); + Assertions + .assertEquals( + 1, tmp + .filter( + r -> r + .getSource() + .equals( + "20|ror_________::" + IdentifierFactory + .md5( + PidCleaner + .normalizePidValue("ROR", "https://ror.org/03argrj65")))) + .count()); - Assertions - .assertEquals( - 1, tmp - .filter( - r -> r - .getTarget() - .equals( - "20|ror_________::" + IdentifierFactory - .md5( - PidCleaner - .normalizePidValue("ROR", "https://ror.org/03argrj65")))) - .count()); + Assertions + .assertEquals( + 1, tmp + .filter( + r -> r + .getTarget() + .equals( + "20|ror_________::" + IdentifierFactory + .md5( + PidCleaner + .normalizePidValue("ROR", "https://ror.org/03argrj65")))) + .count()); - Assertions - .assertEquals( - 5, tmp - .filter( - r -> r - .getSource() - .equals( - "20|ror_________::" + IdentifierFactory - .md5( - PidCleaner - .normalizePidValue("ROR", "https://ror.org/03265fv13")))) - .count()); + Assertions + .assertEquals( + 5, tmp + .filter( + r -> r + .getSource() + .equals( + "20|ror_________::" + IdentifierFactory + .md5( + PidCleaner + .normalizePidValue("ROR", "https://ror.org/03265fv13")))) + .count()); - Assertions - .assertEquals( - 5, tmp - .filter( - r -> r - .getTarget() - .equals( - "20|ror_________::" + IdentifierFactory - .md5( - PidCleaner - .normalizePidValue("ROR", "https://ror.org/03265fv13")))) - .count()); + Assertions + .assertEquals( + 5, tmp + .filter( + r -> r + .getTarget() + .equals( + "20|ror_________::" + IdentifierFactory + .md5( + PidCleaner + .normalizePidValue("ROR", "https://ror.org/03265fv13")))) + .count()); - Assertions - .assertEquals( - 2, tmp - .filter( - r -> r - .getTarget() - .equals( - "20|ror_________::" + IdentifierFactory - .md5( - PidCleaner - .normalizePidValue(PidType.doi.toString(), "https://ror.org/03265fv13"))) - && r.getSource().startsWith("50|doi")) - .count()); + Assertions + .assertEquals( + 2, tmp + .filter( + r -> r + .getTarget() + .equals( + "20|ror_________::" + IdentifierFactory + .md5( + PidCleaner + .normalizePidValue(PidType.doi.toString(), "https://ror.org/03265fv13"))) + && r.getSource().startsWith("50|doi")) + .count()); - Assertions - .assertEquals( - 2, tmp - .filter( - r -> r - .getTarget() - .equals( - "20|ror_________::" + IdentifierFactory - .md5( - PidCleaner - .normalizePidValue(PidType.doi.toString(), "https://ror.org/03265fv13"))) - && r.getSource().startsWith("50|pmid")) - .count()); + Assertions + .assertEquals( + 2, tmp + .filter( + r -> r + .getTarget() + .equals( + "20|ror_________::" + IdentifierFactory + .md5( + PidCleaner + .normalizePidValue(PidType.doi.toString(), "https://ror.org/03265fv13"))) + && r.getSource().startsWith("50|pmid")) + .count()); - Assertions - .assertEquals( - 1, tmp - .filter( - r -> r - .getTarget() - .equals( - "20|ror_________::" + IdentifierFactory - .md5( - PidCleaner - .normalizePidValue(PidType.doi.toString(), "https://ror.org/03265fv13"))) - && r.getSource().startsWith("50|pmc")) - .count()); - } + Assertions + .assertEquals( + 1, tmp + .filter( + r -> r + .getTarget() + .equals( + "20|ror_________::" + IdentifierFactory + .md5( + PidCleaner + .normalizePidValue(PidType.doi.toString(), "https://ror.org/03265fv13"))) + && r.getSource().startsWith("50|pmc")) + .count()); + } - @Test - void testRelationsCollectedFrom() throws Exception { + @Test + void testRelationsCollectedFrom() throws Exception { - String inputPath = getClass() - .getResource( - "/eu/dnetlib/dhp/actionmanager/webcrawl") - .getPath(); + String inputPath = getClass() + .getResource( + "/eu/dnetlib/dhp/actionmanager/webcrawl") + .getPath(); - CreateActionSetFromWebEntries - .main( - new String[] { - "-isSparkSessionManaged", - Boolean.FALSE.toString(), - "-sourcePath", - inputPath, - "-outputPath", - workingDir.toString() + "/actionSet1" - }); + CreateActionSetFromWebEntries + .main( + new String[] { + "-isSparkSessionManaged", + Boolean.FALSE.toString(), + "-sourcePath", + inputPath, + "-outputPath", + workingDir.toString() + "/actionSet1" + }); - final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); + final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); - JavaRDD tmp = sc - .sequenceFile(workingDir.toString() + "/actionSet1", Text.class, Text.class) - .map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class)) - .map(aa -> ((Relation) aa.getPayload())); - - tmp.foreach(r -> { - assertEquals("Web Crawl", r.getCollectedfrom().get(0).getValue()); - assertEquals("10|openaire____::fb98a192f6a055ba495ef414c330834b", r.getCollectedfrom().get(0).getKey()); - }); - - } + JavaRDD tmp = sc + .sequenceFile(workingDir.toString() + "/actionSet1", Text.class, Text.class) + .map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class)) + .map(aa -> ((Relation) aa.getPayload())); + tmp.foreach(r -> { + assertEquals("Web Crawl", r.getCollectedfrom().get(0).getValue()); + assertEquals("10|openaire____::fb98a192f6a055ba495ef414c330834b", r.getCollectedfrom().get(0).getKey()); + }); + } } From 6189879643c6023de280231b1c6e737a5e849e4a Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Tue, 23 Apr 2024 11:09:18 +0200 Subject: [PATCH 19/97] [NOAMI] removed entry for Irish Research eLibray (IReL) Care Board from the list of funders. --- .../eu/dnetlib/dhp/collection/crossref/irish_funder.json | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/crossref/irish_funder.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/crossref/irish_funder.json index 598fe2ba5..f0275e06b 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/crossref/irish_funder.json +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/crossref/irish_funder.json @@ -271,12 +271,6 @@ "name": "An Roinn Sl\u00e1inte", "synonym": [] }, - { - "id": "100018998", - "uri": "http://dx.doi.org/10.13039/100018998", - "name": "Irish Research eLibrary", - "synonym": [] - }, { "id": "100019428", "uri": "http://dx.doi.org/10.13039/100019428", From 93dd9cc639fa52cb36d617740cb097b42b805186 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 23 Apr 2024 11:28:00 +0200 Subject: [PATCH 20/97] code formatting --- .../webcrawl/CreateActionSetFromWebEntries.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateActionSetFromWebEntries.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateActionSetFromWebEntries.java index 4035eb33a..eb370e981 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateActionSetFromWebEntries.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateActionSetFromWebEntries.java @@ -77,7 +77,7 @@ public class CreateActionSetFromWebEntries implements Serializable { isSparkSessionManaged, spark -> { - createActionSet(spark, inputPath, outputPath ); + createActionSet(spark, inputPath, outputPath); }); } @@ -143,7 +143,7 @@ public class CreateActionSetFromWebEntries implements Serializable { return createAffiliatioRelationPair( PMCID_PREFIX + IdentifierFactory - .md5(PidCleaner.normalizePidValue(PidType.pmc.toString(), removeResolver("PMC" , pmcid))), + .md5(PidCleaner.normalizePidValue(PidType.pmc.toString(), removeResolver("PMC", pmcid))), ror); } @@ -159,7 +159,7 @@ public class CreateActionSetFromWebEntries implements Serializable { } private static String removeResolver(String pidType, String pid) { - switch (pidType){ + switch (pidType) { case "PMID": return pid.substring(33); case "PMC": @@ -179,7 +179,7 @@ public class CreateActionSetFromWebEntries implements Serializable { return createAffiliatioRelationPair( DOI_PREFIX + IdentifierFactory - .md5(PidCleaner.normalizePidValue(PidType.doi.toString(), removeResolver("DOI" ,doi))), + .md5(PidCleaner.normalizePidValue(PidType.doi.toString(), removeResolver("DOI", doi))), ror); } From 425c9afc36e2edf3a5a7f7f7c3303f3173431e5d Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 23 Apr 2024 14:30:04 +0200 Subject: [PATCH 21/97] using version 1.2.5-beta for the release --- dhp-build/dhp-build-assembly-resources/pom.xml | 2 +- dhp-build/dhp-build-properties-maven-plugin/pom.xml | 2 +- dhp-build/dhp-code-style/pom.xml | 2 +- dhp-build/pom.xml | 2 +- dhp-common/pom.xml | 2 +- dhp-pace-core/pom.xml | 4 ++-- dhp-workflows/dhp-actionmanager/pom.xml | 2 +- dhp-workflows/dhp-aggregation/pom.xml | 2 +- dhp-workflows/dhp-blacklist/pom.xml | 2 +- dhp-workflows/dhp-broker-events/pom.xml | 2 +- dhp-workflows/dhp-dedup-openaire/pom.xml | 2 +- dhp-workflows/dhp-doiboost/pom.xml | 2 +- dhp-workflows/dhp-enrichment/pom.xml | 4 ++-- dhp-workflows/dhp-graph-mapper/pom.xml | 2 +- dhp-workflows/dhp-graph-provision/pom.xml | 2 +- dhp-workflows/dhp-impact-indicators/pom.xml | 2 +- dhp-workflows/dhp-stats-actionsets/pom.xml | 2 +- dhp-workflows/dhp-stats-hist-snaps/pom.xml | 2 +- dhp-workflows/dhp-stats-monitor-irish/pom.xml | 2 +- dhp-workflows/dhp-stats-monitor-update/pom.xml | 2 +- dhp-workflows/dhp-stats-promote/pom.xml | 2 +- dhp-workflows/dhp-stats-update/pom.xml | 2 +- dhp-workflows/dhp-swh/pom.xml | 2 +- dhp-workflows/dhp-usage-raw-data-update/pom.xml | 2 +- dhp-workflows/dhp-usage-stats-build/pom.xml | 2 +- dhp-workflows/dhp-workflow-profiles/pom.xml | 2 +- dhp-workflows/pom.xml | 2 +- pom.xml | 2 +- 28 files changed, 30 insertions(+), 30 deletions(-) diff --git a/dhp-build/dhp-build-assembly-resources/pom.xml b/dhp-build/dhp-build-assembly-resources/pom.xml index 44165995d..7f5b76fdd 100644 --- a/dhp-build/dhp-build-assembly-resources/pom.xml +++ b/dhp-build/dhp-build-assembly-resources/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp-build - 1.2.5-SNAPSHOT + 1.2.5-beta dhp-build-assembly-resources diff --git a/dhp-build/dhp-build-properties-maven-plugin/pom.xml b/dhp-build/dhp-build-properties-maven-plugin/pom.xml index 7579bdf45..e76dcd8fc 100644 --- a/dhp-build/dhp-build-properties-maven-plugin/pom.xml +++ b/dhp-build/dhp-build-properties-maven-plugin/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp-build - 1.2.5-SNAPSHOT + 1.2.5-beta dhp-build-properties-maven-plugin diff --git a/dhp-build/dhp-code-style/pom.xml b/dhp-build/dhp-code-style/pom.xml index 5a86efe17..8bbe6fac0 100644 --- a/dhp-build/dhp-code-style/pom.xml +++ b/dhp-build/dhp-code-style/pom.xml @@ -5,7 +5,7 @@ eu.dnetlib.dhp dhp-code-style - 1.2.5-SNAPSHOT + 1.2.5-beta jar diff --git a/dhp-build/pom.xml b/dhp-build/pom.xml index 9040ea94e..74a09a23c 100644 --- a/dhp-build/pom.xml +++ b/dhp-build/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp - 1.2.5-SNAPSHOT + 1.2.5-beta dhp-build pom diff --git a/dhp-common/pom.xml b/dhp-common/pom.xml index 6198bd81e..692d2bdc3 100644 --- a/dhp-common/pom.xml +++ b/dhp-common/pom.xml @@ -5,7 +5,7 @@ eu.dnetlib.dhp dhp - 1.2.5-SNAPSHOT + 1.2.5-beta ../pom.xml diff --git a/dhp-pace-core/pom.xml b/dhp-pace-core/pom.xml index fd7f44fc9..7b384f109 100644 --- a/dhp-pace-core/pom.xml +++ b/dhp-pace-core/pom.xml @@ -6,13 +6,13 @@ eu.dnetlib.dhp dhp - 1.2.5-SNAPSHOT + 1.2.5-beta ../pom.xml eu.dnetlib.dhp dhp-pace-core - 1.2.5-SNAPSHOT + 1.2.5-beta jar diff --git a/dhp-workflows/dhp-actionmanager/pom.xml b/dhp-workflows/dhp-actionmanager/pom.xml index ce13502b6..5a5f156fc 100644 --- a/dhp-workflows/dhp-actionmanager/pom.xml +++ b/dhp-workflows/dhp-actionmanager/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp-workflows - 1.2.5-SNAPSHOT + 1.2.5-beta dhp-actionmanager diff --git a/dhp-workflows/dhp-aggregation/pom.xml b/dhp-workflows/dhp-aggregation/pom.xml index 108d25ba6..d67e880b4 100644 --- a/dhp-workflows/dhp-aggregation/pom.xml +++ b/dhp-workflows/dhp-aggregation/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp-workflows - 1.2.5-SNAPSHOT + 1.2.5-beta dhp-aggregation diff --git a/dhp-workflows/dhp-blacklist/pom.xml b/dhp-workflows/dhp-blacklist/pom.xml index 7ecc8b35d..64be812ba 100644 --- a/dhp-workflows/dhp-blacklist/pom.xml +++ b/dhp-workflows/dhp-blacklist/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-SNAPSHOT + 1.2.5-beta 4.0.0 diff --git a/dhp-workflows/dhp-broker-events/pom.xml b/dhp-workflows/dhp-broker-events/pom.xml index 322fc7e93..b9f572527 100644 --- a/dhp-workflows/dhp-broker-events/pom.xml +++ b/dhp-workflows/dhp-broker-events/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-SNAPSHOT + 1.2.5-beta 4.0.0 diff --git a/dhp-workflows/dhp-dedup-openaire/pom.xml b/dhp-workflows/dhp-dedup-openaire/pom.xml index a271efe8e..96a0ae74c 100644 --- a/dhp-workflows/dhp-dedup-openaire/pom.xml +++ b/dhp-workflows/dhp-dedup-openaire/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-SNAPSHOT + 1.2.5-beta 4.0.0 dhp-dedup-openaire diff --git a/dhp-workflows/dhp-doiboost/pom.xml b/dhp-workflows/dhp-doiboost/pom.xml index 6e8911fba..cfa5a3fce 100644 --- a/dhp-workflows/dhp-doiboost/pom.xml +++ b/dhp-workflows/dhp-doiboost/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-SNAPSHOT + 1.2.5-beta 4.0.0 diff --git a/dhp-workflows/dhp-enrichment/pom.xml b/dhp-workflows/dhp-enrichment/pom.xml index 9698dee03..d7f75de8c 100644 --- a/dhp-workflows/dhp-enrichment/pom.xml +++ b/dhp-workflows/dhp-enrichment/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-SNAPSHOT + 1.2.5-beta 4.0.0 @@ -51,7 +51,7 @@ eu.dnetlib.dhp dhp-aggregation - 1.2.5-SNAPSHOT + 1.2.5-beta compile diff --git a/dhp-workflows/dhp-graph-mapper/pom.xml b/dhp-workflows/dhp-graph-mapper/pom.xml index ef35951c0..c7ac55ef6 100644 --- a/dhp-workflows/dhp-graph-mapper/pom.xml +++ b/dhp-workflows/dhp-graph-mapper/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-SNAPSHOT + 1.2.5-beta 4.0.0 diff --git a/dhp-workflows/dhp-graph-provision/pom.xml b/dhp-workflows/dhp-graph-provision/pom.xml index e62fcdf19..7b879e074 100644 --- a/dhp-workflows/dhp-graph-provision/pom.xml +++ b/dhp-workflows/dhp-graph-provision/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-SNAPSHOT + 1.2.5-beta 4.0.0 diff --git a/dhp-workflows/dhp-impact-indicators/pom.xml b/dhp-workflows/dhp-impact-indicators/pom.xml index a9eb0a4a1..d931c2323 100644 --- a/dhp-workflows/dhp-impact-indicators/pom.xml +++ b/dhp-workflows/dhp-impact-indicators/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp-workflows - 1.2.5-SNAPSHOT + 1.2.5-beta dhp-impact-indicators diff --git a/dhp-workflows/dhp-stats-actionsets/pom.xml b/dhp-workflows/dhp-stats-actionsets/pom.xml index 3daa8f995..5d9b60b87 100644 --- a/dhp-workflows/dhp-stats-actionsets/pom.xml +++ b/dhp-workflows/dhp-stats-actionsets/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp-workflows - 1.2.5-SNAPSHOT + 1.2.5-beta dhp-stats-actionsets diff --git a/dhp-workflows/dhp-stats-hist-snaps/pom.xml b/dhp-workflows/dhp-stats-hist-snaps/pom.xml index b31d909f9..94371dc0b 100644 --- a/dhp-workflows/dhp-stats-hist-snaps/pom.xml +++ b/dhp-workflows/dhp-stats-hist-snaps/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-SNAPSHOT + 1.2.5-beta 4.0.0 dhp-stats-hist-snaps diff --git a/dhp-workflows/dhp-stats-monitor-irish/pom.xml b/dhp-workflows/dhp-stats-monitor-irish/pom.xml index 6ab19dced..4887005bb 100644 --- a/dhp-workflows/dhp-stats-monitor-irish/pom.xml +++ b/dhp-workflows/dhp-stats-monitor-irish/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-SNAPSHOT + 1.2.5-beta 4.0.0 dhp-stats-monitor-irish diff --git a/dhp-workflows/dhp-stats-monitor-update/pom.xml b/dhp-workflows/dhp-stats-monitor-update/pom.xml index f2bc35f8d..c8a69c078 100644 --- a/dhp-workflows/dhp-stats-monitor-update/pom.xml +++ b/dhp-workflows/dhp-stats-monitor-update/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-SNAPSHOT + 1.2.5-beta 4.0.0 dhp-stats-monitor-update diff --git a/dhp-workflows/dhp-stats-promote/pom.xml b/dhp-workflows/dhp-stats-promote/pom.xml index 9e17a78dc..1c711c878 100644 --- a/dhp-workflows/dhp-stats-promote/pom.xml +++ b/dhp-workflows/dhp-stats-promote/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-SNAPSHOT + 1.2.5-beta 4.0.0 dhp-stats-promote diff --git a/dhp-workflows/dhp-stats-update/pom.xml b/dhp-workflows/dhp-stats-update/pom.xml index cc15b8a15..246aa63cf 100644 --- a/dhp-workflows/dhp-stats-update/pom.xml +++ b/dhp-workflows/dhp-stats-update/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-SNAPSHOT + 1.2.5-beta 4.0.0 dhp-stats-update diff --git a/dhp-workflows/dhp-swh/pom.xml b/dhp-workflows/dhp-swh/pom.xml index 80fff4587..4ba5cf868 100644 --- a/dhp-workflows/dhp-swh/pom.xml +++ b/dhp-workflows/dhp-swh/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp-workflows - 1.2.5-SNAPSHOT + 1.2.5-beta dhp-swh diff --git a/dhp-workflows/dhp-usage-raw-data-update/pom.xml b/dhp-workflows/dhp-usage-raw-data-update/pom.xml index a9dbb09ae..ed3616fde 100644 --- a/dhp-workflows/dhp-usage-raw-data-update/pom.xml +++ b/dhp-workflows/dhp-usage-raw-data-update/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-SNAPSHOT + 1.2.5-beta 4.0.0 dhp-usage-raw-data-update diff --git a/dhp-workflows/dhp-usage-stats-build/pom.xml b/dhp-workflows/dhp-usage-stats-build/pom.xml index 56aec73b7..52cc3bf44 100644 --- a/dhp-workflows/dhp-usage-stats-build/pom.xml +++ b/dhp-workflows/dhp-usage-stats-build/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-SNAPSHOT + 1.2.5-beta 4.0.0 dhp-usage-stats-build diff --git a/dhp-workflows/dhp-workflow-profiles/pom.xml b/dhp-workflows/dhp-workflow-profiles/pom.xml index 8c71a5ca1..ef4e0ada6 100644 --- a/dhp-workflows/dhp-workflow-profiles/pom.xml +++ b/dhp-workflows/dhp-workflow-profiles/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-SNAPSHOT + 1.2.5-beta 4.0.0 diff --git a/dhp-workflows/pom.xml b/dhp-workflows/pom.xml index 1c331d126..9b87c7b44 100644 --- a/dhp-workflows/pom.xml +++ b/dhp-workflows/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp - 1.2.5-SNAPSHOT + 1.2.5-beta ../pom.xml diff --git a/pom.xml b/pom.xml index 892382b9d..d015acd9e 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 eu.dnetlib.dhp dhp - 1.2.5-SNAPSHOT + 1.2.5-beta pom From b5bcab13ec088aab05d5b3a3512d2c4ab50e645a Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 23 Apr 2024 14:36:39 +0200 Subject: [PATCH 22/97] using version 1.2.5-beta for the release --- dhp-build/dhp-build-assembly-resources/pom.xml | 2 +- dhp-build/dhp-build-properties-maven-plugin/pom.xml | 2 +- dhp-build/dhp-code-style/pom.xml | 2 +- dhp-build/pom.xml | 2 +- dhp-common/pom.xml | 2 +- dhp-pace-core/pom.xml | 4 ++-- dhp-workflows/dhp-actionmanager/pom.xml | 2 +- dhp-workflows/dhp-aggregation/pom.xml | 2 +- dhp-workflows/dhp-blacklist/pom.xml | 2 +- dhp-workflows/dhp-broker-events/pom.xml | 2 +- dhp-workflows/dhp-dedup-openaire/pom.xml | 2 +- dhp-workflows/dhp-doiboost/pom.xml | 2 +- dhp-workflows/dhp-enrichment/pom.xml | 4 ++-- dhp-workflows/dhp-graph-mapper/pom.xml | 2 +- dhp-workflows/dhp-graph-provision/pom.xml | 2 +- dhp-workflows/dhp-impact-indicators/pom.xml | 2 +- dhp-workflows/dhp-stats-actionsets/pom.xml | 2 +- dhp-workflows/dhp-stats-hist-snaps/pom.xml | 2 +- dhp-workflows/dhp-stats-monitor-irish/pom.xml | 2 +- dhp-workflows/dhp-stats-monitor-update/pom.xml | 2 +- dhp-workflows/dhp-stats-promote/pom.xml | 2 +- dhp-workflows/dhp-stats-update/pom.xml | 2 +- dhp-workflows/dhp-swh/pom.xml | 2 +- dhp-workflows/dhp-usage-raw-data-update/pom.xml | 2 +- dhp-workflows/dhp-usage-stats-build/pom.xml | 2 +- dhp-workflows/dhp-workflow-profiles/pom.xml | 2 +- dhp-workflows/pom.xml | 2 +- pom.xml | 2 +- 28 files changed, 30 insertions(+), 30 deletions(-) diff --git a/dhp-build/dhp-build-assembly-resources/pom.xml b/dhp-build/dhp-build-assembly-resources/pom.xml index 7f5b76fdd..9e0674a43 100644 --- a/dhp-build/dhp-build-assembly-resources/pom.xml +++ b/dhp-build/dhp-build-assembly-resources/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp-build - 1.2.5-beta + 1.2.5-beta-SNAPSHOT dhp-build-assembly-resources diff --git a/dhp-build/dhp-build-properties-maven-plugin/pom.xml b/dhp-build/dhp-build-properties-maven-plugin/pom.xml index e76dcd8fc..178cb271a 100644 --- a/dhp-build/dhp-build-properties-maven-plugin/pom.xml +++ b/dhp-build/dhp-build-properties-maven-plugin/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp-build - 1.2.5-beta + 1.2.5-beta-SNAPSHOT dhp-build-properties-maven-plugin diff --git a/dhp-build/dhp-code-style/pom.xml b/dhp-build/dhp-code-style/pom.xml index 8bbe6fac0..093f5a9ad 100644 --- a/dhp-build/dhp-code-style/pom.xml +++ b/dhp-build/dhp-code-style/pom.xml @@ -5,7 +5,7 @@ eu.dnetlib.dhp dhp-code-style - 1.2.5-beta + 1.2.5-beta-SNAPSHOT jar diff --git a/dhp-build/pom.xml b/dhp-build/pom.xml index 74a09a23c..f944d787e 100644 --- a/dhp-build/pom.xml +++ b/dhp-build/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp - 1.2.5-beta + 1.2.5-beta-SNAPSHOT dhp-build pom diff --git a/dhp-common/pom.xml b/dhp-common/pom.xml index 692d2bdc3..b280721b6 100644 --- a/dhp-common/pom.xml +++ b/dhp-common/pom.xml @@ -5,7 +5,7 @@ eu.dnetlib.dhp dhp - 1.2.5-beta + 1.2.5-beta-SNAPSHOT ../pom.xml diff --git a/dhp-pace-core/pom.xml b/dhp-pace-core/pom.xml index 7b384f109..432da4bfd 100644 --- a/dhp-pace-core/pom.xml +++ b/dhp-pace-core/pom.xml @@ -6,13 +6,13 @@ eu.dnetlib.dhp dhp - 1.2.5-beta + 1.2.5-beta-SNAPSHOT ../pom.xml eu.dnetlib.dhp dhp-pace-core - 1.2.5-beta + 1.2.5-beta-SNAPSHOT jar diff --git a/dhp-workflows/dhp-actionmanager/pom.xml b/dhp-workflows/dhp-actionmanager/pom.xml index 5a5f156fc..e7e78e774 100644 --- a/dhp-workflows/dhp-actionmanager/pom.xml +++ b/dhp-workflows/dhp-actionmanager/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp-workflows - 1.2.5-beta + 1.2.5-beta-SNAPSHOT dhp-actionmanager diff --git a/dhp-workflows/dhp-aggregation/pom.xml b/dhp-workflows/dhp-aggregation/pom.xml index d67e880b4..db2ec2052 100644 --- a/dhp-workflows/dhp-aggregation/pom.xml +++ b/dhp-workflows/dhp-aggregation/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp-workflows - 1.2.5-beta + 1.2.5-beta-SNAPSHOT dhp-aggregation diff --git a/dhp-workflows/dhp-blacklist/pom.xml b/dhp-workflows/dhp-blacklist/pom.xml index 64be812ba..2636ac6ec 100644 --- a/dhp-workflows/dhp-blacklist/pom.xml +++ b/dhp-workflows/dhp-blacklist/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-beta-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-broker-events/pom.xml b/dhp-workflows/dhp-broker-events/pom.xml index b9f572527..84d353908 100644 --- a/dhp-workflows/dhp-broker-events/pom.xml +++ b/dhp-workflows/dhp-broker-events/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-beta-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-dedup-openaire/pom.xml b/dhp-workflows/dhp-dedup-openaire/pom.xml index 96a0ae74c..4e7e4d741 100644 --- a/dhp-workflows/dhp-dedup-openaire/pom.xml +++ b/dhp-workflows/dhp-dedup-openaire/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-beta-SNAPSHOT 4.0.0 dhp-dedup-openaire diff --git a/dhp-workflows/dhp-doiboost/pom.xml b/dhp-workflows/dhp-doiboost/pom.xml index cfa5a3fce..a2b238e55 100644 --- a/dhp-workflows/dhp-doiboost/pom.xml +++ b/dhp-workflows/dhp-doiboost/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-beta-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-enrichment/pom.xml b/dhp-workflows/dhp-enrichment/pom.xml index d7f75de8c..7297651d4 100644 --- a/dhp-workflows/dhp-enrichment/pom.xml +++ b/dhp-workflows/dhp-enrichment/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-beta-SNAPSHOT 4.0.0 @@ -51,7 +51,7 @@ eu.dnetlib.dhp dhp-aggregation - 1.2.5-beta + 1.2.5-beta-SNAPSHOT compile diff --git a/dhp-workflows/dhp-graph-mapper/pom.xml b/dhp-workflows/dhp-graph-mapper/pom.xml index c7ac55ef6..9f25f33a6 100644 --- a/dhp-workflows/dhp-graph-mapper/pom.xml +++ b/dhp-workflows/dhp-graph-mapper/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-beta-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-graph-provision/pom.xml b/dhp-workflows/dhp-graph-provision/pom.xml index 7b879e074..8fb84255f 100644 --- a/dhp-workflows/dhp-graph-provision/pom.xml +++ b/dhp-workflows/dhp-graph-provision/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-beta-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-impact-indicators/pom.xml b/dhp-workflows/dhp-impact-indicators/pom.xml index d931c2323..327c067c8 100644 --- a/dhp-workflows/dhp-impact-indicators/pom.xml +++ b/dhp-workflows/dhp-impact-indicators/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp-workflows - 1.2.5-beta + 1.2.5-beta-SNAPSHOT dhp-impact-indicators diff --git a/dhp-workflows/dhp-stats-actionsets/pom.xml b/dhp-workflows/dhp-stats-actionsets/pom.xml index 5d9b60b87..aed43cd2b 100644 --- a/dhp-workflows/dhp-stats-actionsets/pom.xml +++ b/dhp-workflows/dhp-stats-actionsets/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp-workflows - 1.2.5-beta + 1.2.5-beta-SNAPSHOT dhp-stats-actionsets diff --git a/dhp-workflows/dhp-stats-hist-snaps/pom.xml b/dhp-workflows/dhp-stats-hist-snaps/pom.xml index 94371dc0b..132875425 100644 --- a/dhp-workflows/dhp-stats-hist-snaps/pom.xml +++ b/dhp-workflows/dhp-stats-hist-snaps/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-beta-SNAPSHOT 4.0.0 dhp-stats-hist-snaps diff --git a/dhp-workflows/dhp-stats-monitor-irish/pom.xml b/dhp-workflows/dhp-stats-monitor-irish/pom.xml index 4887005bb..0e687b2cf 100644 --- a/dhp-workflows/dhp-stats-monitor-irish/pom.xml +++ b/dhp-workflows/dhp-stats-monitor-irish/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-beta-SNAPSHOT 4.0.0 dhp-stats-monitor-irish diff --git a/dhp-workflows/dhp-stats-monitor-update/pom.xml b/dhp-workflows/dhp-stats-monitor-update/pom.xml index c8a69c078..2010c0a81 100644 --- a/dhp-workflows/dhp-stats-monitor-update/pom.xml +++ b/dhp-workflows/dhp-stats-monitor-update/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-beta-SNAPSHOT 4.0.0 dhp-stats-monitor-update diff --git a/dhp-workflows/dhp-stats-promote/pom.xml b/dhp-workflows/dhp-stats-promote/pom.xml index 1c711c878..e34eb0881 100644 --- a/dhp-workflows/dhp-stats-promote/pom.xml +++ b/dhp-workflows/dhp-stats-promote/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-beta-SNAPSHOT 4.0.0 dhp-stats-promote diff --git a/dhp-workflows/dhp-stats-update/pom.xml b/dhp-workflows/dhp-stats-update/pom.xml index 246aa63cf..c1f1ac7ca 100644 --- a/dhp-workflows/dhp-stats-update/pom.xml +++ b/dhp-workflows/dhp-stats-update/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-beta-SNAPSHOT 4.0.0 dhp-stats-update diff --git a/dhp-workflows/dhp-swh/pom.xml b/dhp-workflows/dhp-swh/pom.xml index 4ba5cf868..54dda262e 100644 --- a/dhp-workflows/dhp-swh/pom.xml +++ b/dhp-workflows/dhp-swh/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp-workflows - 1.2.5-beta + 1.2.5-beta-SNAPSHOT dhp-swh diff --git a/dhp-workflows/dhp-usage-raw-data-update/pom.xml b/dhp-workflows/dhp-usage-raw-data-update/pom.xml index ed3616fde..ee238b78b 100644 --- a/dhp-workflows/dhp-usage-raw-data-update/pom.xml +++ b/dhp-workflows/dhp-usage-raw-data-update/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-beta-SNAPSHOT 4.0.0 dhp-usage-raw-data-update diff --git a/dhp-workflows/dhp-usage-stats-build/pom.xml b/dhp-workflows/dhp-usage-stats-build/pom.xml index 52cc3bf44..f7ef774b8 100644 --- a/dhp-workflows/dhp-usage-stats-build/pom.xml +++ b/dhp-workflows/dhp-usage-stats-build/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-beta-SNAPSHOT 4.0.0 dhp-usage-stats-build diff --git a/dhp-workflows/dhp-workflow-profiles/pom.xml b/dhp-workflows/dhp-workflow-profiles/pom.xml index ef4e0ada6..c0f246172 100644 --- a/dhp-workflows/dhp-workflow-profiles/pom.xml +++ b/dhp-workflows/dhp-workflow-profiles/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-beta-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/pom.xml b/dhp-workflows/pom.xml index 9b87c7b44..4e6076377 100644 --- a/dhp-workflows/pom.xml +++ b/dhp-workflows/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp - 1.2.5-beta + 1.2.5-beta-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index d015acd9e..09e02a8c2 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 eu.dnetlib.dhp dhp - 1.2.5-beta + 1.2.5-beta-SNAPSHOT pom From c3053ef34df15a198b90b3a1aa9e4305dfb14a5d Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 23 Apr 2024 14:52:32 +0200 Subject: [PATCH 23/97] using version 1.2.5-beta for the release --- dhp-build/dhp-build-assembly-resources/pom.xml | 2 +- dhp-build/dhp-build-properties-maven-plugin/pom.xml | 2 +- dhp-build/dhp-code-style/pom.xml | 2 +- dhp-build/pom.xml | 2 +- dhp-common/pom.xml | 2 +- dhp-pace-core/pom.xml | 4 ++-- dhp-workflows/dhp-actionmanager/pom.xml | 2 +- dhp-workflows/dhp-aggregation/pom.xml | 2 +- dhp-workflows/dhp-blacklist/pom.xml | 2 +- dhp-workflows/dhp-broker-events/pom.xml | 2 +- dhp-workflows/dhp-dedup-openaire/pom.xml | 2 +- dhp-workflows/dhp-doiboost/pom.xml | 2 +- dhp-workflows/dhp-enrichment/pom.xml | 4 ++-- dhp-workflows/dhp-graph-mapper/pom.xml | 2 +- dhp-workflows/dhp-graph-provision/pom.xml | 2 +- dhp-workflows/dhp-impact-indicators/pom.xml | 2 +- dhp-workflows/dhp-stats-actionsets/pom.xml | 2 +- dhp-workflows/dhp-stats-hist-snaps/pom.xml | 2 +- dhp-workflows/dhp-stats-monitor-irish/pom.xml | 2 +- dhp-workflows/dhp-stats-monitor-update/pom.xml | 2 +- dhp-workflows/dhp-stats-promote/pom.xml | 2 +- dhp-workflows/dhp-stats-update/pom.xml | 2 +- dhp-workflows/dhp-swh/pom.xml | 2 +- dhp-workflows/dhp-usage-raw-data-update/pom.xml | 2 +- dhp-workflows/dhp-usage-stats-build/pom.xml | 2 +- dhp-workflows/dhp-workflow-profiles/pom.xml | 2 +- dhp-workflows/pom.xml | 2 +- pom.xml | 2 +- 28 files changed, 30 insertions(+), 30 deletions(-) diff --git a/dhp-build/dhp-build-assembly-resources/pom.xml b/dhp-build/dhp-build-assembly-resources/pom.xml index 9e0674a43..7f5b76fdd 100644 --- a/dhp-build/dhp-build-assembly-resources/pom.xml +++ b/dhp-build/dhp-build-assembly-resources/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp-build - 1.2.5-beta-SNAPSHOT + 1.2.5-beta dhp-build-assembly-resources diff --git a/dhp-build/dhp-build-properties-maven-plugin/pom.xml b/dhp-build/dhp-build-properties-maven-plugin/pom.xml index 178cb271a..e76dcd8fc 100644 --- a/dhp-build/dhp-build-properties-maven-plugin/pom.xml +++ b/dhp-build/dhp-build-properties-maven-plugin/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp-build - 1.2.5-beta-SNAPSHOT + 1.2.5-beta dhp-build-properties-maven-plugin diff --git a/dhp-build/dhp-code-style/pom.xml b/dhp-build/dhp-code-style/pom.xml index 093f5a9ad..8bbe6fac0 100644 --- a/dhp-build/dhp-code-style/pom.xml +++ b/dhp-build/dhp-code-style/pom.xml @@ -5,7 +5,7 @@ eu.dnetlib.dhp dhp-code-style - 1.2.5-beta-SNAPSHOT + 1.2.5-beta jar diff --git a/dhp-build/pom.xml b/dhp-build/pom.xml index f944d787e..74a09a23c 100644 --- a/dhp-build/pom.xml +++ b/dhp-build/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp - 1.2.5-beta-SNAPSHOT + 1.2.5-beta dhp-build pom diff --git a/dhp-common/pom.xml b/dhp-common/pom.xml index b280721b6..692d2bdc3 100644 --- a/dhp-common/pom.xml +++ b/dhp-common/pom.xml @@ -5,7 +5,7 @@ eu.dnetlib.dhp dhp - 1.2.5-beta-SNAPSHOT + 1.2.5-beta ../pom.xml diff --git a/dhp-pace-core/pom.xml b/dhp-pace-core/pom.xml index 432da4bfd..7b384f109 100644 --- a/dhp-pace-core/pom.xml +++ b/dhp-pace-core/pom.xml @@ -6,13 +6,13 @@ eu.dnetlib.dhp dhp - 1.2.5-beta-SNAPSHOT + 1.2.5-beta ../pom.xml eu.dnetlib.dhp dhp-pace-core - 1.2.5-beta-SNAPSHOT + 1.2.5-beta jar diff --git a/dhp-workflows/dhp-actionmanager/pom.xml b/dhp-workflows/dhp-actionmanager/pom.xml index e7e78e774..5a5f156fc 100644 --- a/dhp-workflows/dhp-actionmanager/pom.xml +++ b/dhp-workflows/dhp-actionmanager/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp-workflows - 1.2.5-beta-SNAPSHOT + 1.2.5-beta dhp-actionmanager diff --git a/dhp-workflows/dhp-aggregation/pom.xml b/dhp-workflows/dhp-aggregation/pom.xml index db2ec2052..d67e880b4 100644 --- a/dhp-workflows/dhp-aggregation/pom.xml +++ b/dhp-workflows/dhp-aggregation/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp-workflows - 1.2.5-beta-SNAPSHOT + 1.2.5-beta dhp-aggregation diff --git a/dhp-workflows/dhp-blacklist/pom.xml b/dhp-workflows/dhp-blacklist/pom.xml index 2636ac6ec..64be812ba 100644 --- a/dhp-workflows/dhp-blacklist/pom.xml +++ b/dhp-workflows/dhp-blacklist/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta-SNAPSHOT + 1.2.5-beta 4.0.0 diff --git a/dhp-workflows/dhp-broker-events/pom.xml b/dhp-workflows/dhp-broker-events/pom.xml index 84d353908..b9f572527 100644 --- a/dhp-workflows/dhp-broker-events/pom.xml +++ b/dhp-workflows/dhp-broker-events/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta-SNAPSHOT + 1.2.5-beta 4.0.0 diff --git a/dhp-workflows/dhp-dedup-openaire/pom.xml b/dhp-workflows/dhp-dedup-openaire/pom.xml index 4e7e4d741..96a0ae74c 100644 --- a/dhp-workflows/dhp-dedup-openaire/pom.xml +++ b/dhp-workflows/dhp-dedup-openaire/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta-SNAPSHOT + 1.2.5-beta 4.0.0 dhp-dedup-openaire diff --git a/dhp-workflows/dhp-doiboost/pom.xml b/dhp-workflows/dhp-doiboost/pom.xml index a2b238e55..cfa5a3fce 100644 --- a/dhp-workflows/dhp-doiboost/pom.xml +++ b/dhp-workflows/dhp-doiboost/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta-SNAPSHOT + 1.2.5-beta 4.0.0 diff --git a/dhp-workflows/dhp-enrichment/pom.xml b/dhp-workflows/dhp-enrichment/pom.xml index 7297651d4..d7f75de8c 100644 --- a/dhp-workflows/dhp-enrichment/pom.xml +++ b/dhp-workflows/dhp-enrichment/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta-SNAPSHOT + 1.2.5-beta 4.0.0 @@ -51,7 +51,7 @@ eu.dnetlib.dhp dhp-aggregation - 1.2.5-beta-SNAPSHOT + 1.2.5-beta compile diff --git a/dhp-workflows/dhp-graph-mapper/pom.xml b/dhp-workflows/dhp-graph-mapper/pom.xml index 9f25f33a6..c7ac55ef6 100644 --- a/dhp-workflows/dhp-graph-mapper/pom.xml +++ b/dhp-workflows/dhp-graph-mapper/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta-SNAPSHOT + 1.2.5-beta 4.0.0 diff --git a/dhp-workflows/dhp-graph-provision/pom.xml b/dhp-workflows/dhp-graph-provision/pom.xml index 8fb84255f..7b879e074 100644 --- a/dhp-workflows/dhp-graph-provision/pom.xml +++ b/dhp-workflows/dhp-graph-provision/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta-SNAPSHOT + 1.2.5-beta 4.0.0 diff --git a/dhp-workflows/dhp-impact-indicators/pom.xml b/dhp-workflows/dhp-impact-indicators/pom.xml index 327c067c8..d931c2323 100644 --- a/dhp-workflows/dhp-impact-indicators/pom.xml +++ b/dhp-workflows/dhp-impact-indicators/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp-workflows - 1.2.5-beta-SNAPSHOT + 1.2.5-beta dhp-impact-indicators diff --git a/dhp-workflows/dhp-stats-actionsets/pom.xml b/dhp-workflows/dhp-stats-actionsets/pom.xml index aed43cd2b..5d9b60b87 100644 --- a/dhp-workflows/dhp-stats-actionsets/pom.xml +++ b/dhp-workflows/dhp-stats-actionsets/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp-workflows - 1.2.5-beta-SNAPSHOT + 1.2.5-beta dhp-stats-actionsets diff --git a/dhp-workflows/dhp-stats-hist-snaps/pom.xml b/dhp-workflows/dhp-stats-hist-snaps/pom.xml index 132875425..94371dc0b 100644 --- a/dhp-workflows/dhp-stats-hist-snaps/pom.xml +++ b/dhp-workflows/dhp-stats-hist-snaps/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta-SNAPSHOT + 1.2.5-beta 4.0.0 dhp-stats-hist-snaps diff --git a/dhp-workflows/dhp-stats-monitor-irish/pom.xml b/dhp-workflows/dhp-stats-monitor-irish/pom.xml index 0e687b2cf..4887005bb 100644 --- a/dhp-workflows/dhp-stats-monitor-irish/pom.xml +++ b/dhp-workflows/dhp-stats-monitor-irish/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta-SNAPSHOT + 1.2.5-beta 4.0.0 dhp-stats-monitor-irish diff --git a/dhp-workflows/dhp-stats-monitor-update/pom.xml b/dhp-workflows/dhp-stats-monitor-update/pom.xml index 2010c0a81..c8a69c078 100644 --- a/dhp-workflows/dhp-stats-monitor-update/pom.xml +++ b/dhp-workflows/dhp-stats-monitor-update/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta-SNAPSHOT + 1.2.5-beta 4.0.0 dhp-stats-monitor-update diff --git a/dhp-workflows/dhp-stats-promote/pom.xml b/dhp-workflows/dhp-stats-promote/pom.xml index e34eb0881..1c711c878 100644 --- a/dhp-workflows/dhp-stats-promote/pom.xml +++ b/dhp-workflows/dhp-stats-promote/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta-SNAPSHOT + 1.2.5-beta 4.0.0 dhp-stats-promote diff --git a/dhp-workflows/dhp-stats-update/pom.xml b/dhp-workflows/dhp-stats-update/pom.xml index c1f1ac7ca..246aa63cf 100644 --- a/dhp-workflows/dhp-stats-update/pom.xml +++ b/dhp-workflows/dhp-stats-update/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta-SNAPSHOT + 1.2.5-beta 4.0.0 dhp-stats-update diff --git a/dhp-workflows/dhp-swh/pom.xml b/dhp-workflows/dhp-swh/pom.xml index 54dda262e..4ba5cf868 100644 --- a/dhp-workflows/dhp-swh/pom.xml +++ b/dhp-workflows/dhp-swh/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp-workflows - 1.2.5-beta-SNAPSHOT + 1.2.5-beta dhp-swh diff --git a/dhp-workflows/dhp-usage-raw-data-update/pom.xml b/dhp-workflows/dhp-usage-raw-data-update/pom.xml index ee238b78b..ed3616fde 100644 --- a/dhp-workflows/dhp-usage-raw-data-update/pom.xml +++ b/dhp-workflows/dhp-usage-raw-data-update/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta-SNAPSHOT + 1.2.5-beta 4.0.0 dhp-usage-raw-data-update diff --git a/dhp-workflows/dhp-usage-stats-build/pom.xml b/dhp-workflows/dhp-usage-stats-build/pom.xml index f7ef774b8..52cc3bf44 100644 --- a/dhp-workflows/dhp-usage-stats-build/pom.xml +++ b/dhp-workflows/dhp-usage-stats-build/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta-SNAPSHOT + 1.2.5-beta 4.0.0 dhp-usage-stats-build diff --git a/dhp-workflows/dhp-workflow-profiles/pom.xml b/dhp-workflows/dhp-workflow-profiles/pom.xml index c0f246172..ef4e0ada6 100644 --- a/dhp-workflows/dhp-workflow-profiles/pom.xml +++ b/dhp-workflows/dhp-workflow-profiles/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta-SNAPSHOT + 1.2.5-beta 4.0.0 diff --git a/dhp-workflows/pom.xml b/dhp-workflows/pom.xml index 4e6076377..9b87c7b44 100644 --- a/dhp-workflows/pom.xml +++ b/dhp-workflows/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp - 1.2.5-beta-SNAPSHOT + 1.2.5-beta ../pom.xml diff --git a/pom.xml b/pom.xml index 09e02a8c2..d015acd9e 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 eu.dnetlib.dhp dhp - 1.2.5-beta-SNAPSHOT + 1.2.5-beta pom From 1878199dae8092138f1beb5b380d46c4a4348302 Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Wed, 24 Apr 2024 08:12:45 +0200 Subject: [PATCH 24/97] Miscellaneous fixes: - in Merge By ID pick by preference those records coming from delegated Authorities - fix various tests - close spark session in SparkCreateSimRels --- .../dhp/oa/merge/GroupEntitiesSparkJob.java | 2 +- .../dhp/schema/oaf/utils/MergeUtils.java | 44 +++++++++++++------ .../oaf/utils/ResultTypeComparator.java | 9 ++++ .../dhp/schema/oaf/utils/MergeUtilsTest.java | 6 +-- dhp-workflows/dhp-dedup-openaire/pom.xml | 1 - .../dhp/oa/dedup/DedupRecordFactory.java | 2 +- .../dhp/oa/dedup/SparkCreateMergeRels.java | 1 + .../dhp/oa/dedup/SparkCreateSimRels.java | 6 ++- .../dhp/oa/dedup/EntityMergerTest.java | 2 +- .../dnetlib/dhp/oa/dedup/IdGeneratorTest.java | 2 +- .../dhp/oa/dedup/SparkOpenorgsDedupTest.java | 8 ++-- .../oa/dedup/SparkPublicationRootsTest.java | 22 ++++++---- .../dnetlib/dhp/oa/dedup/SparkStatsTest.java | 8 ++-- .../SparkResultToCommunityFromProject.java | 2 +- .../raw/GenerateEntitiesApplicationTest.java | 2 +- 15 files changed, 76 insertions(+), 41 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java index a85afaf25..24de1a787 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java @@ -135,7 +135,7 @@ public class GroupEntitiesSparkJob { .applyCoarVocabularies(entity, vocs), OAFENTITY_KRYO_ENC) .groupByKey((MapFunction) OafEntity::getId, Encoders.STRING()) - .mapGroups((MapGroupsFunction) MergeUtils::mergeGroup, OAFENTITY_KRYO_ENC) + .mapGroups((MapGroupsFunction) MergeUtils::mergeById, OAFENTITY_KRYO_ENC) .map( (MapFunction>) t -> new Tuple2<>( t.getClass().getName(), t), diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils.java index c95c31c51..570389397 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils.java @@ -30,8 +30,16 @@ import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.oaf.*; public class MergeUtils { + public static T mergeById(String s, Iterator oafEntityIterator) { + return mergeGroup(s, oafEntityIterator, true); + } public static T mergeGroup(String s, Iterator oafEntityIterator) { + return mergeGroup(s, oafEntityIterator, false); + } + + public static T mergeGroup(String s, Iterator oafEntityIterator, + boolean checkDelegateAuthority) { TreeSet sortedEntities = new TreeSet<>((o1, o2) -> { int res = 0; @@ -52,18 +60,22 @@ public class MergeUtils { sortedEntities.add(oafEntityIterator.next()); } - T merged = sortedEntities.descendingIterator().next(); - Iterator it = sortedEntities.descendingIterator(); + T merged = it.next(); + while (it.hasNext()) { - merged = checkedMerge(merged, it.next()); + merged = checkedMerge(merged, it.next(), checkDelegateAuthority); } return merged; } - public static T checkedMerge(final T left, final T right) { - return (T) merge(left, right, false); + public static T checkedMerge(final T left, final T right, boolean checkDelegateAuthority) { + return (T) merge(left, right, checkDelegateAuthority); + } + + public static Result mergeResult(final T left, final E right) { + return (Result) merge(left, right, false); } public static Oaf merge(final Oaf left, final Oaf right) { @@ -108,7 +120,7 @@ public class MergeUtils { return mergeSoftware((Software) left, (Software) right); } - return mergeResult((Result) left, (Result) right); + return mergeResultFields((Result) left, (Result) right); } else if (sameClass(left, right, Datasource.class)) { // TODO final int trust = compareTrust(left, right); @@ -151,9 +163,9 @@ public class MergeUtils { } // TODO: raise trust to have preferred fields from one or the other?? if (new ResultTypeComparator().compare(left, right) < 0) { - return mergeResult(left, right); + return mergeResultFields(left, right); } else { - return mergeResult(right, left); + return mergeResultFields(right, left); } } @@ -263,6 +275,12 @@ public class MergeUtils { // TODO review private static List mergeByKey(List left, List right, int trust) { + if (left == null) { + return right; + } else if (right == null) { + return left; + } + if (trust < 0) { List s = left; left = right; @@ -367,7 +385,7 @@ public class MergeUtils { return merge; } - public static T mergeResult(T original, T enrich) { + private static T mergeResultFields(T original, T enrich) { final int trust = compareTrust(original, enrich); T merge = mergeOafEntityFields(original, enrich, trust); @@ -693,7 +711,7 @@ public class MergeUtils { private static T mergeORP(T original, T enrich) { int trust = compareTrust(original, enrich); - final T merge = mergeResult(original, enrich); + final T merge = mergeResultFields(original, enrich); merge.setContactperson(unionDistinctLists(merge.getContactperson(), enrich.getContactperson(), trust)); merge.setContactgroup(unionDistinctLists(merge.getContactgroup(), enrich.getContactgroup(), trust)); @@ -704,7 +722,7 @@ public class MergeUtils { private static T mergeSoftware(T original, T enrich) { int trust = compareTrust(original, enrich); - final T merge = mergeResult(original, enrich); + final T merge = mergeResultFields(original, enrich); merge.setDocumentationUrl(unionDistinctLists(merge.getDocumentationUrl(), enrich.getDocumentationUrl(), trust)); merge.setLicense(unionDistinctLists(merge.getLicense(), enrich.getLicense(), trust)); @@ -718,7 +736,7 @@ public class MergeUtils { private static T mergeDataset(T original, T enrich) { int trust = compareTrust(original, enrich); - T merge = mergeResult(original, enrich); + T merge = mergeResultFields(original, enrich); merge.setStoragedate(chooseReference(merge.getStoragedate(), enrich.getStoragedate(), trust)); merge.setDevice(chooseReference(merge.getDevice(), enrich.getDevice(), trust)); @@ -737,7 +755,7 @@ public class MergeUtils { public static T mergePublication(T original, T enrich) { final int trust = compareTrust(original, enrich); - T merged = mergeResult(original, enrich); + T merged = mergeResultFields(original, enrich); merged.setJournal(chooseReference(merged.getJournal(), enrich.getJournal(), trust)); diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/ResultTypeComparator.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/ResultTypeComparator.java index ba55621e5..e10b281b8 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/ResultTypeComparator.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/ResultTypeComparator.java @@ -36,6 +36,15 @@ public class ResultTypeComparator implements Comparator { return 1; } + if (left.getResulttype() == null || left.getResulttype().getClassid() == null) { + if (right.getResulttype() == null || right.getResulttype().getClassid() == null) { + return 0; + } + return 1; + } else if (right.getResulttype() == null || right.getResulttype().getClassid() == null) { + return -1; + } + String lClass = left.getResulttype().getClassid(); String rClass = right.getResulttype().getClassid(); diff --git a/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtilsTest.java b/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtilsTest.java index 9b9ad0c48..89b1385b3 100644 --- a/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtilsTest.java +++ b/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtilsTest.java @@ -63,7 +63,7 @@ public class MergeUtilsTest { assertEquals(1, d1.getCollectedfrom().size()); assertTrue(cfId(d1.getCollectedfrom()).contains(ModelConstants.CROSSREF_ID)); - final Result p1d2 = MergeUtils.checkedMerge(p1, d2); + final Result p1d2 = MergeUtils.checkedMerge(p1, d2, true); assertEquals(ModelConstants.PUBLICATION_RESULTTYPE_CLASSID, p1d2.getResulttype().getClassid()); assertTrue(p1d2 instanceof Publication); assertEquals(p1.getId(), p1d2.getId()); @@ -74,7 +74,7 @@ public class MergeUtilsTest { Publication p2 = read("publication_2.json", Publication.class); Dataset d1 = read("dataset_1.json", Dataset.class); - final Result p2d1 = MergeUtils.checkedMerge(p2, d1); + final Result p2d1 = MergeUtils.checkedMerge(p2, d1, true); assertEquals((ModelConstants.DATASET_RESULTTYPE_CLASSID), p2d1.getResulttype().getClassid()); assertTrue(p2d1 instanceof Dataset); assertEquals(d1.getId(), p2d1.getId()); @@ -86,7 +86,7 @@ public class MergeUtilsTest { Publication p1 = read("publication_1.json", Publication.class); Publication p2 = read("publication_2.json", Publication.class); - Result p1p2 = MergeUtils.checkedMerge(p1, p2); + Result p1p2 = MergeUtils.checkedMerge(p1, p2, true); assertTrue(p1p2 instanceof Publication); assertEquals(p1.getId(), p1p2.getId()); assertEquals(2, p1p2.getCollectedfrom().size()); diff --git a/dhp-workflows/dhp-dedup-openaire/pom.xml b/dhp-workflows/dhp-dedup-openaire/pom.xml index a271efe8e..8665ebd05 100644 --- a/dhp-workflows/dhp-dedup-openaire/pom.xml +++ b/dhp-workflows/dhp-dedup-openaire/pom.xml @@ -38,7 +38,6 @@ - diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/DedupRecordFactory.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/DedupRecordFactory.java index cf8c9ac3b..36ed4d7c1 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/DedupRecordFactory.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/DedupRecordFactory.java @@ -189,7 +189,7 @@ public class DedupRecordFactory { entity = swap; } - entity = MergeUtils.checkedMerge(entity, duplicate); + entity = MergeUtils.checkedMerge(entity, duplicate, false); if (ModelSupport.isSubClass(duplicate, Result.class)) { Result re = (Result) entity; diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateMergeRels.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateMergeRels.java index 59626c141..fc0e3bdb9 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateMergeRels.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateMergeRels.java @@ -175,6 +175,7 @@ public class SparkCreateMergeRels extends AbstractSparkAction { } // cap pidType at w3id as from there on they are considered equal + UserDefinedFunction mapPid = udf( (String s) -> Math.min(PidType.tryValueOf(s).ordinal(), PidType.w3id.ordinal()), DataTypes.IntegerType); diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateSimRels.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateSimRels.java index 5f54c34df..3d543c8cd 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateSimRels.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateSimRels.java @@ -44,8 +44,10 @@ public class SparkCreateSimRels extends AbstractSparkAction { parser.parseArgument(args); SparkConf conf = new SparkConf(); - new SparkCreateSimRels(parser, getSparkSession(conf)) - .run(ISLookupClientFactory.getLookUpService(parser.get("isLookUpUrl"))); + try (SparkSession session = getSparkSession(conf)) { + new SparkCreateSimRels(parser, session) + .run(ISLookupClientFactory.getLookUpService(parser.get("isLookUpUrl"))); + } } @Override diff --git a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/EntityMergerTest.java b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/EntityMergerTest.java index 42ca1613f..4a5a3bd1b 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/EntityMergerTest.java +++ b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/EntityMergerTest.java @@ -123,7 +123,7 @@ class EntityMergerTest implements Serializable { assertEquals(dataInfo, pub_merged.getDataInfo()); // verify datepicker - assertEquals("2018-09-30", pub_merged.getDateofacceptance().getValue()); + assertEquals("2016-01-01", pub_merged.getDateofacceptance().getValue()); // verify authors assertEquals(13, pub_merged.getAuthor().size()); diff --git a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/IdGeneratorTest.java b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/IdGeneratorTest.java index 2d6637882..cc084e4f3 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/IdGeneratorTest.java +++ b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/IdGeneratorTest.java @@ -78,7 +78,7 @@ public class IdGeneratorTest { System.out.println("winner 3 = " + id2); assertEquals("50|doi_dedup___::1a77a3bba737f8b669dcf330ad3b37e2", id1); - assertEquals("50|dedup_wf_001::0829b5191605bdbea36d6502b8c1ce1g", id2); + assertEquals("50|dedup_wf_002::345e5d1b80537b0d0e0a49241ae9e516", id2); } @Test diff --git a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkOpenorgsDedupTest.java b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkOpenorgsDedupTest.java index a0c7772e9..6f2a6904b 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkOpenorgsDedupTest.java +++ b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkOpenorgsDedupTest.java @@ -143,7 +143,7 @@ public class SparkOpenorgsDedupTest implements Serializable { .load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "organization")) .count(); - assertEquals(145, orgs_simrel); + assertEquals(86, orgs_simrel); } @Test @@ -172,7 +172,7 @@ public class SparkOpenorgsDedupTest implements Serializable { .load(DedupUtility.createSimRelPath(testOutputBasePath, testActionSetId, "organization")) .count(); - assertEquals(181, orgs_simrel); + assertEquals(122, orgs_simrel); } @Test @@ -196,7 +196,9 @@ public class SparkOpenorgsDedupTest implements Serializable { "-la", "lookupurl", "-w", - testOutputBasePath + testOutputBasePath, + "-h", + "" }); new SparkCreateMergeRels(parser, spark).run(isLookUpService); diff --git a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkPublicationRootsTest.java b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkPublicationRootsTest.java index e3fe882ef..9d73475be 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkPublicationRootsTest.java +++ b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkPublicationRootsTest.java @@ -13,14 +13,16 @@ import java.io.Serializable; import java.net.URISyntaxException; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.*; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import org.apache.commons.cli.ParseException; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.spark.SparkConf; -import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.function.FilterFunction; import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.sql.Dataset; @@ -129,7 +131,7 @@ public class SparkPublicationRootsTest implements Serializable { .load(DedupUtility.createSimRelPath(workingPath, testActionSetId, "publication")) .count(); - assertEquals(37, pubs_simrel); + assertEquals(9, pubs_simrel); } @Test @@ -142,7 +144,8 @@ public class SparkPublicationRootsTest implements Serializable { "--actionSetId", testActionSetId, "--isLookUpUrl", "lookupurl", "--workingPath", workingPath, - "--cutConnectedComponent", "3" + "--cutConnectedComponent", "3", + "-h", "" }), spark) .run(isLookUpService); @@ -171,7 +174,8 @@ public class SparkPublicationRootsTest implements Serializable { "--graphBasePath", graphInputPath, "--actionSetId", testActionSetId, "--isLookUpUrl", "lookupurl", - "--workingPath", workingPath + "--workingPath", workingPath, + "-h", "" }), spark) .run(isLookUpService); @@ -207,7 +211,7 @@ public class SparkPublicationRootsTest implements Serializable { assertTrue(dups.contains(r.getSource())); }); - assertEquals(32, merges.count()); + assertEquals(26, merges.count()); } @Test @@ -228,7 +232,7 @@ public class SparkPublicationRootsTest implements Serializable { .textFile(workingPath + "/" + testActionSetId + "/publication_deduprecord") .map(asEntity(Publication.class), Encoders.bean(Publication.class)); - assertEquals(3, roots.count()); + assertEquals(4, roots.count()); final Dataset pubs = spark .read() @@ -369,7 +373,7 @@ public class SparkPublicationRootsTest implements Serializable { .distinct() .count(); - assertEquals(19, publications); // 16 originals + 3 roots + assertEquals(20, publications); // 16 originals + 3 roots long deletedPubs = spark .read() @@ -380,7 +384,7 @@ public class SparkPublicationRootsTest implements Serializable { .distinct() .count(); - assertEquals(mergedPubs, deletedPubs); +// assertEquals(mergedPubs, deletedPubs); } private static String classPathResourceAsString(String path) throws IOException { diff --git a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkStatsTest.java b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkStatsTest.java index 07e993444..19f2c8102 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkStatsTest.java +++ b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkStatsTest.java @@ -169,10 +169,10 @@ public class SparkStatsTest implements Serializable { .count(); assertEquals(414, orgs_blocks); - assertEquals(187, pubs_blocks); - assertEquals(128, sw_blocks); - assertEquals(192, ds_blocks); - assertEquals(194, orp_blocks); + assertEquals(221, pubs_blocks); + assertEquals(134, sw_blocks); + assertEquals(196, ds_blocks); + assertEquals(198, orp_blocks); } @AfterAll diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java index 934856742..7a6238940 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java @@ -161,7 +161,7 @@ public class SparkResultToCommunityFromProject implements Serializable { } } res.setContext(propagatedContexts); - return MergeUtils.checkedMerge(ret, res); + return MergeUtils.checkedMerge(ret, res, true); } return ret; }; diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplicationTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplicationTest.java index c2f3faf29..6ec2f1d51 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplicationTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplicationTest.java @@ -71,7 +71,7 @@ class GenerateEntitiesApplicationTest { protected void verifyMerge(Result publication, Result dataset, Class clazz, String resultType) { - final Result merge = MergeUtils.mergeResult(publication, dataset); + final Result merge = (Result) MergeUtils.merge(publication, dataset); assertTrue(clazz.isAssignableFrom(merge.getClass())); assertEquals(resultType, merge.getResulttype().getClassid()); } From 9cd3bc0f10cc8104cd1dcde539f577ea1a3f3df9 Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Fri, 26 Apr 2024 16:02:07 +0200 Subject: [PATCH 25/97] Added a new generation of the dump for scholexplorer tested with last version of spark, and strongly refactored --- .../scholexplorer/relation/relations.json | 8 + .../dhp/sx/graph/scholix/ScholixUtils.scala | 19 +- .../dhp/sx/create_scholix_dump_params.json | 5 + .../eu/dnetlib/dhp/sx/relation/relations.json | 166 ++++++++++++ .../dhp/sx/graph/ScholexplorerUtils.scala | 256 ++++++++++++++++++ .../graph/SparkCreateScholexplorerDump.scala | 130 +++++++++ .../graph/scholix/ScholixGenerationTest.scala | 17 ++ pom.xml | 2 +- 8 files changed, 597 insertions(+), 6 deletions(-) create mode 100644 dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/create_scholix_dump_params.json create mode 100644 dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/relation/relations.json create mode 100644 dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/ScholexplorerUtils.scala create mode 100644 dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkCreateScholexplorerDump.scala create mode 100644 dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixGenerationTest.scala diff --git a/dhp-common/src/main/resources/eu/dnetlib/scholexplorer/relation/relations.json b/dhp-common/src/main/resources/eu/dnetlib/scholexplorer/relation/relations.json index 98e8daa18..4f0cee53d 100644 --- a/dhp-common/src/main/resources/eu/dnetlib/scholexplorer/relation/relations.json +++ b/dhp-common/src/main/resources/eu/dnetlib/scholexplorer/relation/relations.json @@ -154,5 +154,13 @@ "unknown":{ "original":"Unknown", "inverse":"Unknown" + }, + "isamongtopnsimilardocuments": { + "original": "IsAmongTopNSimilarDocuments", + "inverse": "HasAmongTopNSimilarDocuments" + }, + "hasamongtopnsimilardocuments": { + "original": "HasAmongTopNSimilarDocuments", + "inverse": "IsAmongTopNSimilarDocuments" } } \ No newline at end of file diff --git a/dhp-common/src/main/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixUtils.scala b/dhp-common/src/main/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixUtils.scala index a995016a8..f256ca1a1 100644 --- a/dhp-common/src/main/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixUtils.scala +++ b/dhp-common/src/main/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixUtils.scala @@ -65,7 +65,11 @@ object ScholixUtils extends Serializable { } def generateScholixResourceFromResult(r: Result): ScholixResource = { - generateScholixResourceFromSummary(ScholixUtils.resultToSummary(r)) + val sum = ScholixUtils.resultToSummary(r) + if (sum != null) + generateScholixResourceFromSummary(ScholixUtils.resultToSummary(r)) + else + null } val statsAggregator: Aggregator[(String, String, Long), RelatedEntities, RelatedEntities] = @@ -153,6 +157,14 @@ object ScholixUtils extends Serializable { } + def invRel(rel: String): String = { + val semanticRelation = relations.getOrElse(rel.toLowerCase, null) + if (semanticRelation != null) + semanticRelation.inverse + else + null + } + def extractCollectedFrom(summary: ScholixResource): List[ScholixEntityId] = { if (summary.getCollectedFrom != null && !summary.getCollectedFrom.isEmpty) { val l: List[ScholixEntityId] = summary.getCollectedFrom.asScala.map { d => @@ -377,10 +389,7 @@ object ScholixUtils extends Serializable { if (persistentIdentifiers.isEmpty) return null s.setLocalIdentifier(persistentIdentifiers.asJava) - if (r.isInstanceOf[Publication]) - s.setTypology(Typology.publication) - else - s.setTypology(Typology.dataset) + s.setTypology(r.getResulttype.getClassid) s.setSubType(r.getInstance().get(0).getInstancetype.getClassname) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/create_scholix_dump_params.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/create_scholix_dump_params.json new file mode 100644 index 000000000..fead58ab1 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/create_scholix_dump_params.json @@ -0,0 +1,5 @@ +[ + {"paramName":"mt", "paramLongName":"master", "paramDescription": "should be local or yarn", "paramRequired": true}, + {"paramName":"s", "paramLongName":"sourcePath", "paramDescription": "the source Path", "paramRequired": true}, + {"paramName":"t", "paramLongName":"targetPath", "paramDescription": "the path of the scholix dump", "paramRequired": true} +] \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/relation/relations.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/relation/relations.json new file mode 100644 index 000000000..4f0cee53d --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/relation/relations.json @@ -0,0 +1,166 @@ +{ + "cites":{ + "original":"Cites", + "inverse":"IsCitedBy" + }, + "compiles":{ + "original":"Compiles", + "inverse":"IsCompiledBy" + }, + "continues":{ + "original":"Continues", + "inverse":"IsContinuedBy" + }, + "derives":{ + "original":"IsSourceOf", + "inverse":"IsDerivedFrom" + }, + "describes":{ + "original":"Describes", + "inverse":"IsDescribedBy" + }, + "documents":{ + "original":"Documents", + "inverse":"IsDocumentedBy" + }, + "hasmetadata":{ + "original":"HasMetadata", + "inverse":"IsMetadataOf" + }, + "hasassociationwith":{ + "original":"HasAssociationWith", + "inverse":"HasAssociationWith" + }, + "haspart":{ + "original":"HasPart", + "inverse":"IsPartOf" + }, + "hasversion":{ + "original":"HasVersion", + "inverse":"IsVersionOf" + }, + "iscitedby":{ + "original":"IsCitedBy", + "inverse":"Cites" + }, + "iscompiledby":{ + "original":"IsCompiledBy", + "inverse":"Compiles" + }, + "iscontinuedby":{ + "original":"IsContinuedBy", + "inverse":"Continues" + }, + "isderivedfrom":{ + "original":"IsDerivedFrom", + "inverse":"IsSourceOf" + }, + "isdescribedby":{ + "original":"IsDescribedBy", + "inverse":"Describes" + }, + "isdocumentedby":{ + "original":"IsDocumentedBy", + "inverse":"Documents" + }, + "isidenticalto":{ + "original":"IsIdenticalTo", + "inverse":"IsIdenticalTo" + }, + "ismetadatafor":{ + "original":"IsMetadataFor", + "inverse":"IsMetadataOf" + }, + "ismetadataof":{ + "original":"IsMetadataOf", + "inverse":"IsMetadataFor" + }, + "isnewversionof":{ + "original":"IsNewVersionOf", + "inverse":"IsPreviousVersionOf" + }, + "isobsoletedby":{ + "original":"IsObsoletedBy", + "inverse":"Obsoletes" + }, + "isoriginalformof":{ + "original":"IsOriginalFormOf", + "inverse":"IsVariantFormOf" + }, + "ispartof":{ + "original":"IsPartOf", + "inverse":"HasPart" + }, + "ispreviousversionof":{ + "original":"IsPreviousVersionOf", + "inverse":"IsNewVersionOf" + }, + "isreferencedby":{ + "original":"IsReferencedBy", + "inverse":"References" + }, + "isrelatedto":{ + "original":"IsRelatedTo", + "inverse":"IsRelatedTo" + }, + "isrequiredby":{ + "original":"IsRequiredBy", + "inverse":"Requires" + }, + "isreviewedby":{ + "original":"IsReviewedBy", + "inverse":"Reviews" + }, + "issourceof":{ + "original":"IsSourceOf", + "inverse":"IsDerivedFrom" + }, + "issupplementedby":{ + "original":"IsSupplementedBy", + "inverse":"IsSupplementTo" + }, + "issupplementto":{ + "original":"IsSupplementTo", + "inverse":"IsSupplementedBy" + }, + "isvariantformof":{ + "original":"IsVariantFormOf", + "inverse":"IsOriginalFormOf" + }, + "isversionof":{ + "original":"IsVersionOf", + "inverse":"HasVersion" + }, + "obsoletes":{ + "original":"Obsoletes", + "inverse":"IsObsoletedBy" + }, + "references":{ + "original":"References", + "inverse":"IsReferencedBy" + }, + "requires":{ + "original":"Requires", + "inverse":"IsRequiredBy" + }, + "related":{ + "original":"IsRelatedTo", + "inverse":"IsRelatedTo" + }, + "reviews":{ + "original":"Reviews", + "inverse":"IsReviewedBy" + }, + "unknown":{ + "original":"Unknown", + "inverse":"Unknown" + }, + "isamongtopnsimilardocuments": { + "original": "IsAmongTopNSimilarDocuments", + "inverse": "HasAmongTopNSimilarDocuments" + }, + "hasamongtopnsimilardocuments": { + "original": "HasAmongTopNSimilarDocuments", + "inverse": "IsAmongTopNSimilarDocuments" + } +} \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/ScholexplorerUtils.scala b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/ScholexplorerUtils.scala new file mode 100644 index 000000000..95564d523 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/ScholexplorerUtils.scala @@ -0,0 +1,256 @@ +package eu.dnetlib.dhp.sx.graph + +import eu.dnetlib.dhp.schema.oaf.{KeyValue, Result, StructuredProperty} +import eu.dnetlib.dhp.schema.sx.scholix.{ + Scholix, + ScholixCollectedFrom, + ScholixEntityId, + ScholixIdentifier, + ScholixRelationship, + ScholixResource +} +import org.json4s +import org.json4s.DefaultFormats +import org.json4s.jackson.JsonMethods.parse + +import scala.collection.JavaConverters._ +import scala.io.Source + +case class RelationInfo( + source: String, + target: String, + relclass: String, + id: String, + collectedfrom: Seq[RelKeyValue] +) {} +case class RelKeyValue(key: String, value: String) {} + +object ScholexplorerUtils { + + val OPENAIRE_IDENTIFIER_SCHEMA: String = "OpenAIRE Identifier" + + case class RelationVocabulary(original: String, inverse: String) {} + + val relations: Map[String, RelationVocabulary] = { + val input = Source + .fromInputStream( + getClass.getResourceAsStream("/eu/dnetlib/dhp/sx/relation/relations.json") + ) + .mkString + implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats + + lazy val json: json4s.JValue = parse(input) + + json.extract[Map[String, RelationVocabulary]] + } + + def invRel(rel: String): String = { + val semanticRelation = relations.getOrElse(rel.toLowerCase, null) + if (semanticRelation != null) + semanticRelation.inverse + else + null + } + + def generateDatasourceOpenAIREURLS(id: String): String = { + if (id != null && id.length > 12) + s"https://explore.openaire.eu/search/dataprovider?datasourceId=${id.substring(3)}" + else + null + } + + def findURLForPID( + pidValue: List[StructuredProperty], + urls: List[String] + ): List[(StructuredProperty, String)] = { + pidValue.map { p => + val pv = p.getValue + + val r = urls.find(u => u.toLowerCase.contains(pv.toLowerCase)) + (p, r.orNull) + } + } + + def extractTypedIdentifierFromInstance(r: Result): List[ScholixIdentifier] = { + if (r.getInstance() == null || r.getInstance().isEmpty) + return List() + r.getInstance() + .asScala + .filter(i => i.getUrl != null && !i.getUrl.isEmpty) + .filter(i => i.getPid != null && i.getUrl != null) + .flatMap(i => findURLForPID(i.getPid.asScala.toList, i.getUrl.asScala.toList)) + .map(i => new ScholixIdentifier(i._1.getValue, i._1.getQualifier.getClassid, i._2)) + .distinct + .toList + } + + def generateScholixResourceFromResult(result: Result): ScholixResource = { + + if (result.getInstance() == null || result.getInstance().size() == 0) + return null + + if (result.getPid == null || result.getPid.isEmpty) + return null + + val r = new ScholixResource + r.setDnetIdentifier(result.getId) + + val persistentIdentifiers: List[ScholixIdentifier] = extractTypedIdentifierFromInstance(result) + if (persistentIdentifiers.isEmpty) + return null + + r.setIdentifier(persistentIdentifiers.asJava) + + r.setObjectType(result.getResulttype.getClassid) + + r.setObjectSubType( + result + .getInstance() + .asScala + .filter(i => i != null && i.getInstancetype != null) + .map(i => i.getInstancetype.getClassname) + .distinct + .head + ) + + if (result.getTitle != null && result.getTitle.asScala.nonEmpty) { + val titles: List[String] = result.getTitle.asScala.map(t => t.getValue).toList + if (titles.nonEmpty) + r.setTitle(titles.head) + else + return null + } + if (result.getAuthor != null && !result.getAuthor.isEmpty) { + val authors: List[ScholixEntityId] = + result.getAuthor.asScala + .map(a => { + val entity = new ScholixEntityId() + entity.setName(a.getFullname) + if (a.getPid != null && a.getPid.size() > 0) + entity.setIdentifiers( + a.getPid.asScala + .map(sp => { + val id = new ScholixIdentifier() + id.setIdentifier(sp.getValue) + id.setSchema(sp.getQualifier.getClassid) + id + }) + .take(3) + .toList + .asJava + ) + entity + }) + .toList + if (authors.nonEmpty) + r.setCreator(authors.asJava) + + } + + val dt: List[String] = result + .getInstance() + .asScala + .filter(i => i.getDateofacceptance != null) + .map(i => i.getDateofacceptance.getValue) + .toList + if (dt.nonEmpty) + r.setPublicationDate(dt.distinct.head) + + r.setPublisher( + result + .getInstance() + .asScala + .map(i => i.getHostedby) + .filter(h => !"unknown".equalsIgnoreCase(h.getValue)) + .map(h => { + val eid = new ScholixEntityId() + eid.setName(h.getValue) + val id = new ScholixIdentifier() + id.setIdentifier(h.getKey) + id.setSchema(OPENAIRE_IDENTIFIER_SCHEMA) + id.setUrl(generateDatasourceOpenAIREURLS(h.getKey)) + eid.setIdentifiers(List(id).asJava) + eid + }) + .distinct + .asJava + ) + + r.setCollectedFrom( + result.getCollectedfrom.asScala + .map(cf => { + val scf = new ScholixCollectedFrom() + scf.setProvisionMode("collected") + scf.setCompletionStatus("complete") + val eid = new ScholixEntityId() + eid.setName(cf.getValue) + val id = new ScholixIdentifier() + id.setIdentifier(cf.getKey) + id.setSchema(OPENAIRE_IDENTIFIER_SCHEMA) + id.setUrl(generateDatasourceOpenAIREURLS(cf.getKey)) + eid.setIdentifiers(List(id).asJava) + scf.setProvider(eid) + scf + }) + .asJava + ) + + r + } + + def generateScholix(relation: RelationInfo, source: ScholixResource): Scholix = { + val s: Scholix = new Scholix + s.setSource(source) + if (relation.collectedfrom != null && relation.collectedfrom.nonEmpty) + s.setLinkprovider( + relation.collectedfrom + .map(cf => { + val eid = new ScholixEntityId() + eid.setName(cf.value) + val id = new ScholixIdentifier() + id.setIdentifier(cf.key) + id.setSchema(OPENAIRE_IDENTIFIER_SCHEMA) + id.setUrl(generateDatasourceOpenAIREURLS(cf.key)) + eid.setIdentifiers(List(id).asJava) + eid + }) + .toList + .asJava + ) + else { + val eid = new ScholixEntityId() + eid.setName("OpenAIRE") + val id = new ScholixIdentifier() + id.setIdentifier("10|infrastruct_::f66f1bd369679b5b077dcdf006089556") + id.setSchema(OPENAIRE_IDENTIFIER_SCHEMA) + id.setUrl(generateDatasourceOpenAIREURLS(id.getIdentifier)) + eid.setIdentifiers(List(id).asJava) + s.setLinkprovider(List(eid).asJava) + } + s.setIdentifier(relation.id) + val semanticRelation = relations.getOrElse(relation.relclass.toLowerCase, null) + if (semanticRelation == null) + return null + s.setRelationship( + new ScholixRelationship(semanticRelation.original, "datacite", semanticRelation.inverse) + ) + s.setPublicationDate(source.getPublicationDate) + s.setPublisher(source.getPublisher) + val mockTarget = new ScholixResource + mockTarget.setDnetIdentifier(relation.target) + s.setTarget(mockTarget) + s + } + + def updateTarget(s: Scholix, t: ScholixResource): Scholix = { + + s.setTarget(t) + val spublishers: Seq[ScholixEntityId] = + if (s.getPublisher != null && !s.getPublisher.isEmpty) s.getPublisher.asScala else List() + val tpublishers: Seq[ScholixEntityId] = + if (t.getPublisher != null && !t.getPublisher.isEmpty) t.getPublisher.asScala else List() + val mergedPublishers = spublishers.union(tpublishers).distinct.take(10).toList + s.setPublisher(mergedPublishers.asJava) + s + } +} diff --git a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkCreateScholexplorerDump.scala b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkCreateScholexplorerDump.scala new file mode 100644 index 000000000..9334fc6e0 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkCreateScholexplorerDump.scala @@ -0,0 +1,130 @@ +package eu.dnetlib.dhp.sx.graph + +import eu.dnetlib.dhp.application.AbstractScalaApplication +import eu.dnetlib.dhp.schema.oaf.{ + KeyValue, + OtherResearchProduct, + Publication, + Relation, + Result, + Software, + Dataset => OafDataset +} +import eu.dnetlib.dhp.schema.sx.scholix.{Scholix, ScholixResource} +import org.apache.spark.sql.functions.{col, concat, expr, md5} +import org.apache.spark.sql.types.StructType +import org.apache.spark.sql._ +import org.slf4j.{Logger, LoggerFactory} + +class SparkCreateScholexplorerDump(propertyPath: String, args: Array[String], log: Logger) + extends AbstractScalaApplication(propertyPath, args, log: Logger) { + + /** Here all the spark applications runs this method + * where the whole logic of the spark node is defined + */ + override def run(): Unit = { + val sourcePath = parser.get("sourcePath") + log.info("sourcePath: {}", sourcePath) + val targetPath = parser.get("targetPath") + log.info("targetPath: {}", targetPath) + generateBidirectionalRelations(sourcePath, targetPath, spark) + generateScholixResource(sourcePath, targetPath, spark) + generateScholix(targetPath, spark) + } + + def generateScholixResource(inputPath: String, outputPath: String, spark: SparkSession): Unit = { + val entityMap: Map[String, StructType] = Map( + "publication" -> Encoders.bean(classOf[Publication]).schema, + "dataset" -> Encoders.bean(classOf[OafDataset]).schema, + "software" -> Encoders.bean(classOf[Software]).schema, + "otherresearchproduct" -> Encoders.bean(classOf[OtherResearchProduct]).schema + ) + + implicit val scholixResourceEncoder: Encoder[ScholixResource] = Encoders.bean(classOf[ScholixResource]) + implicit val resultEncoder: Encoder[Result] = Encoders.bean(classOf[Result]) + + val resDs = spark.emptyDataset[ScholixResource] + val scholixResourceDS = entityMap.foldLeft[Dataset[ScholixResource]](resDs)((res, item) => { + println(s"adding ${item._1}") + res.union( + spark.read + .schema(item._2) + .json(s"$inputPath/${item._1}") + .as[Result] + .map(r => ScholexplorerUtils.generateScholixResourceFromResult(r)) + .filter(s => s != null) + ) + }) + scholixResourceDS.write.mode(SaveMode.Overwrite).save(s"$outputPath/resource") + } + + def generateBidirectionalRelations(inputPath: String, otuputPath: String, spark: SparkSession): Unit = { + val relSchema = Encoders.bean(classOf[Relation]).schema + + val relDF = spark.read + .schema(relSchema) + .json(s"$inputPath/relation") + .where( + "datainfo.deletedbyinference is false and source like '50%' and target like '50%' " + + "and relClass <> 'merges' and relClass <> 'isMergedIn'" + ) + .select("source", "target", "collectedfrom", "relClass") + + def invRel: String => String = { s => + ScholexplorerUtils.invRel(s) + } + + import org.apache.spark.sql.functions.udf + val inverseRelationUDF = udf(invRel) + val inverseRelation = relDF.select( + col("target").alias("source"), + col("source").alias("target"), + col("collectedfrom"), + inverseRelationUDF(col("relClass")).alias("relClass") + ) + + val bidRel = inverseRelation + .union(relDF) + .withColumn("id", md5(concat(col("source"), col("relClass"), col("target")))) + .withColumn("cf", expr("transform(collectedfrom, x -> struct(x.key, x.value))")) + .drop("collectedfrom") + .withColumnRenamed("cf", "collectedfrom") + .distinct() + + bidRel.write.mode(SaveMode.Overwrite).save(s"$otuputPath/relation") + + } + + def generateScholix(outputPath: String, spark: SparkSession): Unit = { + implicit val scholixResourceEncoder: Encoder[ScholixResource] = Encoders.bean(classOf[ScholixResource]) + implicit val scholixEncoder: Encoder[Scholix] = Encoders.bean(classOf[Scholix]) + + import spark.implicits._ + val relations = spark.read.load(s"$outputPath/relation").as[RelationInfo] + val resource = spark.read.load(s"$outputPath/resource").as[ScholixResource] + + val scholix_one_verse = relations + .joinWith(resource, relations("source") === resource("dnetIdentifier"), "inner") + .map(res => ScholexplorerUtils.generateScholix(res._1, res._2)) + + scholix_one_verse + .joinWith(resource, scholix_one_verse("target.dnetIdentifier") === resource("dnetIdentifier"), "inner") + .map(k => ScholexplorerUtils.updateTarget(k._1, k._2)) + .write + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(s"$outputPath/scholix") + } +} + +object SparkCreateScholexplorerDump { + val logger: Logger = LoggerFactory.getLogger(SparkCreateScholexplorerDump.getClass) + + def main(args: Array[String]): Unit = { + new SparkCreateScholexplorerDump( + log = logger, + args = args, + propertyPath = "/eu/dnetlib/dhp/sx/create_scholix_dump_params.json" + ).initialize().run() + } +} diff --git a/dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixGenerationTest.scala b/dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixGenerationTest.scala new file mode 100644 index 000000000..0a2872cb4 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixGenerationTest.scala @@ -0,0 +1,17 @@ +package eu.dnetlib.dhp.sx.graph.scholix + +import eu.dnetlib.dhp.sx.graph.SparkCreateScholexplorerDump +import org.apache.spark.sql.SparkSession +import org.junit.jupiter.api.Test + +class ScholixGenerationTest { + + @Test + def generateScholix(): Unit = { + val spark: SparkSession = SparkSession.builder().master("local[*]").getOrCreate() + val app = new SparkCreateScholexplorerDump(null, null, null) +// app.generateScholixResource("/home/sandro/Downloads/scholix_sample/", "/home/sandro/Downloads/scholix/", spark) +// app.generateBidirectionalRelations("/home/sandro/Downloads/scholix_sample/", "/home/sandro/Downloads/scholix/", spark) + app.generateScholix("/home/sandro/Downloads/scholix/", spark) + } +} diff --git a/pom.xml b/pom.xml index d3db1d3d4..9f6f1f2a9 100644 --- a/pom.xml +++ b/pom.xml @@ -960,7 +960,7 @@ 1.1.3 1.7 1.0.7 - [6.1.1] + [6.1.2-SNAPSHOT] cdh5.9.2 3.5 11.0.2 From 052c6aac9d2dd96d37d75120890aa4dc4647a19b Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Fri, 26 Apr 2024 16:03:04 +0200 Subject: [PATCH 26/97] formatted code --- .../dhp/collection/crossref/Crossref2Oaf.scala | 1 + .../dnetlib/dhp/collection/crossref/issn_pub.json | 4 ---- .../collection/crossref/CrossrefMappingTest.scala | 13 ++++++++++++- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/crossref/Crossref2Oaf.scala b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/crossref/Crossref2Oaf.scala index 44c82e256..c4aa64fd4 100644 --- a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/crossref/Crossref2Oaf.scala +++ b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/crossref/Crossref2Oaf.scala @@ -1025,6 +1025,7 @@ case object Crossref2Oaf { tp._1 match { case "electronic" => journal.setIssnOnline(tp._2) case "print" => journal.setIssnPrinted(tp._2) + case _ => } }) } diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/crossref/issn_pub.json b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/crossref/issn_pub.json index 2a9e391df..2f1af2a6e 100644 --- a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/crossref/issn_pub.json +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/crossref/issn_pub.json @@ -789,10 +789,6 @@ "value": "2227-9717", "type": "electronic" }, - { - "value": "VALUE", - "type": "PIPPO" - }, { "value": "1063-4584", "type": "pu" diff --git a/dhp-workflows/dhp-aggregation/src/test/scala/eu/dnetlib/dhp/collection/crossref/CrossrefMappingTest.scala b/dhp-workflows/dhp-aggregation/src/test/scala/eu/dnetlib/dhp/collection/crossref/CrossrefMappingTest.scala index ed43bb1a1..c3ea884eb 100644 --- a/dhp-workflows/dhp-aggregation/src/test/scala/eu/dnetlib/dhp/collection/crossref/CrossrefMappingTest.scala +++ b/dhp-workflows/dhp-aggregation/src/test/scala/eu/dnetlib/dhp/collection/crossref/CrossrefMappingTest.scala @@ -2,7 +2,9 @@ package eu.dnetlib.dhp.collection.crossref import com.fasterxml.jackson.databind.ObjectMapper import eu.dnetlib.dhp.aggregation.AbstractVocabularyTest -import org.junit.jupiter.api.BeforeEach +import eu.dnetlib.dhp.collection.crossref.Crossref2Oaf.TransformationType +import org.apache.commons.io.IOUtils +import org.junit.jupiter.api.{BeforeEach, Test} import org.junit.jupiter.api.extension.ExtendWith import org.mockito.junit.jupiter.MockitoExtension import org.slf4j.{Logger, LoggerFactory} @@ -18,4 +20,13 @@ class CrossrefMappingTest extends AbstractVocabularyTest { super.setUpVocabulary() } + @Test + def mappingRecord(): Unit = { + val input = + IOUtils.toString(getClass.getResourceAsStream("/eu/dnetlib/dhp/collection/crossref/issn_pub.json"), "utf-8") + + println(Crossref2Oaf.convert(input, vocabularies, TransformationType.All)) + + } + } From 133ead1e3ef86be422783eddf9fd3e46738b6e02 Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Mon, 29 Apr 2024 09:00:30 +0200 Subject: [PATCH 27/97] updated new version of scholexplorer Generation --- .../dhp/sx/graph/SparkCreateScholexplorerDump.scala | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkCreateScholexplorerDump.scala b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkCreateScholexplorerDump.scala index 9334fc6e0..1211dcc78 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkCreateScholexplorerDump.scala +++ b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkCreateScholexplorerDump.scala @@ -107,9 +107,13 @@ class SparkCreateScholexplorerDump(propertyPath: String, args: Array[String], lo .joinWith(resource, relations("source") === resource("dnetIdentifier"), "inner") .map(res => ScholexplorerUtils.generateScholix(res._1, res._2)) + val resourceTarget = relations + .joinWith(resource, relations("target") === resource("dnetIdentifier"), "inner") + .map(res => (res._1.id, res._2))(Encoders.tuple(Encoders.STRING, Encoders.kryo(classOf[ScholixResource]))) + scholix_one_verse - .joinWith(resource, scholix_one_verse("target.dnetIdentifier") === resource("dnetIdentifier"), "inner") - .map(k => ScholexplorerUtils.updateTarget(k._1, k._2)) + .joinWith(resourceTarget, scholix_one_verse("identifier") === resourceTarget("_1"), "inner") + .map(k => ScholexplorerUtils.updateTarget(k._1, k._2._2)) .write .mode(SaveMode.Overwrite) .option("compression", "gzip") From 2615136efc0a86ceb92f82f2380e68230330ef83 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Tue, 30 Apr 2024 11:58:42 +0200 Subject: [PATCH 28/97] added a retry mechanism --- .../collection/plugin/rest/RestIterator.java | 379 +++++++++--------- 1 file changed, 200 insertions(+), 179 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java index 1107bcf46..c13f29806 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java @@ -18,7 +18,11 @@ import javax.xml.transform.TransformerConfigurationException; import javax.xml.transform.TransformerFactory; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; -import javax.xml.xpath.*; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpression; +import javax.xml.xpath.XPathExpressionException; +import javax.xml.xpath.XPathFactory; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; @@ -35,7 +39,7 @@ import eu.dnetlib.dhp.common.collection.CollectorException; import eu.dnetlib.dhp.common.collection.HttpClientParams; /** - * log.info(...) equal to log.trace(...) in the application-logs + * log.info(...) equal to log.trace(...) in the application-logs *

* known bug: at resumptionType 'discover' if the (resultTotal % resultSizeValue) == 0 the collecting fails -> change the resultSizeValue * @@ -47,6 +51,7 @@ public class RestIterator implements Iterator { private static final Logger log = LoggerFactory.getLogger(RestIterator.class); public static final String UTF_8 = "UTF-8"; + private static final int MAX_ATTEMPTS = 5; private final HttpClientParams clientParams; @@ -60,8 +65,8 @@ public class RestIterator implements Iterator { private final int resultSizeValue; private int resumptionInt = 0; // integer resumption token (first record to harvest) private int resultTotal = -1; - private String resumptionStr = Integer.toString(resumptionInt); // string resumption token (first record to harvest - // or token scanned from results) + private String resumptionStr = Integer.toString(this.resumptionInt); // string resumption token (first record to harvest + // or token scanned from results) private InputStream resultStream; private Transformer transformer; private XPath xpath; @@ -73,75 +78,75 @@ public class RestIterator implements Iterator { private final String querySize; private final String authMethod; private final String authToken; - private final Queue recordQueue = new PriorityBlockingQueue(); + private final Queue recordQueue = new PriorityBlockingQueue<>(); private int discoverResultSize = 0; private int pagination = 1; /* - * While resultFormatValue is added to the request parameter, this is used to say that the results are retrieved in - * json. useful for cases when the target API expects a resultFormatValue != json, but the results are returned in - * json. An example is the EU Open Data Portal API: resultFormatValue=standard, results are in json format. + * While resultFormatValue is added to the request parameter, this is used to say that the results are retrieved in json. useful for + * cases when the target API expects a resultFormatValue != json, but the results are returned in json. An example is the EU Open Data + * Portal API: resultFormatValue=standard, results are in json format. */ private final String resultOutputFormat; - /** RestIterator class - * compatible to version 1.3.33 + /** + * RestIterator class compatible to version 1.3.33 */ public RestIterator( - final HttpClientParams clientParams, - final String baseUrl, - final String resumptionType, - final String resumptionParam, - final String resumptionXpath, - final String resultTotalXpath, - final String resultFormatParam, - final String resultFormatValue, - final String resultSizeParam, - final String resultSizeValueStr, - final String queryParams, - final String entityXpath, - final String authMethod, - final String authToken, - final String resultOutputFormat) { + final HttpClientParams clientParams, + final String baseUrl, + final String resumptionType, + final String resumptionParam, + final String resumptionXpath, + final String resultTotalXpath, + final String resultFormatParam, + final String resultFormatValue, + final String resultSizeParam, + final String resultSizeValueStr, + final String queryParams, + final String entityXpath, + final String authMethod, + final String authToken, + final String resultOutputFormat) { this.clientParams = clientParams; this.baseUrl = baseUrl; this.resumptionType = resumptionType; this.resumptionParam = resumptionParam; this.resultFormatValue = resultFormatValue; - this.resultSizeValue = Integer.valueOf(resultSizeValueStr); + this.resultSizeValue = Integer.parseInt(resultSizeValueStr); this.queryParams = queryParams; this.authMethod = authMethod; this.authToken = authToken; this.resultOutputFormat = resultOutputFormat; - queryFormat = StringUtils.isNotBlank(resultFormatParam) ? "&" + resultFormatParam + "=" + resultFormatValue - : ""; - querySize = StringUtils.isNotBlank(resultSizeParam) ? "&" + resultSizeParam + "=" + resultSizeValueStr : ""; + this.queryFormat = StringUtils.isNotBlank(resultFormatParam) ? "&" + resultFormatParam + "=" + resultFormatValue + : ""; + this.querySize = StringUtils.isNotBlank(resultSizeParam) ? "&" + resultSizeParam + "=" + resultSizeValueStr : ""; try { initXmlTransformation(resultTotalXpath, resumptionXpath, entityXpath); - } catch (Exception e) { + } catch (final Exception e) { throw new IllegalStateException("xml transformation init failed: " + e.getMessage()); } initQueue(); } - private void initXmlTransformation(String resultTotalXpath, String resumptionXpath, String entityXpath) - throws TransformerConfigurationException, XPathExpressionException { + private void initXmlTransformation(final String resultTotalXpath, final String resumptionXpath, final String entityXpath) + throws TransformerConfigurationException, XPathExpressionException { final TransformerFactory factory = TransformerFactory.newInstance(); - transformer = factory.newTransformer(); - transformer.setOutputProperty(OutputKeys.INDENT, "yes"); - transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "3"); - xpath = XPathFactory.newInstance().newXPath(); - xprResultTotalPath = xpath.compile(resultTotalXpath); - xprResumptionPath = xpath.compile(StringUtils.isBlank(resumptionXpath) ? "/" : resumptionXpath); - xprEntity = xpath.compile(entityXpath); + this.transformer = factory.newTransformer(); + this.transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + this.transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "3"); + this.xpath = XPathFactory.newInstance().newXPath(); + this.xprResultTotalPath = this.xpath.compile(resultTotalXpath); + this.xprResumptionPath = this.xpath.compile(StringUtils.isBlank(resumptionXpath) ? "/" : resumptionXpath); + this.xprEntity = this.xpath.compile(entityXpath); } private void initQueue() { - query = baseUrl + "?" + queryParams + querySize + queryFormat; - log.info("REST calls starting with {}", query); + this.query = this.baseUrl + "?" + this.queryParams + this.querySize + this.queryFormat; + log.info("REST calls starting with {}", this.query); } private void disconnect() { @@ -150,127 +155,140 @@ public class RestIterator implements Iterator { /* * (non-Javadoc) + * * @see java.util.Iterator#hasNext() */ @Override public boolean hasNext() { - if (recordQueue.isEmpty() && query.isEmpty()) { + if (this.recordQueue.isEmpty() && this.query.isEmpty()) { disconnect(); return false; - } else { - return true; } + return true; } /* * (non-Javadoc) + * * @see java.util.Iterator#next() */ @Override public String next() { - synchronized (recordQueue) { - while (recordQueue.isEmpty() && !query.isEmpty()) { + synchronized (this.recordQueue) { + while (this.recordQueue.isEmpty() && !this.query.isEmpty()) { try { - query = downloadPage(query); - } catch (CollectorException e) { + this.query = downloadPage(this.query, 0); + } catch (final CollectorException e) { log.debug("CollectorPlugin.next()-Exception: {}", e); throw new RuntimeException(e); } } - return recordQueue.poll(); + return this.recordQueue.poll(); } } /* - * download page and return nextQuery + * download page and return nextQuery (with number of attempt) */ - private String downloadPage(String query) throws CollectorException { - String resultJson; - String resultXml = ""; - String nextQuery = ""; - String emptyXml = resultXml + "<" + JsonUtils.XML_WRAP_TAG + ">"; - Node resultNode = null; - NodeList nodeList = null; - String qUrlArgument = ""; - int urlOldResumptionSize = 0; - InputStream theHttpInputStream; + private String downloadPage(String query, final int attempt) throws CollectorException { - // check if cursor=* is initial set otherwise add it to the queryParam URL - if (resumptionType.equalsIgnoreCase("deep-cursor")) { - log.debug("check resumptionType deep-cursor and check cursor=*?{}", query); - if (!query.contains("&cursor=")) { - query += "&cursor=*"; + if (attempt > MAX_ATTEMPTS) { throw new CollectorException("Max Number of attempts reached, query:" + query); } + + if (attempt > 0) { + final int delay = (attempt * 5000); + log.debug("Attempt {} with delay {}", attempt, delay); + try { + Thread.sleep(delay); + } catch (final InterruptedException e) { + new CollectorException(e); } } try { - log.info("requestig URL [{}]", query); + String resultJson; + String resultXml = ""; + String nextQuery = ""; + final String emptyXml = resultXml + "<" + JsonUtils.XML_WRAP_TAG + ">"; + Node resultNode = null; + NodeList nodeList = null; + String qUrlArgument = ""; + int urlOldResumptionSize = 0; + InputStream theHttpInputStream; - URL qUrl = new URL(query); - log.debug("authMethod: {}", authMethod); - if ("bearer".equalsIgnoreCase(this.authMethod)) { - log.trace("authMethod before inputStream: {}", resultXml); - HttpURLConnection conn = (HttpURLConnection) qUrl.openConnection(); - conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + authToken); - conn.setRequestProperty(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.getMimeType()); - conn.setRequestMethod("GET"); - theHttpInputStream = conn.getInputStream(); - } else if (BASIC.equalsIgnoreCase(this.authMethod)) { - log.trace("authMethod before inputStream: {}", resultXml); - HttpURLConnection conn = (HttpURLConnection) qUrl.openConnection(); - conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Basic " + authToken); - conn.setRequestProperty(HttpHeaders.ACCEPT, ContentType.APPLICATION_XML.getMimeType()); - conn.setRequestMethod("GET"); - theHttpInputStream = conn.getInputStream(); - } else { - theHttpInputStream = qUrl.openStream(); - } - - resultStream = theHttpInputStream; - if ("json".equals(resultOutputFormat)) { - resultJson = IOUtils.toString(resultStream, StandardCharsets.UTF_8); - resultXml = JsonUtils.convertToXML(resultJson); - resultStream = IOUtils.toInputStream(resultXml, UTF_8); - } - - if (!(emptyXml).equalsIgnoreCase(resultXml)) { - resultNode = (Node) xpath.evaluate("/", new InputSource(resultStream), XPathConstants.NODE); - nodeList = (NodeList) xprEntity.evaluate(resultNode, XPathConstants.NODESET); - log.debug("nodeList.length: {}", nodeList.getLength()); - for (int i = 0; i < nodeList.getLength(); i++) { - StringWriter sw = new StringWriter(); - transformer.transform(new DOMSource(nodeList.item(i)), new StreamResult(sw)); - String toEnqueue = sw.toString(); - if (toEnqueue == null || StringUtils.isBlank(toEnqueue) || emptyXml.equalsIgnoreCase(toEnqueue)) { - log.warn("The following record resulted in empty item for the feeding queue: {}", resultXml); - } else { - recordQueue.add(sw.toString()); - } + // check if cursor=* is initial set otherwise add it to the queryParam URL + if ("deep-cursor".equalsIgnoreCase(this.resumptionType)) { + log.debug("check resumptionType deep-cursor and check cursor=*?{}", query); + if (!query.contains("&cursor=")) { + query += "&cursor=*"; } - } else { - log.warn("resultXml is equal with emptyXml"); } - resumptionInt += resultSizeValue; + try { + log.info("requesting URL [{}]", query); - switch (resumptionType.toLowerCase()) { + final URL qUrl = new URL(query); + log.debug("authMethod: {}", this.authMethod); + if ("bearer".equalsIgnoreCase(this.authMethod)) { + log.trace("authMethod before inputStream: {}", resultXml); + final HttpURLConnection conn = (HttpURLConnection) qUrl.openConnection(); + conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + this.authToken); + conn.setRequestProperty(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.getMimeType()); + conn.setRequestMethod("GET"); + theHttpInputStream = conn.getInputStream(); + } else if (this.BASIC.equalsIgnoreCase(this.authMethod)) { + log.trace("authMethod before inputStream: {}", resultXml); + final HttpURLConnection conn = (HttpURLConnection) qUrl.openConnection(); + conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Basic " + this.authToken); + conn.setRequestProperty(HttpHeaders.ACCEPT, ContentType.APPLICATION_XML.getMimeType()); + conn.setRequestMethod("GET"); + theHttpInputStream = conn.getInputStream(); + } else { + theHttpInputStream = qUrl.openStream(); + } + + this.resultStream = theHttpInputStream; + if ("json".equals(this.resultOutputFormat)) { + resultJson = IOUtils.toString(this.resultStream, StandardCharsets.UTF_8); + resultXml = JsonUtils.convertToXML(resultJson); + this.resultStream = IOUtils.toInputStream(resultXml, UTF_8); + } + + if (!(emptyXml).equalsIgnoreCase(resultXml)) { + resultNode = (Node) this.xpath.evaluate("/", new InputSource(this.resultStream), XPathConstants.NODE); + nodeList = (NodeList) this.xprEntity.evaluate(resultNode, XPathConstants.NODESET); + log.debug("nodeList.length: {}", nodeList.getLength()); + for (int i = 0; i < nodeList.getLength(); i++) { + final StringWriter sw = new StringWriter(); + this.transformer.transform(new DOMSource(nodeList.item(i)), new StreamResult(sw)); + final String toEnqueue = sw.toString(); + if ((toEnqueue == null) || StringUtils.isBlank(toEnqueue) || emptyXml.equalsIgnoreCase(toEnqueue)) { + log.warn("The following record resulted in empty item for the feeding queue: {}", resultXml); + } else { + this.recordQueue.add(sw.toString()); + } + } + } else { + log.warn("resultXml is equal with emptyXml"); + } + + this.resumptionInt += this.resultSizeValue; + + switch (this.resumptionType.toLowerCase()) { case "scan": // read of resumptionToken , evaluate next results, e.g. OAI, iterate over items - resumptionStr = xprResumptionPath.evaluate(resultNode); + this.resumptionStr = this.xprResumptionPath.evaluate(resultNode); break; case "count": // begin at one step for all records, iterate over items - resumptionStr = Integer.toString(resumptionInt); + this.resumptionStr = Integer.toString(this.resumptionInt); break; case "discover": // size of result items unknown, iterate over items (for openDOAR - 201808) - if (resultSizeValue < 2) { - throw new CollectorException("Mode: discover, Param 'resultSizeValue' is less than 2"); - } + if (this.resultSizeValue < 2) { throw new CollectorException("Mode: discover, Param 'resultSizeValue' is less than 2"); } qUrlArgument = qUrl.getQuery(); - String[] arrayQUrlArgument = qUrlArgument.split("&"); - for (String arrayUrlArgStr : arrayQUrlArgument) { - if (arrayUrlArgStr.startsWith(resumptionParam)) { - String[] resumptionKeyValue = arrayUrlArgStr.split("="); + final String[] arrayQUrlArgument = qUrlArgument.split("&"); + for (final String arrayUrlArgStr : arrayQUrlArgument) { + if (arrayUrlArgStr.startsWith(this.resumptionParam)) { + final String[] resumptionKeyValue = arrayUrlArgStr.split("="); if (isInteger(resumptionKeyValue[1])) { urlOldResumptionSize = Integer.parseInt(resumptionKeyValue[1]); log.debug("discover OldResumptionSize from Url (int): {}", urlOldResumptionSize); @@ -281,101 +299,104 @@ public class RestIterator implements Iterator { } if (((emptyXml).equalsIgnoreCase(resultXml)) - || ((nodeList != null) && (nodeList.getLength() < resultSizeValue))) { + || ((nodeList != null) && (nodeList.getLength() < this.resultSizeValue))) { // resumptionStr = ""; if (nodeList != null) { - discoverResultSize += nodeList.getLength(); + this.discoverResultSize += nodeList.getLength(); } - resultTotal = discoverResultSize; + this.resultTotal = this.discoverResultSize; } else { - resumptionStr = Integer.toString(resumptionInt); - resultTotal = resumptionInt + 1; + this.resumptionStr = Integer.toString(this.resumptionInt); + this.resultTotal = this.resumptionInt + 1; if (nodeList != null) { - discoverResultSize += nodeList.getLength(); + this.discoverResultSize += nodeList.getLength(); } } - log.info("discoverResultSize: {}", discoverResultSize); + log.info("discoverResultSize: {}", this.discoverResultSize); break; case "pagination": case "page": // pagination, iterate over page numbers - pagination += 1; + this.pagination += 1; if (nodeList != null) { - discoverResultSize += nodeList.getLength(); + this.discoverResultSize += nodeList.getLength(); } else { - resultTotal = discoverResultSize; - pagination = discoverResultSize; + this.resultTotal = this.discoverResultSize; + this.pagination = this.discoverResultSize; } - resumptionInt = pagination; - resumptionStr = Integer.toString(resumptionInt); + this.resumptionInt = this.pagination; + this.resumptionStr = Integer.toString(this.resumptionInt); break; case "deep-cursor": // size of result items unknown, iterate over items (for supporting deep cursor in - // solr) + // solr) // isn't relevant -- if (resultSizeValue < 2) {throw new CollectorServiceException("Mode: // deep-cursor, Param 'resultSizeValue' is less than 2");} - resumptionStr = encodeValue(xprResumptionPath.evaluate(resultNode)); - queryParams = queryParams.replace("&cursor=*", ""); + this.resumptionStr = encodeValue(this.xprResumptionPath.evaluate(resultNode)); + this.queryParams = this.queryParams.replace("&cursor=*", ""); // terminating if length of nodeList is 0 - if ((nodeList != null) && (nodeList.getLength() < discoverResultSize)) { - resumptionInt += (nodeList.getLength() + 1 - resultSizeValue); + if ((nodeList != null) && (nodeList.getLength() < this.discoverResultSize)) { + this.resumptionInt += ((nodeList.getLength() + 1) - this.resultSizeValue); } else { - resumptionInt += (nodeList.getLength() - resultSizeValue); // subtract the resultSizeValue - // because the iteration is over - // real length and the - // resultSizeValue is added before - // the switch() + this.resumptionInt += (nodeList.getLength() - this.resultSizeValue); // subtract the resultSizeValue + // because the iteration is over + // real length and the + // resultSizeValue is added before + // the switch() } - discoverResultSize = nodeList.getLength(); + this.discoverResultSize = nodeList.getLength(); log - .debug( - "downloadPage().deep-cursor: resumptionStr=" + resumptionStr + " ; queryParams=" - + queryParams + " resumptionLengthIncreased: " + resumptionInt); + .debug("downloadPage().deep-cursor: resumptionStr=" + this.resumptionStr + " ; queryParams=" + + this.queryParams + " resumptionLengthIncreased: " + this.resumptionInt); break; default: // otherwise: abort // resultTotal = resumptionInt; break; + } + + } catch (final Exception e) { + log.error(e.getMessage(), e); + throw new IllegalStateException("collection failed: " + e.getMessage()); } - } catch (Exception e) { - log.error(e.getMessage(), e); - throw new IllegalStateException("collection failed: " + e.getMessage()); - } - - try { - if (resultTotal == -1) { - resultTotal = Integer.parseInt(xprResultTotalPath.evaluate(resultNode)); - if (resumptionType.equalsIgnoreCase("page") && !BASIC.equalsIgnoreCase(authMethod)) { - resultTotal += 1; - } // to correct the upper bound - log.info("resultTotal was -1 is now: " + resultTotal); + try { + if (this.resultTotal == -1) { + this.resultTotal = Integer.parseInt(this.xprResultTotalPath.evaluate(resultNode)); + if ("page".equalsIgnoreCase(this.resumptionType) && !this.BASIC.equalsIgnoreCase(this.authMethod)) { + this.resultTotal += 1; + } // to correct the upper bound + log.info("resultTotal was -1 is now: " + this.resultTotal); + } + } catch (final Exception e) { + log.error(e.getMessage(), e); + throw new IllegalStateException("downloadPage resultTotal couldn't parse: " + e.getMessage()); } - } catch (Exception e) { - log.error(e.getMessage(), e); - throw new IllegalStateException("downloadPage resultTotal couldn't parse: " + e.getMessage()); + log.debug("resultTotal: " + this.resultTotal); + log.debug("resInt: " + this.resumptionInt); + if (this.resumptionInt <= this.resultTotal) { + nextQuery = this.baseUrl + "?" + this.queryParams + this.querySize + "&" + this.resumptionParam + "=" + this.resumptionStr + + this.queryFormat; + } else { + nextQuery = ""; + // if (resumptionType.toLowerCase().equals("deep-cursor")) { resumptionInt -= 1; } // correct the + // resumptionInt and prevent a NullPointer Exception at mdStore + } + log.debug("nextQueryUrl: " + nextQuery); + return nextQuery; + } catch (final Throwable e) { + log.warn(e.getMessage(), e); + return downloadPage(query, attempt + 1); } - log.debug("resultTotal: " + resultTotal); - log.debug("resInt: " + resumptionInt); - if (resumptionInt <= resultTotal) { - nextQuery = baseUrl + "?" + queryParams + querySize + "&" + resumptionParam + "=" + resumptionStr - + queryFormat; - } else { - nextQuery = ""; - // if (resumptionType.toLowerCase().equals("deep-cursor")) { resumptionInt -= 1; } // correct the - // resumptionInt and prevent a NullPointer Exception at mdStore - } - log.debug("nextQueryUrl: " + nextQuery); - return nextQuery; } - private boolean isInteger(String s) { + private boolean isInteger(final String s) { boolean isValidInteger = false; try { Integer.parseInt(s); @@ -383,7 +404,7 @@ public class RestIterator implements Iterator { // s is a valid integer isValidInteger = true; - } catch (NumberFormatException ex) { + } catch (final NumberFormatException ex) { // s is not an integer } @@ -391,20 +412,20 @@ public class RestIterator implements Iterator { } // Method to encode a string value using `UTF-8` encoding scheme - private String encodeValue(String value) { + private String encodeValue(final String value) { try { return URLEncoder.encode(value, StandardCharsets.UTF_8.toString()); - } catch (UnsupportedEncodingException ex) { + } catch (final UnsupportedEncodingException ex) { throw new RuntimeException(ex.getCause()); } } public String getResultFormatValue() { - return resultFormatValue; + return this.resultFormatValue; } public String getResultOutputFormat() { - return resultOutputFormat; + return this.resultOutputFormat; } } From 50c18f7a0b05940a476ed2ef900e15c329b7a398 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 30 Apr 2024 12:34:16 +0200 Subject: [PATCH 29/97] [dedup wf] revised memory settings to address the increased volume of input contents --- .../dedup/consistency/oozie_app/workflow.xml | 2 + .../dhp/oa/dedup/scan/oozie_app/workflow.xml | 46 ++++++------------- 2 files changed, 16 insertions(+), 32 deletions(-) diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/workflow.xml b/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/workflow.xml index 306229e79..46dc71c2c 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/workflow.xml @@ -102,6 +102,8 @@ --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.sql.shuffle.partitions=15000 + --conf spark.network.timeout=300s + --conf spark.shuffle.registration.timeout=50000 --graphBasePath${graphBasePath} --graphOutputPath${graphOutputPath} diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/scan/oozie_app/workflow.xml b/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/scan/oozie_app/workflow.xml index 49a331def..ff37c5074 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/scan/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/scan/oozie_app/workflow.xml @@ -33,16 +33,14 @@ max number of elements in a connected component - sparkDriverMemory - memory for driver process + sparkResourceOpts + --executor-memory=6G --conf spark.executor.memoryOverhead=4G --executor-cores=6 --driver-memory=8G --driver-cores=4 + spark resource options - sparkExecutorMemory - memory for individual executor - - - sparkExecutorCores - number of cores used by single executor + sparkResourceOptsCreateMergeRel + --executor-memory=6G --conf spark.executor.memoryOverhead=4G --executor-cores=6 --driver-memory=8G --driver-cores=4 + spark resource options oozieActionShareLibForSpark2 @@ -119,9 +117,7 @@ eu.dnetlib.dhp.oa.dedup.SparkCreateSimRels dhp-dedup-openaire-${projectVersion}.jar - --executor-memory=${sparkExecutorMemory} - --executor-cores=${sparkExecutorCores} - --driver-memory=${sparkDriverMemory} + ${sparkResourceOpts} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -146,9 +142,7 @@ eu.dnetlib.dhp.oa.dedup.SparkWhitelistSimRels dhp-dedup-openaire-${projectVersion}.jar - --executor-memory=${sparkExecutorMemory} - --executor-cores=${sparkExecutorCores} - --driver-memory=${sparkDriverMemory} + ${sparkResourceOpts} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -174,9 +168,7 @@ eu.dnetlib.dhp.oa.dedup.SparkCreateMergeRels dhp-dedup-openaire-${projectVersion}.jar - --executor-memory=${sparkExecutorMemory} - --executor-cores=${sparkExecutorCores} - --driver-memory=${sparkDriverMemory} + ${sparkResourceOptsCreateMergeRel} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -203,9 +195,7 @@ eu.dnetlib.dhp.oa.dedup.SparkCreateDedupRecord dhp-dedup-openaire-${projectVersion}.jar - --executor-memory=${sparkExecutorMemory} - --executor-cores=${sparkExecutorCores} - --driver-memory=${sparkDriverMemory} + ${sparkResourceOpts} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -230,9 +220,7 @@ eu.dnetlib.dhp.oa.dedup.SparkCopyOpenorgsMergeRels dhp-dedup-openaire-${projectVersion}.jar - --executor-memory=${sparkExecutorMemory} - --executor-cores=${sparkExecutorCores} - --driver-memory=${sparkDriverMemory} + ${sparkResourceOpts} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -257,9 +245,7 @@ eu.dnetlib.dhp.oa.dedup.SparkCreateOrgsDedupRecord dhp-dedup-openaire-${projectVersion}.jar - --executor-memory=${sparkExecutorMemory} - --executor-cores=${sparkExecutorCores} - --driver-memory=${sparkDriverMemory} + ${sparkResourceOpts} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -283,9 +269,7 @@ eu.dnetlib.dhp.oa.dedup.SparkUpdateEntity dhp-dedup-openaire-${projectVersion}.jar - --executor-memory=${sparkExecutorMemory} - --executor-cores=${sparkExecutorCores} - --driver-memory=${sparkDriverMemory} + ${sparkResourceOpts} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -309,9 +293,7 @@ eu.dnetlib.dhp.oa.dedup.SparkCopyRelationsNoOpenorgs dhp-dedup-openaire-${projectVersion}.jar - --executor-memory=${sparkExecutorMemory} - --executor-cores=${sparkExecutorCores} - --driver-memory=${sparkDriverMemory} + ${sparkResourceOpts} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} From e96c2c1606d2ddf4b1f6c0c3f18af7b7de4f57db Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 30 Apr 2024 16:23:25 +0200 Subject: [PATCH 30/97] [ranking wf] set spark.executor.memoryOverhead to fine tune the resource consumption --- .../graph/impact_indicators/oozie_app/workflow.xml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/dhp-workflows/dhp-impact-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/impact_indicators/oozie_app/workflow.xml b/dhp-workflows/dhp-impact-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/impact_indicators/oozie_app/workflow.xml index e43e7cf14..70f5f8d2a 100644 --- a/dhp-workflows/dhp-impact-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/impact_indicators/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-impact-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/impact_indicators/oozie_app/workflow.xml @@ -71,6 +71,7 @@ --executor-memory=${sparkHighExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkHighDriverMemory} + --conf spark.executor.memoryOverhead=${sparkHighExecutorMemory} --conf spark.sql.shuffle.partitions=${sparkShufflePartitions} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} @@ -108,6 +109,7 @@ --executor-memory=${sparkHighExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkNormalDriverMemory} + --conf spark.executor.memoryOverhead=${sparkHighExecutorMemory} --conf spark.sql.shuffle.partitions=${sparkShufflePartitions} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} @@ -141,6 +143,7 @@ --executor-memory=${sparkHighExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkNormalDriverMemory} + --conf spark.executor.memoryOverhead=${sparkHighExecutorMemory} --conf spark.sql.shuffle.partitions=${sparkShufflePartitions} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} @@ -176,6 +179,7 @@ --executor-memory=${sparkHighExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkNormalDriverMemory} + --conf spark.executor.memoryOverhead=${sparkHighExecutorMemory} --conf spark.sql.shuffle.partitions=${sparkShufflePartitions} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} @@ -209,6 +213,7 @@ --executor-memory=${sparkHighExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkNormalDriverMemory} + --conf spark.executor.memoryOverhead=${sparkHighExecutorMemory} --conf spark.sql.shuffle.partitions=${sparkShufflePartitions} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} @@ -245,6 +250,7 @@ --executor-memory=${sparkHighExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkNormalDriverMemory} + --conf spark.executor.memoryOverhead=${sparkHighExecutorMemory} --conf spark.sql.shuffle.partitions=${sparkShufflePartitions} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} @@ -315,6 +321,7 @@ --executor-memory=${sparkNormalExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkNormalDriverMemory} + --conf spark.executor.memoryOverhead=${sparkNormalExecutorMemory} --conf spark.sql.shuffle.partitions=${sparkShufflePartitions} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} @@ -361,6 +368,7 @@ --executor-memory=${sparkNormalExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkNormalDriverMemory} + --conf spark.executor.memoryOverhead=${sparkNormalExecutorMemory} --conf spark.sql.shuffle.partitions=${sparkShufflePartitions} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} @@ -409,6 +417,7 @@ --executor-memory=${sparkHighExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkHighDriverMemory} + --conf spark.executor.memoryOverhead=${sparkHighExecutorMemory} --conf spark.sql.shuffle.partitions=${sparkShufflePartitions} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} @@ -444,6 +453,7 @@ --executor-memory=${sparkHighExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkHighDriverMemory} + --conf spark.executor.memoryOverhead=${sparkHighExecutorMemory} --conf spark.sql.shuffle.partitions=${sparkShufflePartitions} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} @@ -482,6 +492,7 @@ --executor-memory=${sparkHighExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkNormalDriverMemory} + --conf spark.executor.memoryOverhead=${sparkHighExecutorMemory} --conf spark.sql.shuffle.partitions=${sparkShufflePartitions} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} @@ -533,6 +544,7 @@ --executor-memory=${sparkNormalExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkNormalDriverMemory} + --conf spark.executor.memoryOverhead=${sparkNormalExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} From 11bd89e1325ad4f4abbac118322a6f25aafb3419 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Wed, 1 May 2024 08:32:59 +0200 Subject: [PATCH 31/97] [enrichment] use sparkExecutorMemory to define also the memoryOverhead --- .../oozie_app/workflow.xml | 61 +++++-------------- 1 file changed, 15 insertions(+), 46 deletions(-) diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/wf/subworkflows/orcidtoresultfromsemrel/oozie_app/workflow.xml b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/wf/subworkflows/orcidtoresultfromsemrel/oozie_app/workflow.xml index a9642d637..ba3633e07 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/wf/subworkflows/orcidtoresultfromsemrel/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/wf/subworkflows/orcidtoresultfromsemrel/oozie_app/workflow.xml @@ -100,16 +100,12 @@ --executor-cores=${sparkExecutorCores} --executor-memory=${sparkExecutorMemory} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.dynamicAllocation.enabled=true - --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors} - --conf spark.sql.shuffle.partitions=3840 - --conf spark.speculation=false - --conf spark.hadoop.mapreduce.map.speculative=false - --conf spark.hadoop.mapreduce.reduce.speculative=false + --conf spark.sql.shuffle.partitions=8000 --sourcePath${sourcePath} --hive_metastore_uris${hive_metastore_uris} @@ -132,12 +128,11 @@ --executor-cores=${sparkExecutorCores} --executor-memory=${sparkExecutorMemory} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.dynamicAllocation.enabled=true - --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors} --sourcePath${sourcePath} --hive_metastore_uris${hive_metastore_uris} @@ -160,12 +155,11 @@ --executor-cores=${sparkExecutorCores} --executor-memory=${sparkExecutorMemory} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.dynamicAllocation.enabled=true - --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors} --sourcePath${sourcePath} --hive_metastore_uris${hive_metastore_uris} @@ -188,12 +182,11 @@ --executor-cores=${sparkExecutorCores} --executor-memory=${sparkExecutorMemory} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.dynamicAllocation.enabled=true - --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors} --sourcePath${sourcePath} --hive_metastore_uris${hive_metastore_uris} @@ -218,12 +211,11 @@ --executor-cores=${sparkExecutorCores} --executor-memory=${sparkExecutorMemory} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.dynamicAllocation.enabled=true - --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors} --sourcePath${workingDir}/orcid/targetOrcidAssoc --outputPath${workingDir}/orcid/mergedOrcidAssoc @@ -247,19 +239,14 @@ eu.dnetlib.dhp.orcidtoresultfromsemrel.SparkOrcidToResultFromSemRelJob dhp-enrichment-${projectVersion}.jar - --executor-cores=4 - --executor-memory=4G + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} --driver-memory=${sparkDriverMemory} - --conf spark.executor.memoryOverhead=5G + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.dynamicAllocation.enabled=true - --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors} - --conf spark.speculation=false - --conf spark.hadoop.mapreduce.map.speculative=false - --conf spark.hadoop.mapreduce.reduce.speculative=false --conf spark.sql.shuffle.partitions=15000 --possibleUpdatesPath${workingDir}/orcid/mergedOrcidAssoc @@ -282,15 +269,12 @@ --executor-cores=${sparkExecutorCores} --executor-memory=${sparkExecutorMemory} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.dynamicAllocation.enabled=true - --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors} - --conf spark.speculation=false - --conf spark.hadoop.mapreduce.map.speculative=false - --conf spark.hadoop.mapreduce.reduce.speculative=false + --conf spark.sql.shuffle.partitions=8000 --possibleUpdatesPath${workingDir}/orcid/mergedOrcidAssoc --sourcePath${sourcePath}/dataset @@ -312,15 +296,12 @@ --executor-cores=${sparkExecutorCores} --executor-memory=${sparkExecutorMemory} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.dynamicAllocation.enabled=true - --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors} - --conf spark.speculation=false - --conf spark.hadoop.mapreduce.map.speculative=false - --conf spark.hadoop.mapreduce.reduce.speculative=false + --conf spark.sql.shuffle.partitions=8000 --possibleUpdatesPath${workingDir}/orcid/mergedOrcidAssoc --sourcePath${sourcePath}/otherresearchproduct @@ -342,15 +323,12 @@ --executor-cores=${sparkExecutorCores} --executor-memory=${sparkExecutorMemory} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.dynamicAllocation.enabled=true - --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors} - --conf spark.speculation=false - --conf spark.hadoop.mapreduce.map.speculative=false - --conf spark.hadoop.mapreduce.reduce.speculative=false + --conf spark.sql.shuffle.partitions=4000 --possibleUpdatesPath${workingDir}/orcid/mergedOrcidAssoc --sourcePath${sourcePath}/software @@ -362,15 +340,6 @@ - - - - - - - - - From f4068de298af90e8d74463449d0df4ff2d0af55a Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Thu, 2 May 2024 09:51:33 +0200 Subject: [PATCH 32/97] code reindent + tests --- .../collection/plugin/rest/RestIterator.java | 211 ++++++++++-------- .../plugin/rest/OsfPreprintCollectorTest.java | 22 +- 2 files changed, 133 insertions(+), 100 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java index c13f29806..76af6cff1 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java @@ -65,7 +65,8 @@ public class RestIterator implements Iterator { private final int resultSizeValue; private int resumptionInt = 0; // integer resumption token (first record to harvest) private int resultTotal = -1; - private String resumptionStr = Integer.toString(this.resumptionInt); // string resumption token (first record to harvest + private String resumptionStr = Integer.toString(this.resumptionInt); // string resumption token (first record to + // harvest // or token scanned from results) private InputStream resultStream; private Transformer transformer; @@ -82,9 +83,9 @@ public class RestIterator implements Iterator { private int discoverResultSize = 0; private int pagination = 1; /* - * While resultFormatValue is added to the request parameter, this is used to say that the results are retrieved in json. useful for - * cases when the target API expects a resultFormatValue != json, but the results are returned in json. An example is the EU Open Data - * Portal API: resultFormatValue=standard, results are in json format. + * While resultFormatValue is added to the request parameter, this is used to say that the results are retrieved in + * json. useful for cases when the target API expects a resultFormatValue != json, but the results are returned in + * json. An example is the EU Open Data Portal API: resultFormatValue=standard, results are in json format. */ private final String resultOutputFormat; @@ -92,21 +93,21 @@ public class RestIterator implements Iterator { * RestIterator class compatible to version 1.3.33 */ public RestIterator( - final HttpClientParams clientParams, - final String baseUrl, - final String resumptionType, - final String resumptionParam, - final String resumptionXpath, - final String resultTotalXpath, - final String resultFormatParam, - final String resultFormatValue, - final String resultSizeParam, - final String resultSizeValueStr, - final String queryParams, - final String entityXpath, - final String authMethod, - final String authToken, - final String resultOutputFormat) { + final HttpClientParams clientParams, + final String baseUrl, + final String resumptionType, + final String resumptionParam, + final String resumptionXpath, + final String resultTotalXpath, + final String resultFormatParam, + final String resultFormatValue, + final String resultSizeParam, + final String resultSizeValueStr, + final String queryParams, + final String entityXpath, + final String authMethod, + final String authToken, + final String resultOutputFormat) { this.clientParams = clientParams; this.baseUrl = baseUrl; @@ -120,8 +121,9 @@ public class RestIterator implements Iterator { this.resultOutputFormat = resultOutputFormat; this.queryFormat = StringUtils.isNotBlank(resultFormatParam) ? "&" + resultFormatParam + "=" + resultFormatValue - : ""; - this.querySize = StringUtils.isNotBlank(resultSizeParam) ? "&" + resultSizeParam + "=" + resultSizeValueStr : ""; + : ""; + this.querySize = StringUtils.isNotBlank(resultSizeParam) ? "&" + resultSizeParam + "=" + resultSizeValueStr + : ""; try { initXmlTransformation(resultTotalXpath, resumptionXpath, entityXpath); @@ -132,8 +134,9 @@ public class RestIterator implements Iterator { initQueue(); } - private void initXmlTransformation(final String resultTotalXpath, final String resumptionXpath, final String entityXpath) - throws TransformerConfigurationException, XPathExpressionException { + private void initXmlTransformation(final String resultTotalXpath, final String resumptionXpath, + final String entityXpath) + throws TransformerConfigurationException, XPathExpressionException { final TransformerFactory factory = TransformerFactory.newInstance(); this.transformer = factory.newTransformer(); this.transformer.setOutputProperty(OutputKeys.INDENT, "yes"); @@ -155,7 +158,6 @@ public class RestIterator implements Iterator { /* * (non-Javadoc) - * * @see java.util.Iterator#hasNext() */ @Override @@ -169,7 +171,6 @@ public class RestIterator implements Iterator { /* * (non-Javadoc) - * * @see java.util.Iterator#next() */ @Override @@ -192,7 +193,9 @@ public class RestIterator implements Iterator { */ private String downloadPage(String query, final int attempt) throws CollectorException { - if (attempt > MAX_ATTEMPTS) { throw new CollectorException("Max Number of attempts reached, query:" + query); } + if (attempt > MAX_ATTEMPTS) { + throw new CollectorException("Max Number of attempts reached, query:" + query); + } if (attempt > 0) { final int delay = (attempt * 5000); @@ -254,15 +257,19 @@ public class RestIterator implements Iterator { } if (!(emptyXml).equalsIgnoreCase(resultXml)) { - resultNode = (Node) this.xpath.evaluate("/", new InputSource(this.resultStream), XPathConstants.NODE); + resultNode = (Node) this.xpath + .evaluate("/", new InputSource(this.resultStream), XPathConstants.NODE); nodeList = (NodeList) this.xprEntity.evaluate(resultNode, XPathConstants.NODESET); log.debug("nodeList.length: {}", nodeList.getLength()); for (int i = 0; i < nodeList.getLength(); i++) { final StringWriter sw = new StringWriter(); this.transformer.transform(new DOMSource(nodeList.item(i)), new StreamResult(sw)); final String toEnqueue = sw.toString(); - if ((toEnqueue == null) || StringUtils.isBlank(toEnqueue) || emptyXml.equalsIgnoreCase(toEnqueue)) { - log.warn("The following record resulted in empty item for the feeding queue: {}", resultXml); + if ((toEnqueue == null) || StringUtils.isBlank(toEnqueue) + || emptyXml.equalsIgnoreCase(toEnqueue)) { + log + .warn( + "The following record resulted in empty item for the feeding queue: {}", resultXml); } else { this.recordQueue.add(sw.toString()); } @@ -274,90 +281,95 @@ public class RestIterator implements Iterator { this.resumptionInt += this.resultSizeValue; switch (this.resumptionType.toLowerCase()) { - case "scan": // read of resumptionToken , evaluate next results, e.g. OAI, iterate over items - this.resumptionStr = this.xprResumptionPath.evaluate(resultNode); - break; + case "scan": // read of resumptionToken , evaluate next results, e.g. OAI, iterate over items + this.resumptionStr = this.xprResumptionPath.evaluate(resultNode); + break; - case "count": // begin at one step for all records, iterate over items - this.resumptionStr = Integer.toString(this.resumptionInt); - break; + case "count": // begin at one step for all records, iterate over items + this.resumptionStr = Integer.toString(this.resumptionInt); + break; - case "discover": // size of result items unknown, iterate over items (for openDOAR - 201808) - if (this.resultSizeValue < 2) { throw new CollectorException("Mode: discover, Param 'resultSizeValue' is less than 2"); } - qUrlArgument = qUrl.getQuery(); - final String[] arrayQUrlArgument = qUrlArgument.split("&"); - for (final String arrayUrlArgStr : arrayQUrlArgument) { - if (arrayUrlArgStr.startsWith(this.resumptionParam)) { - final String[] resumptionKeyValue = arrayUrlArgStr.split("="); - if (isInteger(resumptionKeyValue[1])) { - urlOldResumptionSize = Integer.parseInt(resumptionKeyValue[1]); - log.debug("discover OldResumptionSize from Url (int): {}", urlOldResumptionSize); - } else { - log.debug("discover OldResumptionSize from Url (str): {}", resumptionKeyValue[1]); + case "discover": // size of result items unknown, iterate over items (for openDOAR - 201808) + if (this.resultSizeValue < 2) { + throw new CollectorException("Mode: discover, Param 'resultSizeValue' is less than 2"); + } + qUrlArgument = qUrl.getQuery(); + final String[] arrayQUrlArgument = qUrlArgument.split("&"); + for (final String arrayUrlArgStr : arrayQUrlArgument) { + if (arrayUrlArgStr.startsWith(this.resumptionParam)) { + final String[] resumptionKeyValue = arrayUrlArgStr.split("="); + if (isInteger(resumptionKeyValue[1])) { + urlOldResumptionSize = Integer.parseInt(resumptionKeyValue[1]); + log.debug("discover OldResumptionSize from Url (int): {}", urlOldResumptionSize); + } else { + log.debug("discover OldResumptionSize from Url (str): {}", resumptionKeyValue[1]); + } } } - } - if (((emptyXml).equalsIgnoreCase(resultXml)) + if (((emptyXml).equalsIgnoreCase(resultXml)) || ((nodeList != null) && (nodeList.getLength() < this.resultSizeValue))) { - // resumptionStr = ""; + // resumptionStr = ""; + if (nodeList != null) { + this.discoverResultSize += nodeList.getLength(); + } + this.resultTotal = this.discoverResultSize; + } else { + this.resumptionStr = Integer.toString(this.resumptionInt); + this.resultTotal = this.resumptionInt + 1; + if (nodeList != null) { + this.discoverResultSize += nodeList.getLength(); + } + } + log.info("discoverResultSize: {}", this.discoverResultSize); + break; + + case "pagination": + case "page": // pagination, iterate over page numbers + this.pagination += 1; if (nodeList != null) { this.discoverResultSize += nodeList.getLength(); + } else { + this.resultTotal = this.discoverResultSize; + this.pagination = this.discoverResultSize; } - this.resultTotal = this.discoverResultSize; - } else { + this.resumptionInt = this.pagination; this.resumptionStr = Integer.toString(this.resumptionInt); - this.resultTotal = this.resumptionInt + 1; - if (nodeList != null) { - this.discoverResultSize += nodeList.getLength(); + break; + + case "deep-cursor": // size of result items unknown, iterate over items (for supporting deep cursor + // in + // solr) + // isn't relevant -- if (resultSizeValue < 2) {throw new CollectorServiceException("Mode: + // deep-cursor, Param 'resultSizeValue' is less than 2");} + + this.resumptionStr = encodeValue(this.xprResumptionPath.evaluate(resultNode)); + this.queryParams = this.queryParams.replace("&cursor=*", ""); + + // terminating if length of nodeList is 0 + if ((nodeList != null) && (nodeList.getLength() < this.discoverResultSize)) { + this.resumptionInt += ((nodeList.getLength() + 1) - this.resultSizeValue); + } else { + this.resumptionInt += (nodeList.getLength() - this.resultSizeValue); // subtract the + // resultSizeValue + // because the iteration is over + // real length and the + // resultSizeValue is added before + // the switch() } - } - log.info("discoverResultSize: {}", this.discoverResultSize); - break; - case "pagination": - case "page": // pagination, iterate over page numbers - this.pagination += 1; - if (nodeList != null) { - this.discoverResultSize += nodeList.getLength(); - } else { - this.resultTotal = this.discoverResultSize; - this.pagination = this.discoverResultSize; - } - this.resumptionInt = this.pagination; - this.resumptionStr = Integer.toString(this.resumptionInt); - break; + this.discoverResultSize = nodeList.getLength(); - case "deep-cursor": // size of result items unknown, iterate over items (for supporting deep cursor in - // solr) - // isn't relevant -- if (resultSizeValue < 2) {throw new CollectorServiceException("Mode: - // deep-cursor, Param 'resultSizeValue' is less than 2");} - - this.resumptionStr = encodeValue(this.xprResumptionPath.evaluate(resultNode)); - this.queryParams = this.queryParams.replace("&cursor=*", ""); - - // terminating if length of nodeList is 0 - if ((nodeList != null) && (nodeList.getLength() < this.discoverResultSize)) { - this.resumptionInt += ((nodeList.getLength() + 1) - this.resultSizeValue); - } else { - this.resumptionInt += (nodeList.getLength() - this.resultSizeValue); // subtract the resultSizeValue - // because the iteration is over - // real length and the - // resultSizeValue is added before - // the switch() - } - - this.discoverResultSize = nodeList.getLength(); - - log - .debug("downloadPage().deep-cursor: resumptionStr=" + this.resumptionStr + " ; queryParams=" + log + .debug( + "downloadPage().deep-cursor: resumptionStr=" + this.resumptionStr + " ; queryParams=" + this.queryParams + " resumptionLengthIncreased: " + this.resumptionInt); - break; + break; - default: // otherwise: abort - // resultTotal = resumptionInt; - break; + default: // otherwise: abort + // resultTotal = resumptionInt; + break; } } catch (final Exception e) { @@ -380,8 +392,9 @@ public class RestIterator implements Iterator { log.debug("resultTotal: " + this.resultTotal); log.debug("resInt: " + this.resumptionInt); if (this.resumptionInt <= this.resultTotal) { - nextQuery = this.baseUrl + "?" + this.queryParams + this.querySize + "&" + this.resumptionParam + "=" + this.resumptionStr - + this.queryFormat; + nextQuery = this.baseUrl + "?" + this.queryParams + this.querySize + "&" + this.resumptionParam + "=" + + this.resumptionStr + + this.queryFormat; } else { nextQuery = ""; // if (resumptionType.toLowerCase().equals("deep-cursor")) { resumptionInt -= 1; } // correct the diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/OsfPreprintCollectorTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/OsfPreprintCollectorTest.java index bc2d12661..90f4c7f25 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/OsfPreprintCollectorTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/OsfPreprintCollectorTest.java @@ -3,6 +3,7 @@ package eu.dnetlib.dhp.collection.plugin.rest; import java.util.HashMap; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Stream; import org.junit.jupiter.api.Assertions; @@ -69,7 +70,7 @@ public class OsfPreprintCollectorTest { @Test @Disabled - void test() throws CollectorException { + void test_limited() throws CollectorException { final AtomicInteger i = new AtomicInteger(0); final Stream stream = this.rcp.collect(this.api, new AggregatorReport()); @@ -82,4 +83,23 @@ public class OsfPreprintCollectorTest { log.info("{}", i.intValue()); Assertions.assertTrue(i.intValue() > 0); } + + @Test + @Disabled + void test_all() throws CollectorException { + final AtomicLong i = new AtomicLong(0); + final Stream stream = this.rcp.collect(this.api, new AggregatorReport()); + + stream.forEach(s -> { + Assertions.assertTrue(s.length() > 0); + if ((i.incrementAndGet() % 1000) == 0) { + log.info("COLLECTED: {}", i.get()); + } + + }); + + log.info("TOTAL: {}", i.get()); + Assertions.assertTrue(i.get() > 0); + } + } From 66680b8b9a69a2801016ee4a9b34f872ce6a766f Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Thu, 2 May 2024 11:16:58 +0200 Subject: [PATCH 33/97] refactoring of common utilities --- dhp-common/pom.xml | 10 +- .../dnetlib/pace/common/PaceCommonUtils.java | 100 ++++++++++++++++++ .../java/eu/dnetlib/pace/model/Person.java | 15 ++- .../java/eu/dnetlib/pace/util/Capitalise.java | 17 +++ .../dnetlib/pace/util/DotAbbreviations.java | 11 ++ .../eu/dnetlib/pace/config/name_particles.txt | 0 dhp-pace-core/pom.xml | 6 ++ .../pace/common/AbstractPaceFunctions.java | 81 ++------------ dhp-workflows/dhp-graph-mapper/pom.xml | 6 ++ 9 files changed, 160 insertions(+), 86 deletions(-) create mode 100644 dhp-common/src/main/java/eu/dnetlib/pace/common/PaceCommonUtils.java rename {dhp-pace-core => dhp-common}/src/main/java/eu/dnetlib/pace/model/Person.java (96%) create mode 100644 dhp-common/src/main/java/eu/dnetlib/pace/util/Capitalise.java create mode 100644 dhp-common/src/main/java/eu/dnetlib/pace/util/DotAbbreviations.java rename {dhp-pace-core => dhp-common}/src/main/resources/eu/dnetlib/pace/config/name_particles.txt (100%) diff --git a/dhp-common/pom.xml b/dhp-common/pom.xml index 692d2bdc3..04735876d 100644 --- a/dhp-common/pom.xml +++ b/dhp-common/pom.xml @@ -63,11 +63,13 @@ - eu.dnetlib.dhp - dhp-pace-core - ${project.version} + edu.cmu + secondstring + + + com.ibm.icu + icu4j - org.apache.hadoop hadoop-common diff --git a/dhp-common/src/main/java/eu/dnetlib/pace/common/PaceCommonUtils.java b/dhp-common/src/main/java/eu/dnetlib/pace/common/PaceCommonUtils.java new file mode 100644 index 000000000..a279271b5 --- /dev/null +++ b/dhp-common/src/main/java/eu/dnetlib/pace/common/PaceCommonUtils.java @@ -0,0 +1,100 @@ + +package eu.dnetlib.pace.common; + +import com.google.common.base.Splitter; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import com.ibm.icu.text.Transliterator; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; + +import java.nio.charset.StandardCharsets; +import java.text.Normalizer; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Set of common functions for the framework + * + * @author claudio + */ +public class PaceCommonUtils { + + // transliterator + protected static Transliterator transliterator = Transliterator.getInstance("Any-Eng"); + + protected static final String aliases_from = "⁰¹²³⁴⁵⁶⁷⁸⁹⁺⁻⁼⁽⁾ⁿ₀₁₂₃₄₅₆₇₈₉₊₋₌₍₎àáâäæãåāèéêëēėęəîïíīįìôöòóœøōõûüùúūßśšłžźżçćčñń"; + protected static final String aliases_to = "0123456789+-=()n0123456789+-=()aaaaaaaaeeeeeeeeiiiiiioooooooouuuuussslzzzcccnn"; + + protected static Pattern hexUnicodePattern = Pattern.compile("\\\\u(\\p{XDigit}{4})"); + + protected static String fixAliases(final String s) { + final StringBuilder sb = new StringBuilder(); + + s.chars().forEach(ch -> { + final int i = StringUtils.indexOf(aliases_from, ch); + sb.append(i >= 0 ? aliases_to.charAt(i) : (char) ch); + }); + + return sb.toString(); + } + + protected static String transliterate(final String s) { + try { + return transliterator.transliterate(s); + } catch (Exception e) { + return s; + } + } + + public static String normalize(final String s) { + return fixAliases(transliterate(nfd(unicodeNormalization(s)))) + .toLowerCase() + // do not compact the regexes in a single expression, would cause StackOverflowError in case of large input + // strings + .replaceAll("[^ \\w]+", "") + .replaceAll("(\\p{InCombiningDiacriticalMarks})+", "") + .replaceAll("(\\p{Punct})+", " ") + .replaceAll("(\\d)+", " ") + .replaceAll("(\\n)+", " ") + .trim(); + } + + public static String nfd(final String s) { + return Normalizer.normalize(s, Normalizer.Form.NFD); + } + + public static String unicodeNormalization(final String s) { + + Matcher m = hexUnicodePattern.matcher(s); + StringBuffer buf = new StringBuffer(s.length()); + while (m.find()) { + String ch = String.valueOf((char) Integer.parseInt(m.group(1), 16)); + m.appendReplacement(buf, Matcher.quoteReplacement(ch)); + } + m.appendTail(buf); + return buf.toString(); + } + + public static Set loadFromClasspath(final String classpath) { + + Transliterator transliterator = Transliterator.getInstance("Any-Eng"); + + final Set h = Sets.newHashSet(); + try { + for (final String s : IOUtils + .readLines(PaceCommonUtils.class.getResourceAsStream(classpath), StandardCharsets.UTF_8)) { + h.add(fixAliases(transliterator.transliterate(s))); // transliteration of the stopwords + } + } catch (final Throwable e) { + return Sets.newHashSet(); + } + return h; + } + + protected static Iterable tokens(final String s, final int maxTokens) { + return Iterables.limit(Splitter.on(" ").omitEmptyStrings().trimResults().split(s), maxTokens); + } + +} diff --git a/dhp-pace-core/src/main/java/eu/dnetlib/pace/model/Person.java b/dhp-common/src/main/java/eu/dnetlib/pace/model/Person.java similarity index 96% rename from dhp-pace-core/src/main/java/eu/dnetlib/pace/model/Person.java rename to dhp-common/src/main/java/eu/dnetlib/pace/model/Person.java index 96120cf4d..c95c9d823 100644 --- a/dhp-pace-core/src/main/java/eu/dnetlib/pace/model/Person.java +++ b/dhp-common/src/main/java/eu/dnetlib/pace/model/Person.java @@ -1,21 +1,20 @@ package eu.dnetlib.pace.model; -import java.nio.charset.Charset; -import java.text.Normalizer; -import java.util.List; -import java.util.Set; - import com.google.common.base.Joiner; import com.google.common.base.Splitter; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.hash.Hashing; - -import eu.dnetlib.pace.common.AbstractPaceFunctions; +import eu.dnetlib.pace.common.PaceCommonUtils; import eu.dnetlib.pace.util.Capitalise; import eu.dnetlib.pace.util.DotAbbreviations; +import java.nio.charset.Charset; +import java.text.Normalizer; +import java.util.List; +import java.util.Set; + public class Person { private static final String UTF8 = "UTF-8"; @@ -86,7 +85,7 @@ public class Person { private List splitTerms(final String s) { if (particles == null) { - particles = AbstractPaceFunctions.loadFromClasspath("/eu/dnetlib/pace/config/name_particles.txt"); + particles = PaceCommonUtils.loadFromClasspath("/eu/dnetlib/pace/config/name_particles.txt"); } final List list = Lists.newArrayList(); diff --git a/dhp-common/src/main/java/eu/dnetlib/pace/util/Capitalise.java b/dhp-common/src/main/java/eu/dnetlib/pace/util/Capitalise.java new file mode 100644 index 000000000..015386423 --- /dev/null +++ b/dhp-common/src/main/java/eu/dnetlib/pace/util/Capitalise.java @@ -0,0 +1,17 @@ + +package eu.dnetlib.pace.util; + +import com.google.common.base.Function; +import org.apache.commons.lang3.text.WordUtils; + +public class Capitalise implements Function { + + private final char[] DELIM = { + ' ', '-' + }; + + @Override + public String apply(final String s) { + return WordUtils.capitalize(s.toLowerCase(), DELIM); + } +} diff --git a/dhp-common/src/main/java/eu/dnetlib/pace/util/DotAbbreviations.java b/dhp-common/src/main/java/eu/dnetlib/pace/util/DotAbbreviations.java new file mode 100644 index 000000000..2c89da4db --- /dev/null +++ b/dhp-common/src/main/java/eu/dnetlib/pace/util/DotAbbreviations.java @@ -0,0 +1,11 @@ + +package eu.dnetlib.pace.util; + +import com.google.common.base.Function; + +public class DotAbbreviations implements Function { + @Override + public String apply(String s) { + return s.length() == 1 ? s + "." : s; + } +} diff --git a/dhp-pace-core/src/main/resources/eu/dnetlib/pace/config/name_particles.txt b/dhp-common/src/main/resources/eu/dnetlib/pace/config/name_particles.txt similarity index 100% rename from dhp-pace-core/src/main/resources/eu/dnetlib/pace/config/name_particles.txt rename to dhp-common/src/main/resources/eu/dnetlib/pace/config/name_particles.txt diff --git a/dhp-pace-core/pom.xml b/dhp-pace-core/pom.xml index 7b384f109..1593575d2 100644 --- a/dhp-pace-core/pom.xml +++ b/dhp-pace-core/pom.xml @@ -49,6 +49,12 @@ + + eu.dnetlib.dhp + dhp-common + ${project.version} + + edu.cmu secondstring diff --git a/dhp-pace-core/src/main/java/eu/dnetlib/pace/common/AbstractPaceFunctions.java b/dhp-pace-core/src/main/java/eu/dnetlib/pace/common/AbstractPaceFunctions.java index ba7639ada..6bfb8b3f4 100644 --- a/dhp-pace-core/src/main/java/eu/dnetlib/pace/common/AbstractPaceFunctions.java +++ b/dhp-pace-core/src/main/java/eu/dnetlib/pace/common/AbstractPaceFunctions.java @@ -1,32 +1,26 @@ package eu.dnetlib.pace.common; +import com.google.common.base.Joiner; +import com.google.common.collect.Sets; +import com.ibm.icu.text.Transliterator; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; + import java.io.IOException; import java.io.StringWriter; import java.nio.charset.StandardCharsets; -import java.text.Normalizer; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; - -import com.google.common.base.Joiner; -import com.google.common.base.Splitter; -import com.google.common.collect.Iterables; -import com.google.common.collect.Sets; -import com.ibm.icu.text.Transliterator; - -import eu.dnetlib.pace.clustering.NGramUtils; - /** * Set of common functions for the framework * * @author claudio */ -public class AbstractPaceFunctions { +public class AbstractPaceFunctions extends PaceCommonUtils { // city map to be used when translating the city names into codes private static Map cityMap = AbstractPaceFunctions @@ -41,9 +35,6 @@ public class AbstractPaceFunctions { protected static Set stopwords_it = loadFromClasspath("/eu/dnetlib/pace/config/stopwords_it.txt"); protected static Set stopwords_pt = loadFromClasspath("/eu/dnetlib/pace/config/stopwords_pt.txt"); - // transliterator - protected static Transliterator transliterator = Transliterator.getInstance("Any-Eng"); - // blacklist of ngrams: to avoid generic keys protected static Set ngramBlacklist = loadFromClasspath("/eu/dnetlib/pace/config/ngram_blacklist.txt"); @@ -51,8 +42,6 @@ public class AbstractPaceFunctions { public static final Pattern HTML_REGEX = Pattern.compile("<[^>]*>"); private static final String alpha = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 "; - private static final String aliases_from = "⁰¹²³⁴⁵⁶⁷⁸⁹⁺⁻⁼⁽⁾ⁿ₀₁₂₃₄₅₆₇₈₉₊₋₌₍₎àáâäæãåāèéêëēėęəîïíīįìôöòóœøōõûüùúūßśšłžźżçćčñń"; - private static final String aliases_to = "0123456789+-=()n0123456789+-=()aaaaaaaaeeeeeeeeiiiiiioooooooouuuuussslzzzcccnn"; // doi prefix for normalization public static final Pattern DOI_PREFIX = Pattern.compile("(https?:\\/\\/dx\\.doi\\.org\\/)|(doi:)"); @@ -129,25 +118,6 @@ public class AbstractPaceFunctions { return numberPattern.matcher(strNum).matches(); } - protected static String fixAliases(final String s) { - final StringBuilder sb = new StringBuilder(); - - s.chars().forEach(ch -> { - final int i = StringUtils.indexOf(aliases_from, ch); - sb.append(i >= 0 ? aliases_to.charAt(i) : (char) ch); - }); - - return sb.toString(); - } - - protected static String transliterate(final String s) { - try { - return transliterator.transliterate(s); - } catch (Exception e) { - return s; - } - } - protected static String removeSymbols(final String s) { final StringBuilder sb = new StringBuilder(); @@ -162,23 +132,6 @@ public class AbstractPaceFunctions { return s != null; } - public static String normalize(final String s) { - return fixAliases(transliterate(nfd(unicodeNormalization(s)))) - .toLowerCase() - // do not compact the regexes in a single expression, would cause StackOverflowError in case of large input - // strings - .replaceAll("[^ \\w]+", "") - .replaceAll("(\\p{InCombiningDiacriticalMarks})+", "") - .replaceAll("(\\p{Punct})+", " ") - .replaceAll("(\\d)+", " ") - .replaceAll("(\\n)+", " ") - .trim(); - } - - public static String nfd(final String s) { - return Normalizer.normalize(s, Normalizer.Form.NFD); - } - public static String utf8(final String s) { byte[] bytes = s.getBytes(StandardCharsets.UTF_8); return new String(bytes, StandardCharsets.UTF_8); @@ -233,22 +186,6 @@ public class AbstractPaceFunctions { return newset; } - public static Set loadFromClasspath(final String classpath) { - - Transliterator transliterator = Transliterator.getInstance("Any-Eng"); - - final Set h = Sets.newHashSet(); - try { - for (final String s : IOUtils - .readLines(NGramUtils.class.getResourceAsStream(classpath), StandardCharsets.UTF_8)) { - h.add(fixAliases(transliterator.transliterate(s))); // transliteration of the stopwords - } - } catch (final Throwable e) { - return Sets.newHashSet(); - } - return h; - } - public static Map loadMapFromClasspath(final String classpath) { Transliterator transliterator = Transliterator.getInstance("Any-Eng"); @@ -303,10 +240,6 @@ public class AbstractPaceFunctions { return StringUtils.substring(s, 0, 1).toLowerCase(); } - protected static Iterable tokens(final String s, final int maxTokens) { - return Iterables.limit(Splitter.on(" ").omitEmptyStrings().trimResults().split(s), maxTokens); - } - public static String normalizePid(String pid) { return DOI_PREFIX.matcher(pid.toLowerCase()).replaceAll(""); } diff --git a/dhp-workflows/dhp-graph-mapper/pom.xml b/dhp-workflows/dhp-graph-mapper/pom.xml index c7ac55ef6..2c93bab83 100644 --- a/dhp-workflows/dhp-graph-mapper/pom.xml +++ b/dhp-workflows/dhp-graph-mapper/pom.xml @@ -90,6 +90,12 @@ ${project.version} + + eu.dnetlib.dhp + dhp-pace-core + ${project.version} + + com.jayway.jsonpath json-path From 4355f648106b1180f2946de9346961b0db2286a4 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Thu, 2 May 2024 11:23:53 +0200 Subject: [PATCH 34/97] reverted to version 1.2.5-SNAPSHOT --- .../dhp-build-assembly-resources/pom.xml | 2 +- .../dhp-build-properties-maven-plugin/pom.xml | 2 +- dhp-build/dhp-code-style/pom.xml | 2 +- dhp-build/pom.xml | 2 +- dhp-common/pom.xml | 2 +- dhp-pace-core/pom.xml | 4 +- .../java/eu/dnetlib/pace/util/Capitalise.java | 18 - .../eu/dnetlib/pace/util/DiffPatchMatch.java | 2553 ----------------- .../dnetlib/pace/util/DotAbbreviations.java | 11 - dhp-workflows/dhp-actionmanager/pom.xml | 2 +- dhp-workflows/dhp-aggregation/pom.xml | 2 +- dhp-workflows/dhp-blacklist/pom.xml | 2 +- dhp-workflows/dhp-broker-events/pom.xml | 2 +- dhp-workflows/dhp-dedup-openaire/pom.xml | 2 +- dhp-workflows/dhp-doiboost/pom.xml | 2 +- dhp-workflows/dhp-enrichment/pom.xml | 4 +- dhp-workflows/dhp-graph-mapper/pom.xml | 2 +- dhp-workflows/dhp-graph-provision/pom.xml | 2 +- dhp-workflows/dhp-impact-indicators/pom.xml | 2 +- dhp-workflows/dhp-stats-actionsets/pom.xml | 2 +- dhp-workflows/dhp-stats-hist-snaps/pom.xml | 2 +- dhp-workflows/dhp-stats-monitor-irish/pom.xml | 2 +- .../dhp-stats-monitor-update/pom.xml | 2 +- dhp-workflows/dhp-stats-promote/pom.xml | 2 +- dhp-workflows/dhp-stats-update/pom.xml | 2 +- dhp-workflows/dhp-swh/pom.xml | 2 +- .../dhp-usage-raw-data-update/pom.xml | 2 +- dhp-workflows/dhp-usage-stats-build/pom.xml | 2 +- dhp-workflows/dhp-workflow-profiles/pom.xml | 2 +- dhp-workflows/pom.xml | 2 +- pom.xml | 2 +- 31 files changed, 30 insertions(+), 2612 deletions(-) delete mode 100644 dhp-pace-core/src/main/java/eu/dnetlib/pace/util/Capitalise.java delete mode 100644 dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java delete mode 100644 dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DotAbbreviations.java diff --git a/dhp-build/dhp-build-assembly-resources/pom.xml b/dhp-build/dhp-build-assembly-resources/pom.xml index 7f5b76fdd..44165995d 100644 --- a/dhp-build/dhp-build-assembly-resources/pom.xml +++ b/dhp-build/dhp-build-assembly-resources/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp-build - 1.2.5-beta + 1.2.5-SNAPSHOT dhp-build-assembly-resources diff --git a/dhp-build/dhp-build-properties-maven-plugin/pom.xml b/dhp-build/dhp-build-properties-maven-plugin/pom.xml index e76dcd8fc..7579bdf45 100644 --- a/dhp-build/dhp-build-properties-maven-plugin/pom.xml +++ b/dhp-build/dhp-build-properties-maven-plugin/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp-build - 1.2.5-beta + 1.2.5-SNAPSHOT dhp-build-properties-maven-plugin diff --git a/dhp-build/dhp-code-style/pom.xml b/dhp-build/dhp-code-style/pom.xml index 8bbe6fac0..5a86efe17 100644 --- a/dhp-build/dhp-code-style/pom.xml +++ b/dhp-build/dhp-code-style/pom.xml @@ -5,7 +5,7 @@ eu.dnetlib.dhp dhp-code-style - 1.2.5-beta + 1.2.5-SNAPSHOT jar diff --git a/dhp-build/pom.xml b/dhp-build/pom.xml index 74a09a23c..9040ea94e 100644 --- a/dhp-build/pom.xml +++ b/dhp-build/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp - 1.2.5-beta + 1.2.5-SNAPSHOT dhp-build pom diff --git a/dhp-common/pom.xml b/dhp-common/pom.xml index 04735876d..c2f76cff7 100644 --- a/dhp-common/pom.xml +++ b/dhp-common/pom.xml @@ -5,7 +5,7 @@ eu.dnetlib.dhp dhp - 1.2.5-beta + 1.2.5-SNAPSHOT ../pom.xml diff --git a/dhp-pace-core/pom.xml b/dhp-pace-core/pom.xml index 1593575d2..6c706b692 100644 --- a/dhp-pace-core/pom.xml +++ b/dhp-pace-core/pom.xml @@ -6,13 +6,13 @@ eu.dnetlib.dhp dhp - 1.2.5-beta + 1.2.5-SNAPSHOT ../pom.xml eu.dnetlib.dhp dhp-pace-core - 1.2.5-beta + 1.2.5-SNAPSHOT jar diff --git a/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/Capitalise.java b/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/Capitalise.java deleted file mode 100644 index 403d91dd9..000000000 --- a/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/Capitalise.java +++ /dev/null @@ -1,18 +0,0 @@ - -package eu.dnetlib.pace.util; - -import org.apache.commons.lang3.text.WordUtils; - -import com.google.common.base.Function; - -public class Capitalise implements Function { - - private final char[] DELIM = { - ' ', '-' - }; - - @Override - public String apply(final String s) { - return WordUtils.capitalize(s.toLowerCase(), DELIM); - } -}; diff --git a/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java b/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java deleted file mode 100644 index cfd9acd70..000000000 --- a/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java +++ /dev/null @@ -1,2553 +0,0 @@ - -package eu.dnetlib.pace.util; - -/* - * Diff Match and Patch - * Copyright 2018 The diff-match-patch Authors. - * https://github.com/google/diff-match-patch - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/* - * Diff Match and Patch - * Copyright 2018 The diff-match-patch Authors. - * https://github.com/google/diff-match-patch - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import java.io.UnsupportedEncodingException; -import java.net.URLDecoder; -import java.net.URLEncoder; -import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/* - * Functions for diff, match and patch. - * Computes the difference between two texts to create a patch. - * Applies the patch onto another text, allowing for errors. - * - * @author fraser@google.com (Neil Fraser) - */ - -/** - * Class containing the diff, match and patch methods. - * Also contains the behaviour settings. - */ -public class DiffPatchMatch { - - // Defaults. - // Set these on your diff_match_patch instance to override the defaults. - - /** - * Number of seconds to map a diff before giving up (0 for infinity). - */ - public float Diff_Timeout = 1.0f; - /** - * Cost of an empty edit operation in terms of edit characters. - */ - public short Diff_EditCost = 4; - /** - * At what point is no match declared (0.0 = perfection, 1.0 = very loose). - */ - public float Match_Threshold = 0.5f; - /** - * How far to search for a match (0 = exact location, 1000+ = broad match). - * A match this many characters away from the expected location will add - * 1.0 to the score (0.0 is a perfect match). - */ - public int Match_Distance = 1000; - /** - * When deleting a large block of text (over ~64 characters), how close do - * the contents have to be to match the expected contents. (0.0 = perfection, - * 1.0 = very loose). Note that Match_Threshold controls how closely the - * end points of a delete need to match. - */ - public float Patch_DeleteThreshold = 0.5f; - /** - * Chunk size for context length. - */ - public short Patch_Margin = 4; - - /** - * The number of bits in an int. - */ - private short Match_MaxBits = 32; - - /** - * Internal class for returning results from diff_linesToChars(). - * Other less paranoid languages just use a three-element array. - */ - protected static class LinesToCharsResult { - protected String chars1; - protected String chars2; - protected List lineArray; - - protected LinesToCharsResult(String chars1, String chars2, - List lineArray) { - this.chars1 = chars1; - this.chars2 = chars2; - this.lineArray = lineArray; - } - } - - // DIFF FUNCTIONS - - /** - * The data structure representing a diff is a Linked list of Diff objects: - * {Diff(Operation.DELETE, "Hello"), Diff(Operation.INSERT, "Goodbye"), - * Diff(Operation.EQUAL, " world.")} - * which means: delete "Hello", add "Goodbye" and keep " world." - */ - public enum Operation { - DELETE, INSERT, EQUAL - } - - /** - * Find the differences between two texts. - * Run a faster, slightly less optimal diff. - * This method allows the 'checklines' of diff_main() to be optional. - * Most of the time checklines is wanted, so default to true. - * @param text1 Old string to be diffed. - * @param text2 New string to be diffed. - * @return Linked List of Diff objects. - */ - public LinkedList diff_main(String text1, String text2) { - return diff_main(text1, text2, true); - } - - /** - * Find the differences between two texts. - * @param text1 Old string to be diffed. - * @param text2 New string to be diffed. - * @param checklines Speedup flag. If false, then don't run a - * line-level diff first to identify the changed areas. - * If true, then run a faster slightly less optimal diff. - * @return Linked List of Diff objects. - */ - public LinkedList diff_main(String text1, String text2, - boolean checklines) { - // Set a deadline by which time the diff must be complete. - long deadline; - if (Diff_Timeout <= 0) { - deadline = Long.MAX_VALUE; - } else { - deadline = System.currentTimeMillis() + (long) (Diff_Timeout * 1000); - } - return diff_main(text1, text2, checklines, deadline); - } - - /** - * Find the differences between two texts. Simplifies the problem by - * stripping any common prefix or suffix off the texts before diffing. - * @param text1 Old string to be diffed. - * @param text2 New string to be diffed. - * @param checklines Speedup flag. If false, then don't run a - * line-level diff first to identify the changed areas. - * If true, then run a faster slightly less optimal diff. - * @param deadline Time when the diff should be complete by. Used - * internally for recursive calls. Users should set DiffTimeout instead. - * @return Linked List of Diff objects. - */ - private LinkedList diff_main(String text1, String text2, - boolean checklines, long deadline) { - // Check for null inputs. - if (text1 == null || text2 == null) { - throw new IllegalArgumentException("Null inputs. (diff_main)"); - } - - // Check for equality (speedup). - LinkedList diffs; - if (text1.equals(text2)) { - diffs = new LinkedList(); - if (text1.length() != 0) { - diffs.add(new Diff(Operation.EQUAL, text1)); - } - return diffs; - } - - // Trim off common prefix (speedup). - int commonlength = diff_commonPrefix(text1, text2); - String commonprefix = text1.substring(0, commonlength); - text1 = text1.substring(commonlength); - text2 = text2.substring(commonlength); - - // Trim off common suffix (speedup). - commonlength = diff_commonSuffix(text1, text2); - String commonsuffix = text1.substring(text1.length() - commonlength); - text1 = text1.substring(0, text1.length() - commonlength); - text2 = text2.substring(0, text2.length() - commonlength); - - // Compute the diff on the middle block. - diffs = diff_compute(text1, text2, checklines, deadline); - - // Restore the prefix and suffix. - if (commonprefix.length() != 0) { - diffs.addFirst(new Diff(Operation.EQUAL, commonprefix)); - } - if (commonsuffix.length() != 0) { - diffs.addLast(new Diff(Operation.EQUAL, commonsuffix)); - } - - diff_cleanupMerge(diffs); - return diffs; - } - - /** - * Find the differences between two texts. Assumes that the texts do not - * have any common prefix or suffix. - * @param text1 Old string to be diffed. - * @param text2 New string to be diffed. - * @param checklines Speedup flag. If false, then don't run a - * line-level diff first to identify the changed areas. - * If true, then run a faster slightly less optimal diff. - * @param deadline Time when the diff should be complete by. - * @return Linked List of Diff objects. - */ - private LinkedList diff_compute(String text1, String text2, - boolean checklines, long deadline) { - LinkedList diffs = new LinkedList(); - - if (text1.length() == 0) { - // Just add some text (speedup). - diffs.add(new Diff(Operation.INSERT, text2)); - return diffs; - } - - if (text2.length() == 0) { - // Just delete some text (speedup). - diffs.add(new Diff(Operation.DELETE, text1)); - return diffs; - } - - String longtext = text1.length() > text2.length() ? text1 : text2; - String shorttext = text1.length() > text2.length() ? text2 : text1; - int i = longtext.indexOf(shorttext); - if (i != -1) { - // Shorter text is inside the longer text (speedup). - Operation op = (text1.length() > text2.length()) ? Operation.DELETE : Operation.INSERT; - diffs.add(new Diff(op, longtext.substring(0, i))); - diffs.add(new Diff(Operation.EQUAL, shorttext)); - diffs.add(new Diff(op, longtext.substring(i + shorttext.length()))); - return diffs; - } - - if (shorttext.length() == 1) { - // Single character string. - // After the previous speedup, the character can't be an equality. - diffs.add(new Diff(Operation.DELETE, text1)); - diffs.add(new Diff(Operation.INSERT, text2)); - return diffs; - } - - // Check to see if the problem can be split in two. - String[] hm = diff_halfMatch(text1, text2); - if (hm != null) { - // A half-match was found, sort out the return data. - String text1_a = hm[0]; - String text1_b = hm[1]; - String text2_a = hm[2]; - String text2_b = hm[3]; - String mid_common = hm[4]; - // Send both pairs off for separate processing. - LinkedList diffs_a = diff_main( - text1_a, text2_a, - checklines, deadline); - LinkedList diffs_b = diff_main( - text1_b, text2_b, - checklines, deadline); - // Merge the results. - diffs = diffs_a; - diffs.add(new Diff(Operation.EQUAL, mid_common)); - diffs.addAll(diffs_b); - return diffs; - } - - if (checklines && text1.length() > 100 && text2.length() > 100) { - return diff_lineMode(text1, text2, deadline); - } - - return diff_bisect(text1, text2, deadline); - } - - /** - * Do a quick line-level diff on both strings, then rediff the parts for - * greater accuracy. - * This speedup can produce non-minimal diffs. - * @param text1 Old string to be diffed. - * @param text2 New string to be diffed. - * @param deadline Time when the diff should be complete by. - * @return Linked List of Diff objects. - */ - private LinkedList diff_lineMode(String text1, String text2, - long deadline) { - // Scan the text on a line-by-line basis first. - LinesToCharsResult a = diff_linesToChars(text1, text2); - text1 = a.chars1; - text2 = a.chars2; - List linearray = a.lineArray; - - LinkedList diffs = diff_main(text1, text2, false, deadline); - - // Convert the diff back to original text. - diff_charsToLines(diffs, linearray); - // Eliminate freak matches (e.g. blank lines) - diff_cleanupSemantic(diffs); - - // Rediff any replacement blocks, this time character-by-character. - // Add a dummy entry at the end. - diffs.add(new Diff(Operation.EQUAL, "")); - int count_delete = 0; - int count_insert = 0; - String text_delete = ""; - String text_insert = ""; - ListIterator pointer = diffs.listIterator(); - Diff thisDiff = pointer.next(); - while (thisDiff != null) { - switch (thisDiff.operation) { - case INSERT: - count_insert++; - text_insert += thisDiff.text; - break; - case DELETE: - count_delete++; - text_delete += thisDiff.text; - break; - case EQUAL: - // Upon reaching an equality, check for prior redundancies. - if (count_delete >= 1 && count_insert >= 1) { - // Delete the offending records and add the merged ones. - pointer.previous(); - for (int j = 0; j < count_delete + count_insert; j++) { - pointer.previous(); - pointer.remove(); - } - for (Diff subDiff : diff_main( - text_delete, text_insert, false, - deadline)) { - pointer.add(subDiff); - } - } - count_insert = 0; - count_delete = 0; - text_delete = ""; - text_insert = ""; - break; - } - thisDiff = pointer.hasNext() ? pointer.next() : null; - } - diffs.removeLast(); // Remove the dummy entry at the end. - - return diffs; - } - - /** - * Find the 'middle snake' of a diff, split the problem in two - * and return the recursively constructed diff. - * See Myers 1986 paper: An O(ND) Difference Algorithm and Its Variations. - * @param text1 Old string to be diffed. - * @param text2 New string to be diffed. - * @param deadline Time at which to bail if not yet complete. - * @return LinkedList of Diff objects. - */ - protected LinkedList diff_bisect(String text1, String text2, - long deadline) { - // Cache the text lengths to prevent multiple calls. - int text1_length = text1.length(); - int text2_length = text2.length(); - int max_d = (text1_length + text2_length + 1) / 2; - int v_offset = max_d; - int v_length = 2 * max_d; - int[] v1 = new int[v_length]; - int[] v2 = new int[v_length]; - for (int x = 0; x < v_length; x++) { - v1[x] = -1; - v2[x] = -1; - } - v1[v_offset + 1] = 0; - v2[v_offset + 1] = 0; - int delta = text1_length - text2_length; - // If the total number of characters is odd, then the front path will - // collide with the reverse path. - boolean front = (delta % 2 != 0); - // Offsets for start and end of k loop. - // Prevents mapping of space beyond the grid. - int k1start = 0; - int k1end = 0; - int k2start = 0; - int k2end = 0; - for (int d = 0; d < max_d; d++) { - // Bail out if deadline is reached. - if (System.currentTimeMillis() > deadline) { - break; - } - - // Walk the front path one step. - for (int k1 = -d + k1start; k1 <= d - k1end; k1 += 2) { - int k1_offset = v_offset + k1; - int x1; - if (k1 == -d || (k1 != d && v1[k1_offset - 1] < v1[k1_offset + 1])) { - x1 = v1[k1_offset + 1]; - } else { - x1 = v1[k1_offset - 1] + 1; - } - int y1 = x1 - k1; - while (x1 < text1_length && y1 < text2_length - && text1.charAt(x1) == text2.charAt(y1)) { - x1++; - y1++; - } - v1[k1_offset] = x1; - if (x1 > text1_length) { - // Ran off the right of the graph. - k1end += 2; - } else if (y1 > text2_length) { - // Ran off the bottom of the graph. - k1start += 2; - } else if (front) { - int k2_offset = v_offset + delta - k1; - if (k2_offset >= 0 && k2_offset < v_length && v2[k2_offset] != -1) { - // Mirror x2 onto top-left coordinate system. - int x2 = text1_length - v2[k2_offset]; - if (x1 >= x2) { - // Overlap detected. - return diff_bisectSplit(text1, text2, x1, y1, deadline); - } - } - } - } - - // Walk the reverse path one step. - for (int k2 = -d + k2start; k2 <= d - k2end; k2 += 2) { - int k2_offset = v_offset + k2; - int x2; - if (k2 == -d || (k2 != d && v2[k2_offset - 1] < v2[k2_offset + 1])) { - x2 = v2[k2_offset + 1]; - } else { - x2 = v2[k2_offset - 1] + 1; - } - int y2 = x2 - k2; - while (x2 < text1_length && y2 < text2_length - && text1.charAt(text1_length - x2 - 1) == text2.charAt(text2_length - y2 - 1)) { - x2++; - y2++; - } - v2[k2_offset] = x2; - if (x2 > text1_length) { - // Ran off the left of the graph. - k2end += 2; - } else if (y2 > text2_length) { - // Ran off the top of the graph. - k2start += 2; - } else if (!front) { - int k1_offset = v_offset + delta - k2; - if (k1_offset >= 0 && k1_offset < v_length && v1[k1_offset] != -1) { - int x1 = v1[k1_offset]; - int y1 = v_offset + x1 - k1_offset; - // Mirror x2 onto top-left coordinate system. - x2 = text1_length - x2; - if (x1 >= x2) { - // Overlap detected. - return diff_bisectSplit(text1, text2, x1, y1, deadline); - } - } - } - } - } - // Diff took too long and hit the deadline or - // number of diffs equals number of characters, no commonality at all. - LinkedList diffs = new LinkedList(); - diffs.add(new Diff(Operation.DELETE, text1)); - diffs.add(new Diff(Operation.INSERT, text2)); - return diffs; - } - - /** - * Given the location of the 'middle snake', split the diff in two parts - * and recurse. - * @param text1 Old string to be diffed. - * @param text2 New string to be diffed. - * @param x Index of split point in text1. - * @param y Index of split point in text2. - * @param deadline Time at which to bail if not yet complete. - * @return LinkedList of Diff objects. - */ - private LinkedList diff_bisectSplit(String text1, String text2, - int x, int y, long deadline) { - String text1a = text1.substring(0, x); - String text2a = text2.substring(0, y); - String text1b = text1.substring(x); - String text2b = text2.substring(y); - - // Compute both diffs serially. - LinkedList diffs = diff_main(text1a, text2a, false, deadline); - LinkedList diffsb = diff_main(text1b, text2b, false, deadline); - - diffs.addAll(diffsb); - return diffs; - } - - /** - * Split two texts into a list of strings. Reduce the texts to a string of - * hashes where each Unicode character represents one line. - * @param text1 First string. - * @param text2 Second string. - * @return An object containing the encoded text1, the encoded text2 and - * the List of unique strings. The zeroth element of the List of - * unique strings is intentionally blank. - */ - protected LinesToCharsResult diff_linesToChars(String text1, String text2) { - List lineArray = new ArrayList(); - Map lineHash = new HashMap(); - // e.g. linearray[4] == "Hello\n" - // e.g. linehash.get("Hello\n") == 4 - - // "\x00" is a valid character, but various debuggers don't like it. - // So we'll insert a junk entry to avoid generating a null character. - lineArray.add(""); - - // Allocate 2/3rds of the space for text1, the rest for text2. - String chars1 = diff_linesToCharsMunge(text1, lineArray, lineHash, 40000); - String chars2 = diff_linesToCharsMunge(text2, lineArray, lineHash, 65535); - return new LinesToCharsResult(chars1, chars2, lineArray); - } - - /** - * Split a text into a list of strings. Reduce the texts to a string of - * hashes where each Unicode character represents one line. - * @param text String to encode. - * @param lineArray List of unique strings. - * @param lineHash Map of strings to indices. - * @param maxLines Maximum length of lineArray. - * @return Encoded string. - */ - private String diff_linesToCharsMunge(String text, List lineArray, - Map lineHash, int maxLines) { - int lineStart = 0; - int lineEnd = -1; - String line; - StringBuilder chars = new StringBuilder(); - // Walk the text, pulling out a substring for each line. - // text.split('\n') would would temporarily double our memory footprint. - // Modifying text would create many large strings to garbage collect. - while (lineEnd < text.length() - 1) { - lineEnd = text.indexOf('\n', lineStart); - if (lineEnd == -1) { - lineEnd = text.length() - 1; - } - line = text.substring(lineStart, lineEnd + 1); - - if (lineHash.containsKey(line)) { - chars.append(String.valueOf((char) (int) lineHash.get(line))); - } else { - if (lineArray.size() == maxLines) { - // Bail out at 65535 because - // String.valueOf((char) 65536).equals(String.valueOf(((char) 0))) - line = text.substring(lineStart); - lineEnd = text.length(); - } - lineArray.add(line); - lineHash.put(line, lineArray.size() - 1); - chars.append(String.valueOf((char) (lineArray.size() - 1))); - } - lineStart = lineEnd + 1; - } - return chars.toString(); - } - - /** - * Rehydrate the text in a diff from a string of line hashes to real lines of - * text. - * @param diffs List of Diff objects. - * @param lineArray List of unique strings. - */ - protected void diff_charsToLines(List diffs, - List lineArray) { - StringBuilder text; - for (Diff diff : diffs) { - text = new StringBuilder(); - for (int j = 0; j < diff.text.length(); j++) { - text.append(lineArray.get(diff.text.charAt(j))); - } - diff.text = text.toString(); - } - } - - /** - * Determine the common prefix of two strings - * @param text1 First string. - * @param text2 Second string. - * @return The number of characters common to the start of each string. - */ - public int diff_commonPrefix(String text1, String text2) { - // Performance analysis: https://neil.fraser.name/news/2007/10/09/ - int n = Math.min(text1.length(), text2.length()); - for (int i = 0; i < n; i++) { - if (text1.charAt(i) != text2.charAt(i)) { - return i; - } - } - return n; - } - - /** - * Determine the common suffix of two strings - * @param text1 First string. - * @param text2 Second string. - * @return The number of characters common to the end of each string. - */ - public int diff_commonSuffix(String text1, String text2) { - // Performance analysis: https://neil.fraser.name/news/2007/10/09/ - int text1_length = text1.length(); - int text2_length = text2.length(); - int n = Math.min(text1_length, text2_length); - for (int i = 1; i <= n; i++) { - if (text1.charAt(text1_length - i) != text2.charAt(text2_length - i)) { - return i - 1; - } - } - return n; - } - - /** - * Determine if the suffix of one string is the prefix of another. - * @param text1 First string. - * @param text2 Second string. - * @return The number of characters common to the end of the first - * string and the start of the second string. - */ - protected int diff_commonOverlap(String text1, String text2) { - // Cache the text lengths to prevent multiple calls. - int text1_length = text1.length(); - int text2_length = text2.length(); - // Eliminate the null case. - if (text1_length == 0 || text2_length == 0) { - return 0; - } - // Truncate the longer string. - if (text1_length > text2_length) { - text1 = text1.substring(text1_length - text2_length); - } else if (text1_length < text2_length) { - text2 = text2.substring(0, text1_length); - } - int text_length = Math.min(text1_length, text2_length); - // Quick check for the worst case. - if (text1.equals(text2)) { - return text_length; - } - - // Start by looking for a single character match - // and increase length until no match is found. - // Performance analysis: https://neil.fraser.name/news/2010/11/04/ - int best = 0; - int length = 1; - while (true) { - String pattern = text1.substring(text_length - length); - int found = text2.indexOf(pattern); - if (found == -1) { - return best; - } - length += found; - if (found == 0 || text1 - .substring(text_length - length) - .equals( - text2.substring(0, length))) { - best = length; - length++; - } - } - } - - /** - * Do the two texts share a substring which is at least half the length of - * the longer text? - * This speedup can produce non-minimal diffs. - * @param text1 First string. - * @param text2 Second string. - * @return Five element String array, containing the prefix of text1, the - * suffix of text1, the prefix of text2, the suffix of text2 and the - * common middle. Or null if there was no match. - */ - protected String[] diff_halfMatch(String text1, String text2) { - if (Diff_Timeout <= 0) { - // Don't risk returning a non-optimal diff if we have unlimited time. - return null; - } - String longtext = text1.length() > text2.length() ? text1 : text2; - String shorttext = text1.length() > text2.length() ? text2 : text1; - if (longtext.length() < 4 || shorttext.length() * 2 < longtext.length()) { - return null; // Pointless. - } - - // First check if the second quarter is the seed for a half-match. - String[] hm1 = diff_halfMatchI( - longtext, shorttext, - (longtext.length() + 3) / 4); - // Check again based on the third quarter. - String[] hm2 = diff_halfMatchI( - longtext, shorttext, - (longtext.length() + 1) / 2); - String[] hm; - if (hm1 == null && hm2 == null) { - return null; - } else if (hm2 == null) { - hm = hm1; - } else if (hm1 == null) { - hm = hm2; - } else { - // Both matched. Select the longest. - hm = hm1[4].length() > hm2[4].length() ? hm1 : hm2; - } - - // A half-match was found, sort out the return data. - if (text1.length() > text2.length()) { - return hm; - // return new String[]{hm[0], hm[1], hm[2], hm[3], hm[4]}; - } else { - return new String[] { - hm[2], hm[3], hm[0], hm[1], hm[4] - }; - } - } - - /** - * Does a substring of shorttext exist within longtext such that the - * substring is at least half the length of longtext? - * @param longtext Longer string. - * @param shorttext Shorter string. - * @param i Start index of quarter length substring within longtext. - * @return Five element String array, containing the prefix of longtext, the - * suffix of longtext, the prefix of shorttext, the suffix of shorttext - * and the common middle. Or null if there was no match. - */ - private String[] diff_halfMatchI(String longtext, String shorttext, int i) { - // Start with a 1/4 length substring at position i as a seed. - String seed = longtext.substring(i, i + longtext.length() / 4); - int j = -1; - String best_common = ""; - String best_longtext_a = "", best_longtext_b = ""; - String best_shorttext_a = "", best_shorttext_b = ""; - while ((j = shorttext.indexOf(seed, j + 1)) != -1) { - int prefixLength = diff_commonPrefix( - longtext.substring(i), - shorttext.substring(j)); - int suffixLength = diff_commonSuffix( - longtext.substring(0, i), - shorttext.substring(0, j)); - if (best_common.length() < suffixLength + prefixLength) { - best_common = shorttext.substring(j - suffixLength, j) - + shorttext.substring(j, j + prefixLength); - best_longtext_a = longtext.substring(0, i - suffixLength); - best_longtext_b = longtext.substring(i + prefixLength); - best_shorttext_a = shorttext.substring(0, j - suffixLength); - best_shorttext_b = shorttext.substring(j + prefixLength); - } - } - if (best_common.length() * 2 >= longtext.length()) { - return new String[] { - best_longtext_a, best_longtext_b, - best_shorttext_a, best_shorttext_b, best_common - }; - } else { - return null; - } - } - - /** - * Reduce the number of edits by eliminating semantically trivial equalities. - * @param diffs LinkedList of Diff objects. - */ - public void diff_cleanupSemantic(LinkedList diffs) { - if (diffs.isEmpty()) { - return; - } - boolean changes = false; - Deque equalities = new ArrayDeque(); // Double-ended queue of qualities. - String lastEquality = null; // Always equal to equalities.peek().text - ListIterator pointer = diffs.listIterator(); - // Number of characters that changed prior to the equality. - int length_insertions1 = 0; - int length_deletions1 = 0; - // Number of characters that changed after the equality. - int length_insertions2 = 0; - int length_deletions2 = 0; - Diff thisDiff = pointer.next(); - while (thisDiff != null) { - if (thisDiff.operation == Operation.EQUAL) { - // Equality found. - equalities.push(thisDiff); - length_insertions1 = length_insertions2; - length_deletions1 = length_deletions2; - length_insertions2 = 0; - length_deletions2 = 0; - lastEquality = thisDiff.text; - } else { - // An insertion or deletion. - if (thisDiff.operation == Operation.INSERT) { - length_insertions2 += thisDiff.text.length(); - } else { - length_deletions2 += thisDiff.text.length(); - } - // Eliminate an equality that is smaller or equal to the edits on both - // sides of it. - if (lastEquality != null && (lastEquality.length() <= Math.max(length_insertions1, length_deletions1)) - && (lastEquality.length() <= Math.max(length_insertions2, length_deletions2))) { - // System.out.println("Splitting: '" + lastEquality + "'"); - // Walk back to offending equality. - while (thisDiff != equalities.peek()) { - thisDiff = pointer.previous(); - } - pointer.next(); - - // Replace equality with a delete. - pointer.set(new Diff(Operation.DELETE, lastEquality)); - // Insert a corresponding an insert. - pointer.add(new Diff(Operation.INSERT, lastEquality)); - - equalities.pop(); // Throw away the equality we just deleted. - if (!equalities.isEmpty()) { - // Throw away the previous equality (it needs to be reevaluated). - equalities.pop(); - } - if (equalities.isEmpty()) { - // There are no previous equalities, walk back to the start. - while (pointer.hasPrevious()) { - pointer.previous(); - } - } else { - // There is a safe equality we can fall back to. - thisDiff = equalities.peek(); - while (thisDiff != pointer.previous()) { - // Intentionally empty loop. - } - } - - length_insertions1 = 0; // Reset the counters. - length_insertions2 = 0; - length_deletions1 = 0; - length_deletions2 = 0; - lastEquality = null; - changes = true; - } - } - thisDiff = pointer.hasNext() ? pointer.next() : null; - } - - // Normalize the diff. - if (changes) { - diff_cleanupMerge(diffs); - } - diff_cleanupSemanticLossless(diffs); - - // Find any overlaps between deletions and insertions. - // e.g: abcxxxxxxdef - // -> abcxxxdef - // e.g: xxxabcdefxxx - // -> defxxxabc - // Only extract an overlap if it is as big as the edit ahead or behind it. - pointer = diffs.listIterator(); - Diff prevDiff = null; - thisDiff = null; - if (pointer.hasNext()) { - prevDiff = pointer.next(); - if (pointer.hasNext()) { - thisDiff = pointer.next(); - } - } - while (thisDiff != null) { - if (prevDiff.operation == Operation.DELETE && - thisDiff.operation == Operation.INSERT) { - String deletion = prevDiff.text; - String insertion = thisDiff.text; - int overlap_length1 = this.diff_commonOverlap(deletion, insertion); - int overlap_length2 = this.diff_commonOverlap(insertion, deletion); - if (overlap_length1 >= overlap_length2) { - if (overlap_length1 >= deletion.length() / 2.0 || - overlap_length1 >= insertion.length() / 2.0) { - // Overlap found. Insert an equality and trim the surrounding edits. - pointer.previous(); - pointer - .add( - new Diff(Operation.EQUAL, - insertion.substring(0, overlap_length1))); - prevDiff.text = deletion.substring(0, deletion.length() - overlap_length1); - thisDiff.text = insertion.substring(overlap_length1); - // pointer.add inserts the element before the cursor, so there is - // no need to step past the new element. - } - } else { - if (overlap_length2 >= deletion.length() / 2.0 || - overlap_length2 >= insertion.length() / 2.0) { - // Reverse overlap found. - // Insert an equality and swap and trim the surrounding edits. - pointer.previous(); - pointer - .add( - new Diff(Operation.EQUAL, - deletion.substring(0, overlap_length2))); - prevDiff.operation = Operation.INSERT; - prevDiff.text = insertion.substring(0, insertion.length() - overlap_length2); - thisDiff.operation = Operation.DELETE; - thisDiff.text = deletion.substring(overlap_length2); - // pointer.add inserts the element before the cursor, so there is - // no need to step past the new element. - } - } - thisDiff = pointer.hasNext() ? pointer.next() : null; - } - prevDiff = thisDiff; - thisDiff = pointer.hasNext() ? pointer.next() : null; - } - } - - /** - * Look for single edits surrounded on both sides by equalities - * which can be shifted sideways to align the edit to a word boundary. - * e.g: The cat came. -> The cat came. - * @param diffs LinkedList of Diff objects. - */ - public void diff_cleanupSemanticLossless(LinkedList diffs) { - String equality1, edit, equality2; - String commonString; - int commonOffset; - int score, bestScore; - String bestEquality1, bestEdit, bestEquality2; - // Create a new iterator at the start. - ListIterator pointer = diffs.listIterator(); - Diff prevDiff = pointer.hasNext() ? pointer.next() : null; - Diff thisDiff = pointer.hasNext() ? pointer.next() : null; - Diff nextDiff = pointer.hasNext() ? pointer.next() : null; - // Intentionally ignore the first and last element (don't need checking). - while (nextDiff != null) { - if (prevDiff.operation == Operation.EQUAL && - nextDiff.operation == Operation.EQUAL) { - // This is a single edit surrounded by equalities. - equality1 = prevDiff.text; - edit = thisDiff.text; - equality2 = nextDiff.text; - - // First, shift the edit as far left as possible. - commonOffset = diff_commonSuffix(equality1, edit); - if (commonOffset != 0) { - commonString = edit.substring(edit.length() - commonOffset); - equality1 = equality1.substring(0, equality1.length() - commonOffset); - edit = commonString + edit.substring(0, edit.length() - commonOffset); - equality2 = commonString + equality2; - } - - // Second, step character by character right, looking for the best fit. - bestEquality1 = equality1; - bestEdit = edit; - bestEquality2 = equality2; - bestScore = diff_cleanupSemanticScore(equality1, edit) - + diff_cleanupSemanticScore(edit, equality2); - while (edit.length() != 0 && equality2.length() != 0 - && edit.charAt(0) == equality2.charAt(0)) { - equality1 += edit.charAt(0); - edit = edit.substring(1) + equality2.charAt(0); - equality2 = equality2.substring(1); - score = diff_cleanupSemanticScore(equality1, edit) - + diff_cleanupSemanticScore(edit, equality2); - // The >= encourages trailing rather than leading whitespace on edits. - if (score >= bestScore) { - bestScore = score; - bestEquality1 = equality1; - bestEdit = edit; - bestEquality2 = equality2; - } - } - - if (!prevDiff.text.equals(bestEquality1)) { - // We have an improvement, save it back to the diff. - if (bestEquality1.length() != 0) { - prevDiff.text = bestEquality1; - } else { - pointer.previous(); // Walk past nextDiff. - pointer.previous(); // Walk past thisDiff. - pointer.previous(); // Walk past prevDiff. - pointer.remove(); // Delete prevDiff. - pointer.next(); // Walk past thisDiff. - pointer.next(); // Walk past nextDiff. - } - thisDiff.text = bestEdit; - if (bestEquality2.length() != 0) { - nextDiff.text = bestEquality2; - } else { - pointer.remove(); // Delete nextDiff. - nextDiff = thisDiff; - thisDiff = prevDiff; - } - } - } - prevDiff = thisDiff; - thisDiff = nextDiff; - nextDiff = pointer.hasNext() ? pointer.next() : null; - } - } - - /** - * Given two strings, compute a score representing whether the internal - * boundary falls on logical boundaries. - * Scores range from 6 (best) to 0 (worst). - * @param one First string. - * @param two Second string. - * @return The score. - */ - private int diff_cleanupSemanticScore(String one, String two) { - if (one.length() == 0 || two.length() == 0) { - // Edges are the best. - return 6; - } - - // Each port of this function behaves slightly differently due to - // subtle differences in each language's definition of things like - // 'whitespace'. Since this function's purpose is largely cosmetic, - // the choice has been made to use each language's native features - // rather than force total conformity. - char char1 = one.charAt(one.length() - 1); - char char2 = two.charAt(0); - boolean nonAlphaNumeric1 = !Character.isLetterOrDigit(char1); - boolean nonAlphaNumeric2 = !Character.isLetterOrDigit(char2); - boolean whitespace1 = nonAlphaNumeric1 && Character.isWhitespace(char1); - boolean whitespace2 = nonAlphaNumeric2 && Character.isWhitespace(char2); - boolean lineBreak1 = whitespace1 - && Character.getType(char1) == Character.CONTROL; - boolean lineBreak2 = whitespace2 - && Character.getType(char2) == Character.CONTROL; - boolean blankLine1 = lineBreak1 && BLANKLINEEND.matcher(one).find(); - boolean blankLine2 = lineBreak2 && BLANKLINESTART.matcher(two).find(); - - if (blankLine1 || blankLine2) { - // Five points for blank lines. - return 5; - } else if (lineBreak1 || lineBreak2) { - // Four points for line breaks. - return 4; - } else if (nonAlphaNumeric1 && !whitespace1 && whitespace2) { - // Three points for end of sentences. - return 3; - } else if (whitespace1 || whitespace2) { - // Two points for whitespace. - return 2; - } else if (nonAlphaNumeric1 || nonAlphaNumeric2) { - // One point for non-alphanumeric. - return 1; - } - return 0; - } - - // Define some regex patterns for matching boundaries. - private Pattern BLANKLINEEND = Pattern.compile("\\n\\r?\\n\\Z", Pattern.DOTALL); - private Pattern BLANKLINESTART = Pattern.compile("\\A\\r?\\n\\r?\\n", Pattern.DOTALL); - - /** - * Reduce the number of edits by eliminating operationally trivial equalities. - * @param diffs LinkedList of Diff objects. - */ - public void diff_cleanupEfficiency(LinkedList diffs) { - if (diffs.isEmpty()) { - return; - } - boolean changes = false; - Deque equalities = new ArrayDeque(); // Double-ended queue of equalities. - String lastEquality = null; // Always equal to equalities.peek().text - ListIterator pointer = diffs.listIterator(); - // Is there an insertion operation before the last equality. - boolean pre_ins = false; - // Is there a deletion operation before the last equality. - boolean pre_del = false; - // Is there an insertion operation after the last equality. - boolean post_ins = false; - // Is there a deletion operation after the last equality. - boolean post_del = false; - Diff thisDiff = pointer.next(); - Diff safeDiff = thisDiff; // The last Diff that is known to be unsplittable. - while (thisDiff != null) { - if (thisDiff.operation == Operation.EQUAL) { - // Equality found. - if (thisDiff.text.length() < Diff_EditCost && (post_ins || post_del)) { - // Candidate found. - equalities.push(thisDiff); - pre_ins = post_ins; - pre_del = post_del; - lastEquality = thisDiff.text; - } else { - // Not a candidate, and can never become one. - equalities.clear(); - lastEquality = null; - safeDiff = thisDiff; - } - post_ins = post_del = false; - } else { - // An insertion or deletion. - if (thisDiff.operation == Operation.DELETE) { - post_del = true; - } else { - post_ins = true; - } - /* - * Five types to be split: ABXYCD - * AXCD ABXC - * AXCD ABXC - */ - if (lastEquality != null - && ((pre_ins && pre_del && post_ins && post_del) - || ((lastEquality.length() < Diff_EditCost / 2) - && ((pre_ins ? 1 : 0) + (pre_del ? 1 : 0) - + (post_ins ? 1 : 0) + (post_del ? 1 : 0)) == 3))) { - // System.out.println("Splitting: '" + lastEquality + "'"); - // Walk back to offending equality. - while (thisDiff != equalities.peek()) { - thisDiff = pointer.previous(); - } - pointer.next(); - - // Replace equality with a delete. - pointer.set(new Diff(Operation.DELETE, lastEquality)); - // Insert a corresponding an insert. - pointer.add(thisDiff = new Diff(Operation.INSERT, lastEquality)); - - equalities.pop(); // Throw away the equality we just deleted. - lastEquality = null; - if (pre_ins && pre_del) { - // No changes made which could affect previous entry, keep going. - post_ins = post_del = true; - equalities.clear(); - safeDiff = thisDiff; - } else { - if (!equalities.isEmpty()) { - // Throw away the previous equality (it needs to be reevaluated). - equalities.pop(); - } - if (equalities.isEmpty()) { - // There are no previous questionable equalities, - // walk back to the last known safe diff. - thisDiff = safeDiff; - } else { - // There is an equality we can fall back to. - thisDiff = equalities.peek(); - } - while (thisDiff != pointer.previous()) { - // Intentionally empty loop. - } - post_ins = post_del = false; - } - - changes = true; - } - } - thisDiff = pointer.hasNext() ? pointer.next() : null; - } - - if (changes) { - diff_cleanupMerge(diffs); - } - } - - /** - * Reorder and merge like edit sections. Merge equalities. - * Any edit section can move as long as it doesn't cross an equality. - * @param diffs LinkedList of Diff objects. - */ - public void diff_cleanupMerge(LinkedList diffs) { - diffs.add(new Diff(Operation.EQUAL, "")); // Add a dummy entry at the end. - ListIterator pointer = diffs.listIterator(); - int count_delete = 0; - int count_insert = 0; - String text_delete = ""; - String text_insert = ""; - Diff thisDiff = pointer.next(); - Diff prevEqual = null; - int commonlength; - while (thisDiff != null) { - switch (thisDiff.operation) { - case INSERT: - count_insert++; - text_insert += thisDiff.text; - prevEqual = null; - break; - case DELETE: - count_delete++; - text_delete += thisDiff.text; - prevEqual = null; - break; - case EQUAL: - if (count_delete + count_insert > 1) { - boolean both_types = count_delete != 0 && count_insert != 0; - // Delete the offending records. - pointer.previous(); // Reverse direction. - while (count_delete-- > 0) { - pointer.previous(); - pointer.remove(); - } - while (count_insert-- > 0) { - pointer.previous(); - pointer.remove(); - } - if (both_types) { - // Factor out any common prefixies. - commonlength = diff_commonPrefix(text_insert, text_delete); - if (commonlength != 0) { - if (pointer.hasPrevious()) { - thisDiff = pointer.previous(); - assert thisDiff.operation == Operation.EQUAL : "Previous diff should have been an equality."; - thisDiff.text += text_insert.substring(0, commonlength); - pointer.next(); - } else { - pointer - .add( - new Diff(Operation.EQUAL, - text_insert.substring(0, commonlength))); - } - text_insert = text_insert.substring(commonlength); - text_delete = text_delete.substring(commonlength); - } - // Factor out any common suffixies. - commonlength = diff_commonSuffix(text_insert, text_delete); - if (commonlength != 0) { - thisDiff = pointer.next(); - thisDiff.text = text_insert - .substring( - text_insert.length() - - commonlength) - + thisDiff.text; - text_insert = text_insert - .substring( - 0, text_insert.length() - - commonlength); - text_delete = text_delete - .substring( - 0, text_delete.length() - - commonlength); - pointer.previous(); - } - } - // Insert the merged records. - if (text_delete.length() != 0) { - pointer.add(new Diff(Operation.DELETE, text_delete)); - } - if (text_insert.length() != 0) { - pointer.add(new Diff(Operation.INSERT, text_insert)); - } - // Step forward to the equality. - thisDiff = pointer.hasNext() ? pointer.next() : null; - } else if (prevEqual != null) { - // Merge this equality with the previous one. - prevEqual.text += thisDiff.text; - pointer.remove(); - thisDiff = pointer.previous(); - pointer.next(); // Forward direction - } - count_insert = 0; - count_delete = 0; - text_delete = ""; - text_insert = ""; - prevEqual = thisDiff; - break; - } - thisDiff = pointer.hasNext() ? pointer.next() : null; - } - if (diffs.getLast().text.length() == 0) { - diffs.removeLast(); // Remove the dummy entry at the end. - } - - /* - * Second pass: look for single edits surrounded on both sides by equalities which can be shifted sideways to - * eliminate an equality. e.g: ABAC -> ABAC - */ - boolean changes = false; - // Create a new iterator at the start. - // (As opposed to walking the current one back.) - pointer = diffs.listIterator(); - Diff prevDiff = pointer.hasNext() ? pointer.next() : null; - thisDiff = pointer.hasNext() ? pointer.next() : null; - Diff nextDiff = pointer.hasNext() ? pointer.next() : null; - // Intentionally ignore the first and last element (don't need checking). - while (nextDiff != null) { - if (prevDiff.operation == Operation.EQUAL && - nextDiff.operation == Operation.EQUAL) { - // This is a single edit surrounded by equalities. - if (thisDiff.text.endsWith(prevDiff.text)) { - // Shift the edit over the previous equality. - thisDiff.text = prevDiff.text - + thisDiff.text - .substring( - 0, thisDiff.text.length() - - prevDiff.text.length()); - nextDiff.text = prevDiff.text + nextDiff.text; - pointer.previous(); // Walk past nextDiff. - pointer.previous(); // Walk past thisDiff. - pointer.previous(); // Walk past prevDiff. - pointer.remove(); // Delete prevDiff. - pointer.next(); // Walk past thisDiff. - thisDiff = pointer.next(); // Walk past nextDiff. - nextDiff = pointer.hasNext() ? pointer.next() : null; - changes = true; - } else if (thisDiff.text.startsWith(nextDiff.text)) { - // Shift the edit over the next equality. - prevDiff.text += nextDiff.text; - thisDiff.text = thisDiff.text.substring(nextDiff.text.length()) - + nextDiff.text; - pointer.remove(); // Delete nextDiff. - nextDiff = pointer.hasNext() ? pointer.next() : null; - changes = true; - } - } - prevDiff = thisDiff; - thisDiff = nextDiff; - nextDiff = pointer.hasNext() ? pointer.next() : null; - } - // If shifts were made, the diff needs reordering and another shift sweep. - if (changes) { - diff_cleanupMerge(diffs); - } - } - - /** - * loc is a location in text1, compute and return the equivalent location in - * text2. - * e.g. "The cat" vs "The big cat", 1->1, 5->8 - * @param diffs List of Diff objects. - * @param loc Location within text1. - * @return Location within text2. - */ - public int diff_xIndex(List diffs, int loc) { - int chars1 = 0; - int chars2 = 0; - int last_chars1 = 0; - int last_chars2 = 0; - Diff lastDiff = null; - for (Diff aDiff : diffs) { - if (aDiff.operation != Operation.INSERT) { - // Equality or deletion. - chars1 += aDiff.text.length(); - } - if (aDiff.operation != Operation.DELETE) { - // Equality or insertion. - chars2 += aDiff.text.length(); - } - if (chars1 > loc) { - // Overshot the location. - lastDiff = aDiff; - break; - } - last_chars1 = chars1; - last_chars2 = chars2; - } - if (lastDiff != null && lastDiff.operation == Operation.DELETE) { - // The location was deleted. - return last_chars2; - } - // Add the remaining character length. - return last_chars2 + (loc - last_chars1); - } - - /** - * Convert a Diff list into a pretty HTML report. - * @param diffs List of Diff objects. - * @return HTML representation. - */ - public String diff_prettyHtml(List diffs) { - StringBuilder html = new StringBuilder(); - for (Diff aDiff : diffs) { - String text = aDiff.text - .replace("&", "&") - .replace("<", "<") - .replace(">", ">") - .replace("\n", "¶
"); - switch (aDiff.operation) { - case INSERT: - html - .append("") - .append(text) - .append(""); - break; - case DELETE: - html - .append("") - .append(text) - .append(""); - break; - case EQUAL: - html.append("").append(text).append(""); - break; - } - } - return html.toString(); - } - - /** - * Compute and return the source text (all equalities and deletions). - * @param diffs List of Diff objects. - * @return Source text. - */ - public String diff_text1(List diffs) { - StringBuilder text = new StringBuilder(); - for (Diff aDiff : diffs) { - if (aDiff.operation != Operation.INSERT) { - text.append(aDiff.text); - } - } - return text.toString(); - } - - /** - * Compute and return the destination text (all equalities and insertions). - * @param diffs List of Diff objects. - * @return Destination text. - */ - public String diff_text2(List diffs) { - StringBuilder text = new StringBuilder(); - for (Diff aDiff : diffs) { - if (aDiff.operation != Operation.DELETE) { - text.append(aDiff.text); - } - } - return text.toString(); - } - - /** - * Compute the Levenshtein compare; the number of inserted, deleted or - * substituted characters. - * @param diffs List of Diff objects. - * @return Number of changes. - */ - public int diff_levenshtein(List diffs) { - int levenshtein = 0; - int insertions = 0; - int deletions = 0; - for (Diff aDiff : diffs) { - switch (aDiff.operation) { - case INSERT: - insertions += aDiff.text.length(); - break; - case DELETE: - deletions += aDiff.text.length(); - break; - case EQUAL: - // A deletion and an insertion is one substitution. - levenshtein += Math.max(insertions, deletions); - insertions = 0; - deletions = 0; - break; - } - } - levenshtein += Math.max(insertions, deletions); - return levenshtein; - } - - /** - * Crush the diff into an encoded string which describes the operations - * required to transform text1 into text2. - * E.g. =3\t-2\t+ing -> Keep 3 chars, delete 2 chars, insert 'ing'. - * Operations are tab-separated. Inserted text is escaped using %xx notation. - * @param diffs List of Diff objects. - * @return Delta text. - */ - public String diff_toDelta(List diffs) { - StringBuilder text = new StringBuilder(); - for (Diff aDiff : diffs) { - switch (aDiff.operation) { - case INSERT: - try { - text - .append("+") - .append( - URLEncoder - .encode(aDiff.text, "UTF-8") - .replace('+', ' ')) - .append("\t"); - } catch (UnsupportedEncodingException e) { - // Not likely on modern system. - throw new Error("This system does not support UTF-8.", e); - } - break; - case DELETE: - text.append("-").append(aDiff.text.length()).append("\t"); - break; - case EQUAL: - text.append("=").append(aDiff.text.length()).append("\t"); - break; - } - } - String delta = text.toString(); - if (delta.length() != 0) { - // Strip off trailing tab character. - delta = delta.substring(0, delta.length() - 1); - delta = unescapeForEncodeUriCompatability(delta); - } - return delta; - } - - /** - * Given the original text1, and an encoded string which describes the - * operations required to transform text1 into text2, compute the full diff. - * @param text1 Source string for the diff. - * @param delta Delta text. - * @return Array of Diff objects or null if invalid. - * @throws IllegalArgumentException If invalid input. - */ - public LinkedList diff_fromDelta(String text1, String delta) - throws IllegalArgumentException { - LinkedList diffs = new LinkedList(); - int pointer = 0; // Cursor in text1 - String[] tokens = delta.split("\t"); - for (String token : tokens) { - if (token.length() == 0) { - // Blank tokens are ok (from a trailing \t). - continue; - } - // Each token begins with a one character parameter which specifies the - // operation of this token (delete, insert, equality). - String param = token.substring(1); - switch (token.charAt(0)) { - case '+': - // decode would change all "+" to " " - param = param.replace("+", "%2B"); - try { - param = URLDecoder.decode(param, "UTF-8"); - } catch (UnsupportedEncodingException e) { - // Not likely on modern system. - throw new Error("This system does not support UTF-8.", e); - } catch (IllegalArgumentException e) { - // Malformed URI sequence. - throw new IllegalArgumentException( - "Illegal escape in diff_fromDelta: " + param, e); - } - diffs.add(new Diff(Operation.INSERT, param)); - break; - case '-': - // Fall through. - case '=': - int n; - try { - n = Integer.parseInt(param); - } catch (NumberFormatException e) { - throw new IllegalArgumentException( - "Invalid number in diff_fromDelta: " + param, e); - } - if (n < 0) { - throw new IllegalArgumentException( - "Negative number in diff_fromDelta: " + param); - } - String text; - try { - text = text1.substring(pointer, pointer += n); - } catch (StringIndexOutOfBoundsException e) { - throw new IllegalArgumentException("Delta length (" + pointer - + ") larger than source text length (" + text1.length() - + ").", e); - } - if (token.charAt(0) == '=') { - diffs.add(new Diff(Operation.EQUAL, text)); - } else { - diffs.add(new Diff(Operation.DELETE, text)); - } - break; - default: - // Anything else is an error. - throw new IllegalArgumentException( - "Invalid diff operation in diff_fromDelta: " + token.charAt(0)); - } - } - if (pointer != text1.length()) { - throw new IllegalArgumentException("Delta length (" + pointer - + ") smaller than source text length (" + text1.length() + ")."); - } - return diffs; - } - - // MATCH FUNCTIONS - - /** - * Locate the best instance of 'pattern' in 'text' near 'loc'. - * Returns -1 if no match found. - * @param text The text to search. - * @param pattern The pattern to search for. - * @param loc The location to search around. - * @return Best match index or -1. - */ - public int match_main(String text, String pattern, int loc) { - // Check for null inputs. - if (text == null || pattern == null) { - throw new IllegalArgumentException("Null inputs. (match_main)"); - } - - loc = Math.max(0, Math.min(loc, text.length())); - if (text.equals(pattern)) { - // Shortcut (potentially not guaranteed by the algorithm) - return 0; - } else if (text.length() == 0) { - // Nothing to match. - return -1; - } else if (loc + pattern.length() <= text.length() - && text.substring(loc, loc + pattern.length()).equals(pattern)) { - // Perfect match at the perfect spot! (Includes case of null pattern) - return loc; - } else { - // Do a fuzzy compare. - return match_bitap(text, pattern, loc); - } - } - - /** - * Locate the best instance of 'pattern' in 'text' near 'loc' using the - * Bitap algorithm. Returns -1 if no match found. - * @param text The text to search. - * @param pattern The pattern to search for. - * @param loc The location to search around. - * @return Best match index or -1. - */ - protected int match_bitap(String text, String pattern, int loc) { - assert (Match_MaxBits == 0 || pattern.length() <= Match_MaxBits) : "Pattern too long for this application."; - - // Initialise the alphabet. - Map s = match_alphabet(pattern); - - // Highest score beyond which we give up. - double score_threshold = Match_Threshold; - // Is there a nearby exact match? (speedup) - int best_loc = text.indexOf(pattern, loc); - if (best_loc != -1) { - score_threshold = Math - .min( - match_bitapScore(0, best_loc, loc, pattern), - score_threshold); - // What about in the other direction? (speedup) - best_loc = text.lastIndexOf(pattern, loc + pattern.length()); - if (best_loc != -1) { - score_threshold = Math - .min( - match_bitapScore(0, best_loc, loc, pattern), - score_threshold); - } - } - - // Initialise the bit arrays. - int matchmask = 1 << (pattern.length() - 1); - best_loc = -1; - - int bin_min, bin_mid; - int bin_max = pattern.length() + text.length(); - // Empty initialization added to appease Java compiler. - int[] last_rd = new int[0]; - for (int d = 0; d < pattern.length(); d++) { - // Scan for the best match; each iteration allows for one more error. - // Run a binary search to determine how far from 'loc' we can stray at - // this error level. - bin_min = 0; - bin_mid = bin_max; - while (bin_min < bin_mid) { - if (match_bitapScore(d, loc + bin_mid, loc, pattern) <= score_threshold) { - bin_min = bin_mid; - } else { - bin_max = bin_mid; - } - bin_mid = (bin_max - bin_min) / 2 + bin_min; - } - // Use the result from this iteration as the maximum for the next. - bin_max = bin_mid; - int start = Math.max(1, loc - bin_mid + 1); - int finish = Math.min(loc + bin_mid, text.length()) + pattern.length(); - - int[] rd = new int[finish + 2]; - rd[finish + 1] = (1 << d) - 1; - for (int j = finish; j >= start; j--) { - int charMatch; - if (text.length() <= j - 1 || !s.containsKey(text.charAt(j - 1))) { - // Out of range. - charMatch = 0; - } else { - charMatch = s.get(text.charAt(j - 1)); - } - if (d == 0) { - // First pass: exact match. - rd[j] = ((rd[j + 1] << 1) | 1) & charMatch; - } else { - // Subsequent passes: fuzzy match. - rd[j] = (((rd[j + 1] << 1) | 1) & charMatch) - | (((last_rd[j + 1] | last_rd[j]) << 1) | 1) | last_rd[j + 1]; - } - if ((rd[j] & matchmask) != 0) { - double score = match_bitapScore(d, j - 1, loc, pattern); - // This match will almost certainly be better than any existing - // match. But check anyway. - if (score <= score_threshold) { - // Told you so. - score_threshold = score; - best_loc = j - 1; - if (best_loc > loc) { - // When passing loc, don't exceed our current compare from loc. - start = Math.max(1, 2 * loc - best_loc); - } else { - // Already passed loc, downhill from here on in. - break; - } - } - } - } - if (match_bitapScore(d + 1, loc, loc, pattern) > score_threshold) { - // No hope for a (better) match at greater error levels. - break; - } - last_rd = rd; - } - return best_loc; - } - - /** - * Compute and return the score for a match with e errors and x location. - * @param e Number of errors in match. - * @param x Location of match. - * @param loc Expected location of match. - * @param pattern Pattern being sought. - * @return Overall score for match (0.0 = good, 1.0 = bad). - */ - private double match_bitapScore(int e, int x, int loc, String pattern) { - float accuracy = (float) e / pattern.length(); - int proximity = Math.abs(loc - x); - if (Match_Distance == 0) { - // Dodge divide by zero error. - return proximity == 0 ? accuracy : 1.0; - } - return accuracy + (proximity / (float) Match_Distance); - } - - /** - * Initialise the alphabet for the Bitap algorithm. - * @param pattern The text to encode. - * @return Hash of character locations. - */ - protected Map match_alphabet(String pattern) { - Map s = new HashMap(); - char[] char_pattern = pattern.toCharArray(); - for (char c : char_pattern) { - s.put(c, 0); - } - int i = 0; - for (char c : char_pattern) { - s.put(c, s.get(c) | (1 << (pattern.length() - i - 1))); - i++; - } - return s; - } - - // PATCH FUNCTIONS - - /** - * Increase the context until it is unique, - * but don't let the pattern expand beyond Match_MaxBits. - * @param patch The patch to grow. - * @param text Source text. - */ - protected void patch_addContext(Patch patch, String text) { - if (text.length() == 0) { - return; - } - String pattern = text.substring(patch.start2, patch.start2 + patch.length1); - int padding = 0; - - // Look for the first and last matches of pattern in text. If two different - // matches are found, increase the pattern length. - while (text.indexOf(pattern) != text.lastIndexOf(pattern) - && pattern.length() < Match_MaxBits - Patch_Margin - Patch_Margin) { - padding += Patch_Margin; - pattern = text - .substring( - Math.max(0, patch.start2 - padding), - Math.min(text.length(), patch.start2 + patch.length1 + padding)); - } - // Add one chunk for good luck. - padding += Patch_Margin; - - // Add the prefix. - String prefix = text - .substring( - Math.max(0, patch.start2 - padding), - patch.start2); - if (prefix.length() != 0) { - patch.diffs.addFirst(new Diff(Operation.EQUAL, prefix)); - } - // Add the suffix. - String suffix = text - .substring( - patch.start2 + patch.length1, - Math.min(text.length(), patch.start2 + patch.length1 + padding)); - if (suffix.length() != 0) { - patch.diffs.addLast(new Diff(Operation.EQUAL, suffix)); - } - - // Roll back the start points. - patch.start1 -= prefix.length(); - patch.start2 -= prefix.length(); - // Extend the lengths. - patch.length1 += prefix.length() + suffix.length(); - patch.length2 += prefix.length() + suffix.length(); - } - - /** - * Compute a list of patches to turn text1 into text2. - * A set of diffs will be computed. - * @param text1 Old text. - * @param text2 New text. - * @return LinkedList of Patch objects. - */ - public LinkedList patch_make(String text1, String text2) { - if (text1 == null || text2 == null) { - throw new IllegalArgumentException("Null inputs. (patch_make)"); - } - // No diffs provided, compute our own. - LinkedList diffs = diff_main(text1, text2, true); - if (diffs.size() > 2) { - diff_cleanupSemantic(diffs); - diff_cleanupEfficiency(diffs); - } - return patch_make(text1, diffs); - } - - /** - * Compute a list of patches to turn text1 into text2. - * text1 will be derived from the provided diffs. - * @param diffs Array of Diff objects for text1 to text2. - * @return LinkedList of Patch objects. - */ - public LinkedList patch_make(LinkedList diffs) { - if (diffs == null) { - throw new IllegalArgumentException("Null inputs. (patch_make)"); - } - // No origin string provided, compute our own. - String text1 = diff_text1(diffs); - return patch_make(text1, diffs); - } - - /** - * Compute a list of patches to turn text1 into text2. - * text2 is ignored, diffs are the delta between text1 and text2. - * @param text1 Old text - * @param text2 Ignored. - * @param diffs Array of Diff objects for text1 to text2. - * @return LinkedList of Patch objects. - * @deprecated Prefer patch_make(String text1, LinkedList diffs). - */ - @Deprecated - public LinkedList patch_make(String text1, String text2, - LinkedList diffs) { - return patch_make(text1, diffs); - } - - /** - * Compute a list of patches to turn text1 into text2. - * text2 is not provided, diffs are the delta between text1 and text2. - * @param text1 Old text. - * @param diffs Array of Diff objects for text1 to text2. - * @return LinkedList of Patch objects. - */ - public LinkedList patch_make(String text1, LinkedList diffs) { - if (text1 == null || diffs == null) { - throw new IllegalArgumentException("Null inputs. (patch_make)"); - } - - LinkedList patches = new LinkedList(); - if (diffs.isEmpty()) { - return patches; // Get rid of the null case. - } - Patch patch = new Patch(); - int char_count1 = 0; // Number of characters into the text1 string. - int char_count2 = 0; // Number of characters into the text2 string. - // Start with text1 (prepatch_text) and apply the diffs until we arrive at - // text2 (postpatch_text). We recreate the patches one by one to determine - // context info. - String prepatch_text = text1; - String postpatch_text = text1; - for (Diff aDiff : diffs) { - if (patch.diffs.isEmpty() && aDiff.operation != Operation.EQUAL) { - // A new patch starts here. - patch.start1 = char_count1; - patch.start2 = char_count2; - } - - switch (aDiff.operation) { - case INSERT: - patch.diffs.add(aDiff); - patch.length2 += aDiff.text.length(); - postpatch_text = postpatch_text.substring(0, char_count2) - + aDiff.text + postpatch_text.substring(char_count2); - break; - case DELETE: - patch.length1 += aDiff.text.length(); - patch.diffs.add(aDiff); - postpatch_text = postpatch_text.substring(0, char_count2) - + postpatch_text.substring(char_count2 + aDiff.text.length()); - break; - case EQUAL: - if (aDiff.text.length() <= 2 * Patch_Margin - && !patch.diffs.isEmpty() && aDiff != diffs.getLast()) { - // Small equality inside a patch. - patch.diffs.add(aDiff); - patch.length1 += aDiff.text.length(); - patch.length2 += aDiff.text.length(); - } - - if (aDiff.text.length() >= 2 * Patch_Margin && !patch.diffs.isEmpty()) { - // Time for a new patch. - if (!patch.diffs.isEmpty()) { - patch_addContext(patch, prepatch_text); - patches.add(patch); - patch = new Patch(); - // Unlike Unidiff, our patch lists have a rolling context. - // https://github.com/google/diff-match-patch/wiki/Unidiff - // Update prepatch text & pos to reflect the application of the - // just completed patch. - prepatch_text = postpatch_text; - char_count1 = char_count2; - } - } - break; - } - - // Update the current character count. - if (aDiff.operation != Operation.INSERT) { - char_count1 += aDiff.text.length(); - } - if (aDiff.operation != Operation.DELETE) { - char_count2 += aDiff.text.length(); - } - } - // Pick up the leftover patch if not empty. - if (!patch.diffs.isEmpty()) { - patch_addContext(patch, prepatch_text); - patches.add(patch); - } - - return patches; - } - - /** - * Given an array of patches, return another array that is identical. - * @param patches Array of Patch objects. - * @return Array of Patch objects. - */ - public LinkedList patch_deepCopy(LinkedList patches) { - LinkedList patchesCopy = new LinkedList(); - for (Patch aPatch : patches) { - Patch patchCopy = new Patch(); - for (Diff aDiff : aPatch.diffs) { - Diff diffCopy = new Diff(aDiff.operation, aDiff.text); - patchCopy.diffs.add(diffCopy); - } - patchCopy.start1 = aPatch.start1; - patchCopy.start2 = aPatch.start2; - patchCopy.length1 = aPatch.length1; - patchCopy.length2 = aPatch.length2; - patchesCopy.add(patchCopy); - } - return patchesCopy; - } - - /** - * Merge a set of patches onto the text. Return a patched text, as well - * as an array of true/false values indicating which patches were applied. - * @param patches Array of Patch objects - * @param text Old text. - * @return Two element Object array, containing the new text and an array of - * boolean values. - */ - public Object[] patch_apply(LinkedList patches, String text) { - if (patches.isEmpty()) { - return new Object[] { - text, new boolean[0] - }; - } - - // Deep copy the patches so that no changes are made to originals. - patches = patch_deepCopy(patches); - - String nullPadding = patch_addPadding(patches); - text = nullPadding + text + nullPadding; - patch_splitMax(patches); - - int x = 0; - // delta keeps track of the offset between the expected and actual location - // of the previous patch. If there are patches expected at positions 10 and - // 20, but the first patch was found at 12, delta is 2 and the second patch - // has an effective expected position of 22. - int delta = 0; - boolean[] results = new boolean[patches.size()]; - for (Patch aPatch : patches) { - int expected_loc = aPatch.start2 + delta; - String text1 = diff_text1(aPatch.diffs); - int start_loc; - int end_loc = -1; - if (text1.length() > this.Match_MaxBits) { - // patch_splitMax will only provide an oversized pattern in the case of - // a monster delete. - start_loc = match_main( - text, - text1.substring(0, this.Match_MaxBits), expected_loc); - if (start_loc != -1) { - end_loc = match_main( - text, - text1.substring(text1.length() - this.Match_MaxBits), - expected_loc + text1.length() - this.Match_MaxBits); - if (end_loc == -1 || start_loc >= end_loc) { - // Can't find valid trailing context. Drop this patch. - start_loc = -1; - } - } - } else { - start_loc = match_main(text, text1, expected_loc); - } - if (start_loc == -1) { - // No match found. :( - results[x] = false; - // Subtract the delta for this failed patch from subsequent patches. - delta -= aPatch.length2 - aPatch.length1; - } else { - // Found a match. :) - results[x] = true; - delta = start_loc - expected_loc; - String text2; - if (end_loc == -1) { - text2 = text - .substring( - start_loc, - Math.min(start_loc + text1.length(), text.length())); - } else { - text2 = text - .substring( - start_loc, - Math.min(end_loc + this.Match_MaxBits, text.length())); - } - if (text1.equals(text2)) { - // Perfect match, just shove the replacement text in. - text = text.substring(0, start_loc) + diff_text2(aPatch.diffs) - + text.substring(start_loc + text1.length()); - } else { - // Imperfect match. Run a diff to get a framework of equivalent - // indices. - LinkedList diffs = diff_main(text1, text2, false); - if (text1.length() > this.Match_MaxBits - && diff_levenshtein(diffs) / (float) text1.length() > this.Patch_DeleteThreshold) { - // The end points match, but the content is unacceptably bad. - results[x] = false; - } else { - diff_cleanupSemanticLossless(diffs); - int index1 = 0; - for (Diff aDiff : aPatch.diffs) { - if (aDiff.operation != Operation.EQUAL) { - int index2 = diff_xIndex(diffs, index1); - if (aDiff.operation == Operation.INSERT) { - // Insertion - text = text.substring(0, start_loc + index2) + aDiff.text - + text.substring(start_loc + index2); - } else if (aDiff.operation == Operation.DELETE) { - // Deletion - text = text.substring(0, start_loc + index2) - + text - .substring( - start_loc + diff_xIndex( - diffs, - index1 + aDiff.text.length())); - } - } - if (aDiff.operation != Operation.DELETE) { - index1 += aDiff.text.length(); - } - } - } - } - } - x++; - } - // Strip the padding off. - text = text - .substring( - nullPadding.length(), text.length() - - nullPadding.length()); - return new Object[] { - text, results - }; - } - - /** - * Add some padding on text start and end so that edges can match something. - * Intended to be called only from within patch_apply. - * @param patches Array of Patch objects. - * @return The padding string added to each side. - */ - public String patch_addPadding(LinkedList patches) { - short paddingLength = this.Patch_Margin; - String nullPadding = ""; - for (short x = 1; x <= paddingLength; x++) { - nullPadding += String.valueOf((char) x); - } - - // Bump all the patches forward. - for (Patch aPatch : patches) { - aPatch.start1 += paddingLength; - aPatch.start2 += paddingLength; - } - - // Add some padding on start of first diff. - Patch patch = patches.getFirst(); - LinkedList diffs = patch.diffs; - if (diffs.isEmpty() || diffs.getFirst().operation != Operation.EQUAL) { - // Add nullPadding equality. - diffs.addFirst(new Diff(Operation.EQUAL, nullPadding)); - patch.start1 -= paddingLength; // Should be 0. - patch.start2 -= paddingLength; // Should be 0. - patch.length1 += paddingLength; - patch.length2 += paddingLength; - } else if (paddingLength > diffs.getFirst().text.length()) { - // Grow first equality. - Diff firstDiff = diffs.getFirst(); - int extraLength = paddingLength - firstDiff.text.length(); - firstDiff.text = nullPadding.substring(firstDiff.text.length()) - + firstDiff.text; - patch.start1 -= extraLength; - patch.start2 -= extraLength; - patch.length1 += extraLength; - patch.length2 += extraLength; - } - - // Add some padding on end of last diff. - patch = patches.getLast(); - diffs = patch.diffs; - if (diffs.isEmpty() || diffs.getLast().operation != Operation.EQUAL) { - // Add nullPadding equality. - diffs.addLast(new Diff(Operation.EQUAL, nullPadding)); - patch.length1 += paddingLength; - patch.length2 += paddingLength; - } else if (paddingLength > diffs.getLast().text.length()) { - // Grow last equality. - Diff lastDiff = diffs.getLast(); - int extraLength = paddingLength - lastDiff.text.length(); - lastDiff.text += nullPadding.substring(0, extraLength); - patch.length1 += extraLength; - patch.length2 += extraLength; - } - - return nullPadding; - } - - /** - * Look through the patches and break up any which are longer than the - * maximum limit of the match algorithm. - * Intended to be called only from within patch_apply. - * @param patches LinkedList of Patch objects. - */ - public void patch_splitMax(LinkedList patches) { - short patch_size = Match_MaxBits; - String precontext, postcontext; - Patch patch; - int start1, start2; - boolean empty; - Operation diff_type; - String diff_text; - ListIterator pointer = patches.listIterator(); - Patch bigpatch = pointer.hasNext() ? pointer.next() : null; - while (bigpatch != null) { - if (bigpatch.length1 <= Match_MaxBits) { - bigpatch = pointer.hasNext() ? pointer.next() : null; - continue; - } - // Remove the big old patch. - pointer.remove(); - start1 = bigpatch.start1; - start2 = bigpatch.start2; - precontext = ""; - while (!bigpatch.diffs.isEmpty()) { - // Create one of several smaller patches. - patch = new Patch(); - empty = true; - patch.start1 = start1 - precontext.length(); - patch.start2 = start2 - precontext.length(); - if (precontext.length() != 0) { - patch.length1 = patch.length2 = precontext.length(); - patch.diffs.add(new Diff(Operation.EQUAL, precontext)); - } - while (!bigpatch.diffs.isEmpty() - && patch.length1 < patch_size - Patch_Margin) { - diff_type = bigpatch.diffs.getFirst().operation; - diff_text = bigpatch.diffs.getFirst().text; - if (diff_type == Operation.INSERT) { - // Insertions are harmless. - patch.length2 += diff_text.length(); - start2 += diff_text.length(); - patch.diffs.addLast(bigpatch.diffs.removeFirst()); - empty = false; - } else if (diff_type == Operation.DELETE && patch.diffs.size() == 1 - && patch.diffs.getFirst().operation == Operation.EQUAL - && diff_text.length() > 2 * patch_size) { - // This is a large deletion. Let it pass in one chunk. - patch.length1 += diff_text.length(); - start1 += diff_text.length(); - empty = false; - patch.diffs.add(new Diff(diff_type, diff_text)); - bigpatch.diffs.removeFirst(); - } else { - // Deletion or equality. Only take as much as we can stomach. - diff_text = diff_text - .substring( - 0, Math - .min( - diff_text.length(), - patch_size - patch.length1 - Patch_Margin)); - patch.length1 += diff_text.length(); - start1 += diff_text.length(); - if (diff_type == Operation.EQUAL) { - patch.length2 += diff_text.length(); - start2 += diff_text.length(); - } else { - empty = false; - } - patch.diffs.add(new Diff(diff_type, diff_text)); - if (diff_text.equals(bigpatch.diffs.getFirst().text)) { - bigpatch.diffs.removeFirst(); - } else { - bigpatch.diffs.getFirst().text = bigpatch.diffs.getFirst().text - .substring(diff_text.length()); - } - } - } - // Compute the head context for the next patch. - precontext = diff_text2(patch.diffs); - precontext = precontext - .substring( - Math - .max( - 0, precontext.length() - - Patch_Margin)); - // Append the end context for this patch. - if (diff_text1(bigpatch.diffs).length() > Patch_Margin) { - postcontext = diff_text1(bigpatch.diffs).substring(0, Patch_Margin); - } else { - postcontext = diff_text1(bigpatch.diffs); - } - if (postcontext.length() != 0) { - patch.length1 += postcontext.length(); - patch.length2 += postcontext.length(); - if (!patch.diffs.isEmpty() - && patch.diffs.getLast().operation == Operation.EQUAL) { - patch.diffs.getLast().text += postcontext; - } else { - patch.diffs.add(new Diff(Operation.EQUAL, postcontext)); - } - } - if (!empty) { - pointer.add(patch); - } - } - bigpatch = pointer.hasNext() ? pointer.next() : null; - } - } - - /** - * Take a list of patches and return a textual representation. - * @param patches List of Patch objects. - * @return Text representation of patches. - */ - public String patch_toText(List patches) { - StringBuilder text = new StringBuilder(); - for (Patch aPatch : patches) { - text.append(aPatch); - } - return text.toString(); - } - - /** - * Parse a textual representation of patches and return a List of Patch - * objects. - * @param textline Text representation of patches. - * @return List of Patch objects. - * @throws IllegalArgumentException If invalid input. - */ - public List patch_fromText(String textline) - throws IllegalArgumentException { - List patches = new LinkedList(); - if (textline.length() == 0) { - return patches; - } - List textList = Arrays.asList(textline.split("\n")); - LinkedList text = new LinkedList(textList); - Patch patch; - Pattern patchHeader = Pattern.compile("^@@ -(\\d+),?(\\d*) \\+(\\d+),?(\\d*) @@$"); - Matcher m; - char sign; - String line; - while (!text.isEmpty()) { - m = patchHeader.matcher(text.getFirst()); - if (!m.matches()) { - throw new IllegalArgumentException( - "Invalid patch string: " + text.getFirst()); - } - patch = new Patch(); - patches.add(patch); - patch.start1 = Integer.parseInt(m.group(1)); - if (m.group(2).length() == 0) { - patch.start1--; - patch.length1 = 1; - } else if (m.group(2).equals("0")) { - patch.length1 = 0; - } else { - patch.start1--; - patch.length1 = Integer.parseInt(m.group(2)); - } - - patch.start2 = Integer.parseInt(m.group(3)); - if (m.group(4).length() == 0) { - patch.start2--; - patch.length2 = 1; - } else if (m.group(4).equals("0")) { - patch.length2 = 0; - } else { - patch.start2--; - patch.length2 = Integer.parseInt(m.group(4)); - } - text.removeFirst(); - - while (!text.isEmpty()) { - try { - sign = text.getFirst().charAt(0); - } catch (IndexOutOfBoundsException e) { - // Blank line? Whatever. - text.removeFirst(); - continue; - } - line = text.getFirst().substring(1); - line = line.replace("+", "%2B"); // decode would change all "+" to " " - try { - line = URLDecoder.decode(line, "UTF-8"); - } catch (UnsupportedEncodingException e) { - // Not likely on modern system. - throw new Error("This system does not support UTF-8.", e); - } catch (IllegalArgumentException e) { - // Malformed URI sequence. - throw new IllegalArgumentException( - "Illegal escape in patch_fromText: " + line, e); - } - if (sign == '-') { - // Deletion. - patch.diffs.add(new Diff(Operation.DELETE, line)); - } else if (sign == '+') { - // Insertion. - patch.diffs.add(new Diff(Operation.INSERT, line)); - } else if (sign == ' ') { - // Minor equality. - patch.diffs.add(new Diff(Operation.EQUAL, line)); - } else if (sign == '@') { - // Start of next patch. - break; - } else { - // WTF? - throw new IllegalArgumentException( - "Invalid patch mode '" + sign + "' in: " + line); - } - text.removeFirst(); - } - } - return patches; - } - - /** - * Class representing one diff operation. - */ - public static class Diff { - /** - * One of: INSERT, DELETE or EQUAL. - */ - public Operation operation; - /** - * The text associated with this diff operation. - */ - public String text; - - /** - * Constructor. Initializes the diff with the provided values. - * @param operation One of INSERT, DELETE or EQUAL. - * @param text The text being applied. - */ - public Diff(Operation operation, String text) { - // Construct a diff with the specified operation and text. - this.operation = operation; - this.text = text; - } - - /** - * Display a human-readable version of this Diff. - * @return text version. - */ - public String toString() { - String prettyText = this.text.replace('\n', '\u00b6'); - return "Diff(" + this.operation + ",\"" + prettyText + "\")"; - } - - /** - * Create a numeric hash value for a Diff. - * This function is not used by DMP. - * @return Hash value. - */ - @Override - public int hashCode() { - final int prime = 31; - int result = (operation == null) ? 0 : operation.hashCode(); - result += prime * ((text == null) ? 0 : text.hashCode()); - return result; - } - - /** - * Is this Diff equivalent to another Diff? - * @param obj Another Diff to compare against. - * @return true or false. - */ - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - Diff other = (Diff) obj; - if (operation != other.operation) { - return false; - } - if (text == null) { - if (other.text != null) { - return false; - } - } else if (!text.equals(other.text)) { - return false; - } - return true; - } - } - - /** - * Class representing one patch operation. - */ - public static class Patch { - public LinkedList diffs; - public int start1; - public int start2; - public int length1; - public int length2; - - /** - * Constructor. Initializes with an empty list of diffs. - */ - public Patch() { - this.diffs = new LinkedList(); - } - - /** - * Emulate GNU diff's format. - * Header: @@ -382,8 +481,9 @@ - * Indices are printed as 1-based, not 0-based. - * @return The GNU diff string. - */ - public String toString() { - String coords1, coords2; - if (this.length1 == 0) { - coords1 = this.start1 + ",0"; - } else if (this.length1 == 1) { - coords1 = Integer.toString(this.start1 + 1); - } else { - coords1 = (this.start1 + 1) + "," + this.length1; - } - if (this.length2 == 0) { - coords2 = this.start2 + ",0"; - } else if (this.length2 == 1) { - coords2 = Integer.toString(this.start2 + 1); - } else { - coords2 = (this.start2 + 1) + "," + this.length2; - } - StringBuilder text = new StringBuilder(); - text - .append("@@ -") - .append(coords1) - .append(" +") - .append(coords2) - .append(" @@\n"); - // Escape the body of the patch with %xx notation. - for (Diff aDiff : this.diffs) { - switch (aDiff.operation) { - case INSERT: - text.append('+'); - break; - case DELETE: - text.append('-'); - break; - case EQUAL: - text.append(' '); - break; - } - try { - text - .append(URLEncoder.encode(aDiff.text, "UTF-8").replace('+', ' ')) - .append("\n"); - } catch (UnsupportedEncodingException e) { - // Not likely on modern system. - throw new Error("This system does not support UTF-8.", e); - } - } - return unescapeForEncodeUriCompatability(text.toString()); - } - } - - /** - * Unescape selected chars for compatability with JavaScript's encodeURI. - * In speed critical applications this could be dropped since the - * receiving application will certainly decode these fine. - * Note that this function is case-sensitive. Thus "%3f" would not be - * unescaped. But this is ok because it is only called with the output of - * URLEncoder.encode which returns uppercase hex. - * - * Example: "%3F" -> "?", "%24" -> "$", etc. - * - * @param str The string to escape. - * @return The escaped string. - */ - private static String unescapeForEncodeUriCompatability(String str) { - return str - .replace("%21", "!") - .replace("%7E", "~") - .replace("%27", "'") - .replace("%28", "(") - .replace("%29", ")") - .replace("%3B", ";") - .replace("%2F", "/") - .replace("%3F", "?") - .replace("%3A", ":") - .replace("%40", "@") - .replace("%26", "&") - .replace("%3D", "=") - .replace("%2B", "+") - .replace("%24", "$") - .replace("%2C", ",") - .replace("%23", "#"); - } -} diff --git a/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DotAbbreviations.java b/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DotAbbreviations.java deleted file mode 100644 index 33183b0f6..000000000 --- a/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DotAbbreviations.java +++ /dev/null @@ -1,11 +0,0 @@ - -package eu.dnetlib.pace.util; - -import com.google.common.base.Function; - -public class DotAbbreviations implements Function { - @Override - public String apply(String s) { - return s.length() == 1 ? s + "." : s; - } -}; diff --git a/dhp-workflows/dhp-actionmanager/pom.xml b/dhp-workflows/dhp-actionmanager/pom.xml index 5a5f156fc..ce13502b6 100644 --- a/dhp-workflows/dhp-actionmanager/pom.xml +++ b/dhp-workflows/dhp-actionmanager/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp-workflows - 1.2.5-beta + 1.2.5-SNAPSHOT dhp-actionmanager diff --git a/dhp-workflows/dhp-aggregation/pom.xml b/dhp-workflows/dhp-aggregation/pom.xml index d67e880b4..108d25ba6 100644 --- a/dhp-workflows/dhp-aggregation/pom.xml +++ b/dhp-workflows/dhp-aggregation/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp-workflows - 1.2.5-beta + 1.2.5-SNAPSHOT dhp-aggregation diff --git a/dhp-workflows/dhp-blacklist/pom.xml b/dhp-workflows/dhp-blacklist/pom.xml index 64be812ba..7ecc8b35d 100644 --- a/dhp-workflows/dhp-blacklist/pom.xml +++ b/dhp-workflows/dhp-blacklist/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-broker-events/pom.xml b/dhp-workflows/dhp-broker-events/pom.xml index b9f572527..322fc7e93 100644 --- a/dhp-workflows/dhp-broker-events/pom.xml +++ b/dhp-workflows/dhp-broker-events/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-dedup-openaire/pom.xml b/dhp-workflows/dhp-dedup-openaire/pom.xml index bc1538e17..8665ebd05 100644 --- a/dhp-workflows/dhp-dedup-openaire/pom.xml +++ b/dhp-workflows/dhp-dedup-openaire/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-SNAPSHOT 4.0.0 dhp-dedup-openaire diff --git a/dhp-workflows/dhp-doiboost/pom.xml b/dhp-workflows/dhp-doiboost/pom.xml index cfa5a3fce..6e8911fba 100644 --- a/dhp-workflows/dhp-doiboost/pom.xml +++ b/dhp-workflows/dhp-doiboost/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-enrichment/pom.xml b/dhp-workflows/dhp-enrichment/pom.xml index d7f75de8c..9698dee03 100644 --- a/dhp-workflows/dhp-enrichment/pom.xml +++ b/dhp-workflows/dhp-enrichment/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-SNAPSHOT 4.0.0 @@ -51,7 +51,7 @@ eu.dnetlib.dhp dhp-aggregation - 1.2.5-beta + 1.2.5-SNAPSHOT compile diff --git a/dhp-workflows/dhp-graph-mapper/pom.xml b/dhp-workflows/dhp-graph-mapper/pom.xml index 2c93bab83..d7ae60a91 100644 --- a/dhp-workflows/dhp-graph-mapper/pom.xml +++ b/dhp-workflows/dhp-graph-mapper/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-graph-provision/pom.xml b/dhp-workflows/dhp-graph-provision/pom.xml index 7b879e074..e62fcdf19 100644 --- a/dhp-workflows/dhp-graph-provision/pom.xml +++ b/dhp-workflows/dhp-graph-provision/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/dhp-impact-indicators/pom.xml b/dhp-workflows/dhp-impact-indicators/pom.xml index d931c2323..a9eb0a4a1 100644 --- a/dhp-workflows/dhp-impact-indicators/pom.xml +++ b/dhp-workflows/dhp-impact-indicators/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp-workflows - 1.2.5-beta + 1.2.5-SNAPSHOT dhp-impact-indicators diff --git a/dhp-workflows/dhp-stats-actionsets/pom.xml b/dhp-workflows/dhp-stats-actionsets/pom.xml index 5d9b60b87..3daa8f995 100644 --- a/dhp-workflows/dhp-stats-actionsets/pom.xml +++ b/dhp-workflows/dhp-stats-actionsets/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp-workflows - 1.2.5-beta + 1.2.5-SNAPSHOT dhp-stats-actionsets diff --git a/dhp-workflows/dhp-stats-hist-snaps/pom.xml b/dhp-workflows/dhp-stats-hist-snaps/pom.xml index 94371dc0b..b31d909f9 100644 --- a/dhp-workflows/dhp-stats-hist-snaps/pom.xml +++ b/dhp-workflows/dhp-stats-hist-snaps/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-SNAPSHOT 4.0.0 dhp-stats-hist-snaps diff --git a/dhp-workflows/dhp-stats-monitor-irish/pom.xml b/dhp-workflows/dhp-stats-monitor-irish/pom.xml index 4887005bb..6ab19dced 100644 --- a/dhp-workflows/dhp-stats-monitor-irish/pom.xml +++ b/dhp-workflows/dhp-stats-monitor-irish/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-SNAPSHOT 4.0.0 dhp-stats-monitor-irish diff --git a/dhp-workflows/dhp-stats-monitor-update/pom.xml b/dhp-workflows/dhp-stats-monitor-update/pom.xml index c8a69c078..f2bc35f8d 100644 --- a/dhp-workflows/dhp-stats-monitor-update/pom.xml +++ b/dhp-workflows/dhp-stats-monitor-update/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-SNAPSHOT 4.0.0 dhp-stats-monitor-update diff --git a/dhp-workflows/dhp-stats-promote/pom.xml b/dhp-workflows/dhp-stats-promote/pom.xml index 1c711c878..9e17a78dc 100644 --- a/dhp-workflows/dhp-stats-promote/pom.xml +++ b/dhp-workflows/dhp-stats-promote/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-SNAPSHOT 4.0.0 dhp-stats-promote diff --git a/dhp-workflows/dhp-stats-update/pom.xml b/dhp-workflows/dhp-stats-update/pom.xml index 246aa63cf..cc15b8a15 100644 --- a/dhp-workflows/dhp-stats-update/pom.xml +++ b/dhp-workflows/dhp-stats-update/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-SNAPSHOT 4.0.0 dhp-stats-update diff --git a/dhp-workflows/dhp-swh/pom.xml b/dhp-workflows/dhp-swh/pom.xml index 4ba5cf868..80fff4587 100644 --- a/dhp-workflows/dhp-swh/pom.xml +++ b/dhp-workflows/dhp-swh/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dhp-workflows - 1.2.5-beta + 1.2.5-SNAPSHOT dhp-swh diff --git a/dhp-workflows/dhp-usage-raw-data-update/pom.xml b/dhp-workflows/dhp-usage-raw-data-update/pom.xml index ed3616fde..a9dbb09ae 100644 --- a/dhp-workflows/dhp-usage-raw-data-update/pom.xml +++ b/dhp-workflows/dhp-usage-raw-data-update/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-SNAPSHOT 4.0.0 dhp-usage-raw-data-update diff --git a/dhp-workflows/dhp-usage-stats-build/pom.xml b/dhp-workflows/dhp-usage-stats-build/pom.xml index 52cc3bf44..56aec73b7 100644 --- a/dhp-workflows/dhp-usage-stats-build/pom.xml +++ b/dhp-workflows/dhp-usage-stats-build/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-SNAPSHOT 4.0.0 dhp-usage-stats-build diff --git a/dhp-workflows/dhp-workflow-profiles/pom.xml b/dhp-workflows/dhp-workflow-profiles/pom.xml index ef4e0ada6..8c71a5ca1 100644 --- a/dhp-workflows/dhp-workflow-profiles/pom.xml +++ b/dhp-workflows/dhp-workflow-profiles/pom.xml @@ -3,7 +3,7 @@ dhp-workflows eu.dnetlib.dhp - 1.2.5-beta + 1.2.5-SNAPSHOT 4.0.0 diff --git a/dhp-workflows/pom.xml b/dhp-workflows/pom.xml index 9b87c7b44..1c331d126 100644 --- a/dhp-workflows/pom.xml +++ b/dhp-workflows/pom.xml @@ -6,7 +6,7 @@ eu.dnetlib.dhp dhp - 1.2.5-beta + 1.2.5-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index d015acd9e..892382b9d 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 eu.dnetlib.dhp dhp - 1.2.5-beta + 1.2.5-SNAPSHOT pom From 0646d0d0645341020ee12c284e0872e6e450cc11 Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Thu, 2 May 2024 15:15:03 +0200 Subject: [PATCH 35/97] Updated main sparkApplication to avoid to require master variable --- .../eu/dnetlib/dhp/application/SparkScalaApplication.scala | 7 ++++--- .../eu/dnetlib/dhp/sx/create_scholix_dump_params.json | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dhp-common/src/main/scala/eu/dnetlib/dhp/application/SparkScalaApplication.scala b/dhp-common/src/main/scala/eu/dnetlib/dhp/application/SparkScalaApplication.scala index a14c25837..526bbd295 100644 --- a/dhp-common/src/main/scala/eu/dnetlib/dhp/application/SparkScalaApplication.scala +++ b/dhp-common/src/main/scala/eu/dnetlib/dhp/application/SparkScalaApplication.scala @@ -65,12 +65,13 @@ abstract class AbstractScalaApplication( val conf: SparkConf = new SparkConf() val master = parser.get("master") log.info(s"Creating Spark session: Master: $master") - SparkSession + val b = SparkSession .builder() .config(conf) .appName(getClass.getSimpleName) - .master(master) - .getOrCreate() + if (master != null) + b.master(master) + b.getOrCreate() } def reportTotalSize(targetPath: String, outputBasePath: String): Unit = { diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/create_scholix_dump_params.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/create_scholix_dump_params.json index fead58ab1..53fe95895 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/create_scholix_dump_params.json +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/sx/create_scholix_dump_params.json @@ -1,5 +1,5 @@ [ - {"paramName":"mt", "paramLongName":"master", "paramDescription": "should be local or yarn", "paramRequired": true}, + {"paramName":"mt", "paramLongName":"master", "paramDescription": "should be local or yarn", "paramRequired": false}, {"paramName":"s", "paramLongName":"sourcePath", "paramDescription": "the source Path", "paramRequired": true}, {"paramName":"t", "paramLongName":"targetPath", "paramDescription": "the path of the scholix dump", "paramRequired": true} ] \ No newline at end of file From a860c57bbc2c6ae788c91c103873dc942e7ff473 Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Thu, 2 May 2024 15:16:00 +0200 Subject: [PATCH 36/97] updated .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 14cd4d345..6fafc7055 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ spark-warehouse /**/.factorypath /**/.scalafmt.conf /.java-version +/dhp-shade-package/dependency-reduced-pom.xml From db358ad0d2ffb63cd7215ec89e693274982b78e1 Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Thu, 2 May 2024 15:25:57 +0200 Subject: [PATCH 37/97] code formatted --- .../eu/dnetlib/pace/common/PaceCommonUtils.java | 15 ++++++++------- .../main/java/eu/dnetlib/pace/model/Person.java | 11 ++++++----- .../java/eu/dnetlib/pace/util/Capitalise.java | 3 ++- .../pace/common/AbstractPaceFunctions.java | 13 +++++++------ 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/pace/common/PaceCommonUtils.java b/dhp-common/src/main/java/eu/dnetlib/pace/common/PaceCommonUtils.java index a279271b5..61fbc2470 100644 --- a/dhp-common/src/main/java/eu/dnetlib/pace/common/PaceCommonUtils.java +++ b/dhp-common/src/main/java/eu/dnetlib/pace/common/PaceCommonUtils.java @@ -1,19 +1,20 @@ package eu.dnetlib.pace.common; -import com.google.common.base.Splitter; -import com.google.common.collect.Iterables; -import com.google.common.collect.Sets; -import com.ibm.icu.text.Transliterator; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; - import java.nio.charset.StandardCharsets; import java.text.Normalizer; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; + +import com.google.common.base.Splitter; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import com.ibm.icu.text.Transliterator; + /** * Set of common functions for the framework * diff --git a/dhp-common/src/main/java/eu/dnetlib/pace/model/Person.java b/dhp-common/src/main/java/eu/dnetlib/pace/model/Person.java index c95c9d823..6a1957183 100644 --- a/dhp-common/src/main/java/eu/dnetlib/pace/model/Person.java +++ b/dhp-common/src/main/java/eu/dnetlib/pace/model/Person.java @@ -1,20 +1,21 @@ package eu.dnetlib.pace.model; +import java.nio.charset.Charset; +import java.text.Normalizer; +import java.util.List; +import java.util.Set; + import com.google.common.base.Joiner; import com.google.common.base.Splitter; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.hash.Hashing; + import eu.dnetlib.pace.common.PaceCommonUtils; import eu.dnetlib.pace.util.Capitalise; import eu.dnetlib.pace.util.DotAbbreviations; -import java.nio.charset.Charset; -import java.text.Normalizer; -import java.util.List; -import java.util.Set; - public class Person { private static final String UTF8 = "UTF-8"; diff --git a/dhp-common/src/main/java/eu/dnetlib/pace/util/Capitalise.java b/dhp-common/src/main/java/eu/dnetlib/pace/util/Capitalise.java index 015386423..671320c71 100644 --- a/dhp-common/src/main/java/eu/dnetlib/pace/util/Capitalise.java +++ b/dhp-common/src/main/java/eu/dnetlib/pace/util/Capitalise.java @@ -1,9 +1,10 @@ package eu.dnetlib.pace.util; -import com.google.common.base.Function; import org.apache.commons.lang3.text.WordUtils; +import com.google.common.base.Function; + public class Capitalise implements Function { private final char[] DELIM = { diff --git a/dhp-pace-core/src/main/java/eu/dnetlib/pace/common/AbstractPaceFunctions.java b/dhp-pace-core/src/main/java/eu/dnetlib/pace/common/AbstractPaceFunctions.java index 6bfb8b3f4..b055077d8 100644 --- a/dhp-pace-core/src/main/java/eu/dnetlib/pace/common/AbstractPaceFunctions.java +++ b/dhp-pace-core/src/main/java/eu/dnetlib/pace/common/AbstractPaceFunctions.java @@ -1,12 +1,6 @@ package eu.dnetlib.pace.common; -import com.google.common.base.Joiner; -import com.google.common.collect.Sets; -import com.ibm.icu.text.Transliterator; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; - import java.io.IOException; import java.io.StringWriter; import java.nio.charset.StandardCharsets; @@ -15,6 +9,13 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; + +import com.google.common.base.Joiner; +import com.google.common.collect.Sets; +import com.ibm.icu.text.Transliterator; + /** * Set of common functions for the framework * From 69c5efbd8b2015f993a04205e117cbb4b204f0e2 Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Fri, 3 May 2024 13:57:56 +0200 Subject: [PATCH 38/97] Fix: when applying enrichments with no instance information the resulting merge entity was generated with no instance instead of keeping the original information --- .../java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils.java index 9eb1ec01d..28db94766 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils.java @@ -874,9 +874,11 @@ public class MergeUtils { if (toEnrichInstances == null) { return enrichmentResult; } - if (enrichmentInstances == null) { - return enrichmentResult; + + if (enrichmentInstances == null || enrichmentInstances.isEmpty()) { + return toEnrichInstances; } + Map ri = toInstanceMap(enrichmentInstances); toEnrichInstances.forEach(i -> { From e1a0fb89334da1f6f8944c1138f3f9ba841e6493 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 3 May 2024 14:14:18 +0200 Subject: [PATCH 39/97] fixed id prefix creation for the fosnodoi records --- .../createunresolvedentities/PrepareFOSSparkJob.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareFOSSparkJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareFOSSparkJob.java index ffcaedda7..dd85f6a4e 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareFOSSparkJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareFOSSparkJob.java @@ -80,9 +80,10 @@ public class PrepareFOSSparkJob implements Serializable { fosDataset .groupByKey((MapFunction) v -> v.getOaid().toLowerCase(), Encoders.STRING()) - .mapGroups((MapGroupsFunction) (k, it) -> { - return getResult(ModelSupport.getIdPrefix(Result.class) + "|" + k, it); - }, Encoders.bean(Result.class)) + .mapGroups( + (MapGroupsFunction) (k, + it) -> getResult(ModelSupport.entityIdPrefix.get(Result.class.getSimpleName()) + "|" + k, it), + Encoders.bean(Result.class)) .write() .mode(SaveMode.Overwrite) .option("compression", "gzip") From a5d13d5d2777f36124a86a563a18052d3b41c2a2 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 3 May 2024 14:14:34 +0200 Subject: [PATCH 40/97] code formatting --- .../eu/dnetlib/pace/common/PaceCommonUtils.java | 15 ++++++++------- .../main/java/eu/dnetlib/pace/model/Person.java | 11 ++++++----- .../java/eu/dnetlib/pace/util/Capitalise.java | 3 ++- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/pace/common/PaceCommonUtils.java b/dhp-common/src/main/java/eu/dnetlib/pace/common/PaceCommonUtils.java index a279271b5..61fbc2470 100644 --- a/dhp-common/src/main/java/eu/dnetlib/pace/common/PaceCommonUtils.java +++ b/dhp-common/src/main/java/eu/dnetlib/pace/common/PaceCommonUtils.java @@ -1,19 +1,20 @@ package eu.dnetlib.pace.common; -import com.google.common.base.Splitter; -import com.google.common.collect.Iterables; -import com.google.common.collect.Sets; -import com.ibm.icu.text.Transliterator; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; - import java.nio.charset.StandardCharsets; import java.text.Normalizer; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; + +import com.google.common.base.Splitter; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import com.ibm.icu.text.Transliterator; + /** * Set of common functions for the framework * diff --git a/dhp-common/src/main/java/eu/dnetlib/pace/model/Person.java b/dhp-common/src/main/java/eu/dnetlib/pace/model/Person.java index c95c9d823..6a1957183 100644 --- a/dhp-common/src/main/java/eu/dnetlib/pace/model/Person.java +++ b/dhp-common/src/main/java/eu/dnetlib/pace/model/Person.java @@ -1,20 +1,21 @@ package eu.dnetlib.pace.model; +import java.nio.charset.Charset; +import java.text.Normalizer; +import java.util.List; +import java.util.Set; + import com.google.common.base.Joiner; import com.google.common.base.Splitter; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.hash.Hashing; + import eu.dnetlib.pace.common.PaceCommonUtils; import eu.dnetlib.pace.util.Capitalise; import eu.dnetlib.pace.util.DotAbbreviations; -import java.nio.charset.Charset; -import java.text.Normalizer; -import java.util.List; -import java.util.Set; - public class Person { private static final String UTF8 = "UTF-8"; diff --git a/dhp-common/src/main/java/eu/dnetlib/pace/util/Capitalise.java b/dhp-common/src/main/java/eu/dnetlib/pace/util/Capitalise.java index 015386423..671320c71 100644 --- a/dhp-common/src/main/java/eu/dnetlib/pace/util/Capitalise.java +++ b/dhp-common/src/main/java/eu/dnetlib/pace/util/Capitalise.java @@ -1,9 +1,10 @@ package eu.dnetlib.pace.util; -import com.google.common.base.Function; import org.apache.commons.lang3.text.WordUtils; +import com.google.common.base.Function; + public class Capitalise implements Function { private final char[] DELIM = { From 04862271850f22c92145e878005b62217af8d1d2 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 3 May 2024 14:31:12 +0200 Subject: [PATCH 41/97] [cleaning] deactivating the cleaning of FOS subjects found in the metadata provided by repositories --- .../dhp/oa/graph/clean/CleaningRuleMap.java | 38 ++++++++++++++----- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleMap.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleMap.java index 807055adb..732471f99 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleMap.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/CleaningRuleMap.java @@ -4,6 +4,7 @@ package eu.dnetlib.dhp.oa.graph.clean; import java.io.Serializable; import java.util.HashMap; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; import org.apache.commons.lang3.SerializationUtils; @@ -29,7 +30,10 @@ public class CleaningRuleMap extends HashMap, SerializableConsumer cleanQualifier(vocabularies, (AccessRight) o)); mapping.put(Country.class, o -> cleanCountry(vocabularies, (Country) o)); mapping.put(Relation.class, o -> cleanRelation(vocabularies, (Relation) o)); - mapping.put(Subject.class, o -> cleanSubject(vocabularies, (Subject) o)); + + // commenting out the subject cleaning until we decide if we want to it or not and the implementation will + // be completed. At the moment it is not capable of expanding the whole hierarchy. + // mapping.put(Subject.class, o -> cleanSubject(vocabularies, (Subject) o)); return mapping; } @@ -38,8 +42,15 @@ public class CleaningRuleMap extends HashMap, SerializableConsumer { if (ModelConstants.DNET_SUBJECT_KEYWORD.equalsIgnoreCase(subject.getQualifier().getClassid())) { @@ -49,14 +60,21 @@ public class CleaningRuleMap extends HashMap, SerializableConsumer Date: Fri, 3 May 2024 15:53:52 +0200 Subject: [PATCH 42/97] fixed id prefix creation for the fosnodoi records, again --- .../createunresolvedentities/PrepareFOSSparkJob.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareFOSSparkJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareFOSSparkJob.java index dd85f6a4e..c248423d4 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareFOSSparkJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareFOSSparkJob.java @@ -82,7 +82,8 @@ public class PrepareFOSSparkJob implements Serializable { .groupByKey((MapFunction) v -> v.getOaid().toLowerCase(), Encoders.STRING()) .mapGroups( (MapGroupsFunction) (k, - it) -> getResult(ModelSupport.entityIdPrefix.get(Result.class.getSimpleName()) + "|" + k, it), + it) -> getResult( + ModelSupport.entityIdPrefix.get(Result.class.getSimpleName().toLowerCase()) + "|" + k, it), Encoders.bean(Result.class)) .write() .mode(SaveMode.Overwrite) From ed052a3476bf5c8980412b0d1b8387491d761ab2 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 6 May 2024 16:08:33 +0200 Subject: [PATCH 43/97] job for the population of the oai database --- .../dhp/oa/oaipmh/IrishOaiExporterJob.java | 156 ++++++++++++++++++ .../dhp/oa/oaipmh/OaiRecordWrapper.java | 50 ++++++ 2 files changed, 206 insertions(+) create mode 100644 dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java create mode 100644 dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/OaiRecordWrapper.java diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java new file mode 100644 index 000000000..9a608b6fa --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java @@ -0,0 +1,156 @@ +package eu.dnetlib.dhp.oa.oaipmh; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Optional; +import java.util.Properties; +import java.util.zip.GZIPOutputStream; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.FilterFunction; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoder; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SaveMode; +import org.dom4j.Document; +import org.dom4j.DocumentHelper; +import org.dom4j.Node; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.oa.provision.XmlConverterJob; +import eu.dnetlib.dhp.oa.provision.model.SerializableSolrInputDocument; +import eu.dnetlib.dhp.oa.provision.model.TupleWrapper; + +public class IrishOaiExporterJob { + + private static final Logger log = LoggerFactory.getLogger(IrishOaiExporterJob.class); + + protected static final int NUM_CONNECTIONS = 20; + + public static void main(final String[] args) throws Exception { + + final ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString(XmlConverterJob.class + .getResourceAsStream("/eu/dnetlib/dhp/oa/oaipmh/input_params_irish_oai_exporter.json"))); + parser.parseArgument(args); + + final Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + final String inputPath = parser.get("inputPath"); + final String dbUrl = parser.get("dbUrl"); + final String dbTable = parser.get("dbTable"); + final String dbUser = parser.get("dbUser"); + final String dbPwd = parser.get("dbPwd"); + final int numConnections = Optional + .ofNullable(parser.get("numConnections")) + .map(Integer::valueOf) + .orElse(NUM_CONNECTIONS); + + log.info("inputPath: '{}'", inputPath); + log.info("dbUrl: '{}'", dbUrl); + log.info("dbUser: '{}'", dbUser); + log.info("table: '{}'", dbTable); + log.info("dbPwd: '{}'", "xxx"); + log.info("numPartitions: '{}'", numConnections); + + final Properties connectionProperties = new Properties(); + connectionProperties.put("user", dbUser); + connectionProperties.put("password", dbPwd); + + final SparkConf conf = new SparkConf(); + conf.registerKryoClasses(new Class[] { + SerializableSolrInputDocument.class + }); + + final Encoder encoderTuple = Encoders.bean(TupleWrapper.class); + final Encoder encoderOaiRecord = Encoders.bean(OaiRecordWrapper.class); + + runWithSparkSession(conf, isSparkSessionManaged, spark -> { + + final Dataset docs = spark + .read() + .schema(encoderTuple.schema()) + .json(inputPath) + .as(encoderTuple) + .map((MapFunction) TupleWrapper::getXml, Encoders.STRING()) + .map((MapFunction) IrishOaiExporterJob::asIrishOaiResult, encoderOaiRecord) + .filter((FilterFunction) obj -> (obj != null) && StringUtils.isNotBlank(obj.getId())); + + docs.repartition(numConnections) + .write() + .mode(SaveMode.Overwrite) + .jdbc(dbUrl, dbTable, connectionProperties); + + }); + } + + private static OaiRecordWrapper asIrishOaiResult(final String xml) { + try { + final Document doc = DocumentHelper.parseText(xml); + final OaiRecordWrapper r = new OaiRecordWrapper(); + + if (isValid(doc)) { + r.setId(doc.valueOf("//*[local-name()='objIdentifier']").trim()); + r.setBody(gzip(xml)); + r.setDate(LocalDateTime.now()); + r.setSets(new ArrayList<>()); + } + return r; + } catch (final Exception e) { + log.error("Error parsing record: " + xml, e); + throw new RuntimeException("Error parsing record: " + xml, e); + } + } + + private static boolean isValid(final Document doc) { + + final Node n = doc.selectSingleNode("//*[local-name()='entity']/*[local-name()='result']"); + + if (n != null) { + for (final Object o : n.selectNodes(".//*[local-name()='datainfo']/*[local-name()='deletedbyinference']")) { + if ("true".equals(((Node) o).getText().trim())) { return false; } + } + + for (final Object o : n.selectNodes("./*[local-name()='country']")) { + if ("IE".equals(((Node) o).valueOf("@classid").trim())) { return true; } + } + + for (final Object o : n.selectNodes(".//*[local-name()='rel']")) { + final String relType = ((Node) o).valueOf("./[local-name() = 'to']/@type").trim(); + final String relCountry = ((Node) o).valueOf("./*[local-name() = 'country']/@classid").trim(); + if ("organization".equals(relType) && "IE".equals(relCountry)) { return true; } + } + } + return false; + + } + + private static byte[] gzip(final String str) { + if (StringUtils.isBlank(str)) { return null; } + + try { + final ByteArrayOutputStream obj = new ByteArrayOutputStream(); + final GZIPOutputStream gzip = new GZIPOutputStream(obj); + gzip.write(str.getBytes("UTF-8")); + gzip.flush(); + gzip.close(); + return obj.toByteArray(); + } catch (final IOException e) { + throw new RuntimeException("error in gzip", e); + } + } +} diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/OaiRecordWrapper.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/OaiRecordWrapper.java new file mode 100644 index 000000000..4c2766754 --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/OaiRecordWrapper.java @@ -0,0 +1,50 @@ +package eu.dnetlib.dhp.oa.oaipmh; + +import java.io.Serializable; +import java.time.LocalDateTime; +import java.util.List; + +public class OaiRecordWrapper implements Serializable { + + private static final long serialVersionUID = 8997046455575004880L; + + private String id; + private byte[] body; + private LocalDateTime date; + private List sets; + + public OaiRecordWrapper() {} + + public String getId() { + return this.id; + } + + public void setId(final String id) { + this.id = id; + } + + public byte[] getBody() { + return this.body; + } + + public void setBody(final byte[] body) { + this.body = body; + } + + public LocalDateTime getDate() { + return this.date; + } + + public void setDate(final LocalDateTime date) { + this.date = date; + } + + public List getSets() { + return this.sets; + } + + public void setSets(final List sets) { + this.sets = sets; + } + +} From aa40e53c19acf6c8007b7819bea3e65ba642e057 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Tue, 7 May 2024 08:01:19 +0200 Subject: [PATCH 44/97] oai exporter parameters --- .../input_params_irish_oai_exporter.json | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/oaipmh/input_params_irish_oai_exporter.json diff --git a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/oaipmh/input_params_irish_oai_exporter.json b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/oaipmh/input_params_irish_oai_exporter.json new file mode 100644 index 000000000..99a12927b --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/oaipmh/input_params_irish_oai_exporter.json @@ -0,0 +1,38 @@ +[ + { + "paramName": "i", + "paramLongName": "inputPath", + "paramDescription": "The path of the input records on HDFS", + "paramRequired": true + }, + { + "paramName": "nc", + "paramLongName": "numConnections", + "paramDescription": "number of connections to the postgres db (for the write operation)", + "paramRequired": false + }, + { + "paramName": "du", + "paramLongName": "dbUrl", + "paramDescription": "the url of the database", + "paramRequired": true + }, + { + "paramName": "dusr", + "paramLongName": "dbUser", + "paramDescription": "the user of the database", + "paramRequired": true + }, + { + "paramName": "t", + "paramLongName": "dbTable", + "paramDescription": "the name of the table in the database", + "paramRequired": true + }, + { + "paramName": "dpwd", + "paramLongName": "dbPwd", + "paramDescription": "the password for the user of the database", + "paramRequired": true + } +] From 70bf6ac41561d487109a04ef60b0659a8785d989 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Tue, 7 May 2024 09:36:26 +0200 Subject: [PATCH 45/97] oai exporter tests --- .../dhp/oa/oaipmh/IrishOaiExporterJob.java | 11 ++- .../oa/oaipmh/IrishOaiExporterJobTest.java | 93 +++++++++++++++++++ .../eu/dnetlib/dhp/oa/oaipmh/record_IE.xml | 89 ++++++++++++++++++ .../dhp/oa/oaipmh/record_IE_deleted.xml | 89 ++++++++++++++++++ .../eu/dnetlib/dhp/oa/oaipmh/record_IT.xml | 66 +++++++++++++ 5 files changed, 344 insertions(+), 4 deletions(-) create mode 100644 dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJobTest.java create mode 100644 dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/oaipmh/record_IE.xml create mode 100644 dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/oaipmh/record_IE_deleted.xml create mode 100644 dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/oaipmh/record_IT.xml diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java index 9a608b6fa..e2ae890e5 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java @@ -98,7 +98,7 @@ public class IrishOaiExporterJob { }); } - private static OaiRecordWrapper asIrishOaiResult(final String xml) { + protected static OaiRecordWrapper asIrishOaiResult(final String xml) { try { final Document doc = DocumentHelper.parseText(xml); final OaiRecordWrapper r = new OaiRecordWrapper(); @@ -116,21 +116,24 @@ public class IrishOaiExporterJob { } } - private static boolean isValid(final Document doc) { + protected static boolean isValid(final Document doc) { final Node n = doc.selectSingleNode("//*[local-name()='entity']/*[local-name()='result']"); if (n != null) { + for (final Object o : n.selectNodes(".//*[local-name()='datainfo']/*[local-name()='deletedbyinference']")) { if ("true".equals(((Node) o).getText().trim())) { return false; } } + // verify the main country of the result for (final Object o : n.selectNodes("./*[local-name()='country']")) { if ("IE".equals(((Node) o).valueOf("@classid").trim())) { return true; } } + // verify the countries of the related organizations for (final Object o : n.selectNodes(".//*[local-name()='rel']")) { - final String relType = ((Node) o).valueOf("./[local-name() = 'to']/@type").trim(); + final String relType = ((Node) o).valueOf("./*[local-name() = 'to']/@type").trim(); final String relCountry = ((Node) o).valueOf("./*[local-name() = 'country']/@classid").trim(); if ("organization".equals(relType) && "IE".equals(relCountry)) { return true; } } @@ -139,7 +142,7 @@ public class IrishOaiExporterJob { } - private static byte[] gzip(final String str) { + protected static byte[] gzip(final String str) { if (StringUtils.isBlank(str)) { return null; } try { diff --git a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJobTest.java b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJobTest.java new file mode 100644 index 000000000..6140b0907 --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJobTest.java @@ -0,0 +1,93 @@ +package eu.dnetlib.dhp.oa.oaipmh; + +import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.zip.GZIPInputStream; + +import org.apache.commons.io.IOUtils; +import org.dom4j.Document; +import org.dom4j.DocumentException; +import org.dom4j.io.SAXReader; +import org.junit.jupiter.api.Test; + +class IrishOaiExporterJobTest { + + @Test + void testAsIrishOaiResult() throws Exception { + final String xml = IOUtils.toString(getClass().getResourceAsStream("record_IE.xml")); + final OaiRecordWrapper res = IrishOaiExporterJob.asIrishOaiResult(xml); + assertNotNull(res.getId()); + assertNotNull(res.getBody()); + assertNotNull(res.getSets()); + assertNotNull(res.getDate()); + assertEquals("dedup_wf_002::532be02f990b479a1da46d71f1a4c3f0", res.getId()); + assertTrue(res.getBody().length > 0); + assertTrue(res.getSets().isEmpty()); + } + + @Test + void testIsValid_IE() throws DocumentException { + final Document doc = new SAXReader().read(getClass().getResourceAsStream("record_IE.xml")); + assertTrue(IrishOaiExporterJob.isValid(doc)); + } + + @Test + void testIsValid_invalid_country() throws DocumentException { + final Document doc = new SAXReader().read(getClass().getResourceAsStream("record_IT.xml")); + assertFalse(IrishOaiExporterJob.isValid(doc)); + } + + @Test + void testIsValid_deleted() throws DocumentException { + final Document doc = new SAXReader().read(getClass().getResourceAsStream("record_IE_deleted.xml")); + assertFalse(IrishOaiExporterJob.isValid(doc)); + } + + @Test + void testGzip_simple() { + final String message = ""; + final byte[] bytes = IrishOaiExporterJob.gzip(message); + assertNotNull(bytes); + assertTrue(bytes.length > 0); + assertEquals(message, decompress(bytes)); + } + + @Test + void testGzip_empty() { + assertNull(IrishOaiExporterJob.gzip("")); + assertNull(IrishOaiExporterJob.gzip(null)); + } + + private static String decompress(final byte[] compressed) { + final StringBuilder outStr = new StringBuilder(); + if ((compressed == null) || (compressed.length == 0)) { return null; } + try { + if (isCompressed(compressed)) { + final GZIPInputStream gis = new GZIPInputStream(new ByteArrayInputStream(compressed)); + final BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(gis, "UTF-8")); + + String line; + while ((line = bufferedReader.readLine()) != null) { + outStr.append(line); + } + } else { + outStr.append(compressed); + } + return outStr.toString(); + } catch (final IOException e) { + throw new RuntimeException("error in gunzip", e); + } + } + + private static boolean isCompressed(final byte[] compressed) { + return (compressed[0] == (byte) GZIPInputStream.GZIP_MAGIC) && (compressed[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8)); + } +} diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/oaipmh/record_IE.xml b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/oaipmh/record_IE.xml new file mode 100644 index 000000000..01b7334f8 --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/oaipmh/record_IE.xml @@ -0,0 +1,89 @@ + + +

+ dedup_wf_002::532be02f990b479a1da46d71f1a4c3f0 + 2023-03-31T18:37:45.599Z + 2023-03-31T18:45:52.701Z +
+ + + + + + 50|od______6005::55a12e2e0fee45ce8005633c6c17fe9f + oai:repository.wit.ie:3029 + 50|od_______934::e7162a5632264cd622ee7180ca66fdce + oai:generic.eprints.org:3029 + 50|od_______934::55a12e2e0fee45ce8005633c6c17fe9f + + + + + + http://repository.wit.ie/3029/1/Research%20Day%202015%20-%20Poster%20Tadhg%20Blommerde.pdf + A service innovation capability maturity model for SMEs + + Blommerde, Tadhg + Lynch, Patrick + + 2015-04-28 + There is general consensus that service innovations are prerequisite to sustained competitive advantage and are an essential mechanism for responding to changes in customer needs and the operating environment of firms (Giannopoulou et al., 2011; Stryja et al., 2013). Services have been described as ubiquitous in their role of generating economic growth and wellbeing and represent over 70% of employment and GDP in developed nations (Janssen et al., 2012; Mustak, 2014). As a consequence, service innovations must be a core ambition of all countries, regions, and firms wishing to remain competitive (van Ark et al., 2003). While acknowledging the importance of once-off innovations, more critical still is the capability to repeatedly introduce and exploit service innovations (Siguaw et al., 2006). This is generally referred to as service innovation capability (SIC) and describes the repeatable routines and behaviours that organisations have in place to transform ideas and knowledge into innovations (Basterretxea and Martínez, 2012). However, despite links between SIC and continuous, sustainable, and consistent service innovations, there is evidence that many organisations struggle with its effective management (Adams et al., 2006; den Hertog et al., 2010). This is often attributed to the lack of formal guidance available and the absence of metrics to determine an organisation’s SIC performance (Hogan et al., 2011; Szczygielski, 2011). Maturity modelling research in this discipline remains at an embryonic stage, thus far presenting only conceptual and opaque discussions that fail to address the necessity for an assessment and strategic management framework (Gryszkiewicz et al., 2013; Hipp and Grupp, 2005). Therefore, the purpose of this ongoing research project is to evaluate the maturity of an organisation’s SIC to inform its effective management and enhancement. To achieve this it dimensionalises the concept into four constituent capabilities, specifically, strategising, customer involvement, knowledge management, and networking (Blommerde and Lynch, 2014). The study then tracks the maturity of these capabilities as they progress through eight evolutionary plateaus towards a fully developed or optimal state. This is accomplished through a capability maturity model that enables organisations to rapidly diagnose key areas of strength and weakness to systematically cultivate behaviours that leverage their untapped innovative potential (Wendler, 2012; Essmann and du Preez, 2010). As a result of the immense knowledge vacuum characteristic of this discipline, it is anticipated that this ongoing research project will make a substantial contribution to both academic understanding and take strides towards filling the void in practical support (Rapaccini et al., 2013). It expands the service innovation literature by detailing key service innovation levers, bolsters the discipline through clear definitions of terminology, provides a powerful explanation of the development of SICs, and operationalises the dynamic capabilities view through a novel self-assessment reference model (Jochem et al., 2011). The next step in the project is the evaluation of the, as yet, conceptual service innovation capability maturity model. Adopting a positivistic philosophical stance, the study proposes the use of structural equation modelling on data gathered through an extensive survey to confirm the model and support theoretical assumptions. + RIKON (Research in Inovation, Knowledge & Organisational Networks) + + application/pdf + + + false + false + true + + + + true + false + 0.8 + dedup-result-decisiontree-v4 + + + + + openorgs____::54cd984fc7d3b153ec2181f985041f02 + + WIT + South East Technological University + + + + + A service innovation capability maturity model for SMEs + 2015-04-28 + + + + A service innovation capability maturity model for SMEs + 2015-04-28 + + + + A service innovation capability maturity model for SMEs + 2015-04-28 + + + + + + + 2015-04-28 + + + http://repository.wit.ie/3029/1/Research%20Day%202015%20-%20Poster%20Tadhg%20Blommerde.pdf + + http://repository.wit.ie/3029/ + + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/oaipmh/record_IE_deleted.xml b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/oaipmh/record_IE_deleted.xml new file mode 100644 index 000000000..00d225aa5 --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/oaipmh/record_IE_deleted.xml @@ -0,0 +1,89 @@ + + +
+ dedup_wf_002::532be02f990b479a1da46d71f1a4c3f0 + 2023-03-31T18:37:45.599Z + 2023-03-31T18:45:52.701Z +
+ + + + + + 50|od______6005::55a12e2e0fee45ce8005633c6c17fe9f + oai:repository.wit.ie:3029 + 50|od_______934::e7162a5632264cd622ee7180ca66fdce + oai:generic.eprints.org:3029 + 50|od_______934::55a12e2e0fee45ce8005633c6c17fe9f + + + + + + http://repository.wit.ie/3029/1/Research%20Day%202015%20-%20Poster%20Tadhg%20Blommerde.pdf + A service innovation capability maturity model for SMEs + + Blommerde, Tadhg + Lynch, Patrick + + 2015-04-28 + There is general consensus that service innovations are prerequisite to sustained competitive advantage and are an essential mechanism for responding to changes in customer needs and the operating environment of firms (Giannopoulou et al., 2011; Stryja et al., 2013). Services have been described as ubiquitous in their role of generating economic growth and wellbeing and represent over 70% of employment and GDP in developed nations (Janssen et al., 2012; Mustak, 2014). As a consequence, service innovations must be a core ambition of all countries, regions, and firms wishing to remain competitive (van Ark et al., 2003). While acknowledging the importance of once-off innovations, more critical still is the capability to repeatedly introduce and exploit service innovations (Siguaw et al., 2006). This is generally referred to as service innovation capability (SIC) and describes the repeatable routines and behaviours that organisations have in place to transform ideas and knowledge into innovations (Basterretxea and Martínez, 2012). However, despite links between SIC and continuous, sustainable, and consistent service innovations, there is evidence that many organisations struggle with its effective management (Adams et al., 2006; den Hertog et al., 2010). This is often attributed to the lack of formal guidance available and the absence of metrics to determine an organisation’s SIC performance (Hogan et al., 2011; Szczygielski, 2011). Maturity modelling research in this discipline remains at an embryonic stage, thus far presenting only conceptual and opaque discussions that fail to address the necessity for an assessment and strategic management framework (Gryszkiewicz et al., 2013; Hipp and Grupp, 2005). Therefore, the purpose of this ongoing research project is to evaluate the maturity of an organisation’s SIC to inform its effective management and enhancement. To achieve this it dimensionalises the concept into four constituent capabilities, specifically, strategising, customer involvement, knowledge management, and networking (Blommerde and Lynch, 2014). The study then tracks the maturity of these capabilities as they progress through eight evolutionary plateaus towards a fully developed or optimal state. This is accomplished through a capability maturity model that enables organisations to rapidly diagnose key areas of strength and weakness to systematically cultivate behaviours that leverage their untapped innovative potential (Wendler, 2012; Essmann and du Preez, 2010). As a result of the immense knowledge vacuum characteristic of this discipline, it is anticipated that this ongoing research project will make a substantial contribution to both academic understanding and take strides towards filling the void in practical support (Rapaccini et al., 2013). It expands the service innovation literature by detailing key service innovation levers, bolsters the discipline through clear definitions of terminology, provides a powerful explanation of the development of SICs, and operationalises the dynamic capabilities view through a novel self-assessment reference model (Jochem et al., 2011). The next step in the project is the evaluation of the, as yet, conceptual service innovation capability maturity model. Adopting a positivistic philosophical stance, the study proposes the use of structural equation modelling on data gathered through an extensive survey to confirm the model and support theoretical assumptions. + RIKON (Research in Inovation, Knowledge & Organisational Networks) + + application/pdf + + + false + false + true + + + + true + true + 0.8 + dedup-result-decisiontree-v4 + + + + + openorgs____::54cd984fc7d3b153ec2181f985041f02 + + WIT + South East Technological University + + + + + A service innovation capability maturity model for SMEs + 2015-04-28 + + + + A service innovation capability maturity model for SMEs + 2015-04-28 + + + + A service innovation capability maturity model for SMEs + 2015-04-28 + + + + + + + 2015-04-28 + + + http://repository.wit.ie/3029/1/Research%20Day%202015%20-%20Poster%20Tadhg%20Blommerde.pdf + + http://repository.wit.ie/3029/ + + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/oaipmh/record_IT.xml b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/oaipmh/record_IT.xml new file mode 100644 index 000000000..7649589d1 --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/oaipmh/record_IT.xml @@ -0,0 +1,66 @@ + + +
+ od_______310::02365c51a0ed7cbb54b2bbc7c0426d1b + 2024-04-06T06:05:16+0000 + 2024-04-06T06:56:01.776Z +
+ + + + + 50|od_______310::02365c51a0ed7cbb54b2bbc7c0426d1b + oai:flore.unifi.it:2158/608965 + 2158/608965 + + + + + + Estorsione (art. 629) + + MACRI', FRANCESCO + + 2011-01-01 + + 2011-01-01 + 2011-01-01 + 2015-04-28 + UTET + + + + false + false + 0.9 + null + + + + + openorgs____::41406edad82942e9e0b29317b8a847e2 + University of Florence + + University of Florence + + + + + + + + 2011-01-01 + + 2158/608965 + http://hdl.handle.net/2158/608965 + + + https://hdl.handle.net/2158/608965 + + + + + + + + \ No newline at end of file From 711048ceedc99383c291bc532373e09294fe0815 Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Tue, 7 May 2024 15:44:33 +0200 Subject: [PATCH 46/97] PrepareRelationsJob rewritten to use Spark Dataframe API and Windowing functions --- .../dhp/oa/provision/PrepareRelationsJob.java | 190 ++++-------------- 1 file changed, 38 insertions(+), 152 deletions(-) diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/PrepareRelationsJob.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/PrepareRelationsJob.java index fdf397ad7..c2eb8c408 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/PrepareRelationsJob.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/PrepareRelationsJob.java @@ -1,43 +1,31 @@ package eu.dnetlib.dhp.oa.provision; -import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; - -import java.util.HashSet; -import java.util.Optional; -import java.util.PriorityQueue; -import java.util.Set; -import java.util.stream.Collectors; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.spark.SparkConf; -import org.apache.spark.api.java.JavaRDD; -import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.api.java.function.FilterFunction; -import org.apache.spark.api.java.function.FlatMapFunction; -import org.apache.spark.api.java.function.Function; -import org.apache.spark.api.java.function.MapFunction; -import org.apache.spark.sql.Encoder; -import org.apache.spark.sql.Encoders; -import org.apache.spark.sql.SaveMode; -import org.apache.spark.sql.SparkSession; -import org.apache.spark.sql.expressions.Aggregator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Joiner; import com.google.common.base.Splitter; -import com.google.common.collect.Iterables; import com.google.common.collect.Sets; - import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.common.HdfsSupport; import eu.dnetlib.dhp.oa.provision.model.ProvisionModelSupport; -import eu.dnetlib.dhp.oa.provision.model.SortableRelationKey; -import eu.dnetlib.dhp.oa.provision.utils.RelationPartitioner; import eu.dnetlib.dhp.schema.oaf.Relation; -import scala.Tuple2; +import org.apache.commons.io.IOUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SaveMode; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.expressions.Window; +import org.apache.spark.sql.expressions.WindowSpec; +import org.apache.spark.sql.functions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; +import static org.apache.spark.sql.functions.col; /** * PrepareRelationsJob prunes the relationships: only consider relationships that are not virtually deleted @@ -130,130 +118,28 @@ public class PrepareRelationsJob { private static void prepareRelationsRDD(SparkSession spark, String inputRelationsPath, String outputPath, Set relationFilter, int sourceMaxRelations, int targetMaxRelations, int relPartitions) { - JavaRDD rels = readPathRelationRDD(spark, inputRelationsPath) - .filter(rel -> !(rel.getSource().startsWith("unresolved") || rel.getTarget().startsWith("unresolved"))) - .filter(rel -> !rel.getDataInfo().getDeletedbyinference()) - .filter(rel -> !relationFilter.contains(StringUtils.lowerCase(rel.getRelClass()))); + WindowSpec source_w = Window + .partitionBy("source", "subRelType") + .orderBy(col("target").desc_nulls_last()); - JavaRDD pruned = pruneRels( - pruneRels( - rels, - sourceMaxRelations, relPartitions, (Function) Relation::getSource), - targetMaxRelations, relPartitions, (Function) Relation::getTarget); - spark - .createDataset(pruned.rdd(), Encoders.bean(Relation.class)) - .repartition(relPartitions) - .write() - .mode(SaveMode.Overwrite) - .parquet(outputPath); - } + WindowSpec target_w = Window + .partitionBy("target", "subRelType") + .orderBy(col("source").desc_nulls_last()); - private static JavaRDD pruneRels(JavaRDD rels, int maxRelations, - int relPartitions, Function idFn) { - return rels - .mapToPair(r -> new Tuple2<>(SortableRelationKey.create(r, idFn.call(r)), r)) - .repartitionAndSortWithinPartitions(new RelationPartitioner(relPartitions)) - .groupBy(Tuple2::_1) - .map(Tuple2::_2) - .map(t -> Iterables.limit(t, maxRelations)) - .flatMap(Iterable::iterator) - .map(Tuple2::_2); - } - - // experimental - private static void prepareRelationsDataset( - SparkSession spark, String inputRelationsPath, String outputPath, Set relationFilter, int maxRelations, - int relPartitions) { - spark - .read() - .textFile(inputRelationsPath) - .repartition(relPartitions) - .map( - (MapFunction) s -> OBJECT_MAPPER.readValue(s, Relation.class), - Encoders.kryo(Relation.class)) - .filter((FilterFunction) rel -> !rel.getDataInfo().getDeletedbyinference()) - .filter((FilterFunction) rel -> !relationFilter.contains(rel.getRelClass())) - .groupByKey( - (MapFunction) Relation::getSource, - Encoders.STRING()) - .agg(new RelationAggregator(maxRelations).toColumn()) - .flatMap( - (FlatMapFunction, Relation>) t -> Iterables - .limit(t._2().getRelations(), maxRelations) - .iterator(), - Encoders.bean(Relation.class)) - .repartition(relPartitions) - .write() - .mode(SaveMode.Overwrite) - .parquet(outputPath); - } - - public static class RelationAggregator - extends Aggregator { - - private final int maxRelations; - - public RelationAggregator(int maxRelations) { - this.maxRelations = maxRelations; - } - - @Override - public RelationList zero() { - return new RelationList(); - } - - @Override - public RelationList reduce(RelationList b, Relation a) { - b.getRelations().add(a); - return getSortableRelationList(b); - } - - @Override - public RelationList merge(RelationList b1, RelationList b2) { - b1.getRelations().addAll(b2.getRelations()); - return getSortableRelationList(b1); - } - - @Override - public RelationList finish(RelationList r) { - return getSortableRelationList(r); - } - - private RelationList getSortableRelationList(RelationList b1) { - RelationList sr = new RelationList(); - sr - .setRelations( - b1 - .getRelations() - .stream() - .limit(maxRelations) - .collect(Collectors.toCollection(() -> new PriorityQueue<>(new RelationComparator())))); - return sr; - } - - @Override - public Encoder bufferEncoder() { - return Encoders.kryo(RelationList.class); - } - - @Override - public Encoder outputEncoder() { - return Encoders.kryo(RelationList.class); - } - } - - /** - * Reads a JavaRDD of eu.dnetlib.dhp.oa.provision.model.SortableRelation objects from a newline delimited json text - * file, - * - * @param spark - * @param inputPath - * @return the JavaRDD containing all the relationships - */ - private static JavaRDD readPathRelationRDD( - SparkSession spark, final String inputPath) { - JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); - return sc.textFile(inputPath).map(s -> OBJECT_MAPPER.readValue(s, Relation.class)); + spark.read().schema(Encoders.bean(Relation.class).schema()).json(inputRelationsPath) + .where("source NOT LIKE 'unresolved%' AND target NOT LIKE 'unresolved%'") + .where("datainfo.deletedbyinference != true") + .where(relationFilter.isEmpty() ? "" : "lower(relClass) NOT IN ("+ Joiner.on(',').join(relationFilter) +")") + .withColumn("source_w_pos", functions.row_number().over(source_w)) + .where("source_w_pos < " + sourceMaxRelations ) + .drop("source_w_pos") + .withColumn("target_w_pos", functions.row_number().over(target_w)) + .where("target_w_pos < " + targetMaxRelations) + .drop( "target_w_pos") + .coalesce(relPartitions) + .write() + .mode(SaveMode.Overwrite) + .parquet(outputPath); } private static void removeOutputDir(SparkSession spark, String path) { From b4e33894322d1693460be2cfcf0afb23d3b9135f Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 7 May 2024 16:25:17 +0200 Subject: [PATCH 47/97] fixed property mapping creating the RelatedEntity transient objects. spark cores & memory adjustments. Code formatting --- .../CreateRelatedEntitiesJob_phase1.java | 9 ++- .../dhp/oa/provision/PrepareRelationsJob.java | 72 +++++++++++-------- .../dhp/oa/provision/oozie_app/workflow.xml | 10 +-- 3 files changed, 54 insertions(+), 37 deletions(-) diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/CreateRelatedEntitiesJob_phase1.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/CreateRelatedEntitiesJob_phase1.java index da80deee0..63f3c2ead 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/CreateRelatedEntitiesJob_phase1.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/CreateRelatedEntitiesJob_phase1.java @@ -153,10 +153,15 @@ public class CreateRelatedEntitiesJob_phase1 { result .getTitle() .stream() + .filter(t -> StringUtils.isNotBlank(t.getValue())) .findFirst() - .map(StructuredProperty::getValue) .ifPresent( - title -> re.getTitle().setValue(StringUtils.left(title, ModelHardLimits.MAX_TITLE_LENGTH))); + title -> { + re.setTitle(title); + re + .getTitle() + .setValue(StringUtils.left(title.getValue(), ModelHardLimits.MAX_TITLE_LENGTH)); + }); } if (Objects.nonNull(result.getDescription()) && !result.getDescription().isEmpty()) { result diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/PrepareRelationsJob.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/PrepareRelationsJob.java index c2eb8c408..f50c7774b 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/PrepareRelationsJob.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/PrepareRelationsJob.java @@ -1,14 +1,15 @@ package eu.dnetlib.dhp.oa.provision; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Joiner; -import com.google.common.base.Splitter; -import com.google.common.collect.Sets; -import eu.dnetlib.dhp.application.ArgumentApplicationParser; -import eu.dnetlib.dhp.common.HdfsSupport; -import eu.dnetlib.dhp.oa.provision.model.ProvisionModelSupport; -import eu.dnetlib.dhp.schema.oaf.Relation; +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; +import static org.apache.spark.sql.functions.col; + +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + import org.apache.commons.io.IOUtils; import org.apache.spark.SparkConf; import org.apache.spark.sql.Encoders; @@ -20,12 +21,15 @@ import org.apache.spark.sql.functions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.HashSet; -import java.util.Optional; -import java.util.Set; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.collect.Sets; -import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; -import static org.apache.spark.sql.functions.col; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.HdfsSupport; +import eu.dnetlib.dhp.oa.provision.model.ProvisionModelSupport; +import eu.dnetlib.dhp.schema.oaf.Relation; /** * PrepareRelationsJob prunes the relationships: only consider relationships that are not virtually deleted @@ -119,27 +123,33 @@ public class PrepareRelationsJob { Set relationFilter, int sourceMaxRelations, int targetMaxRelations, int relPartitions) { WindowSpec source_w = Window - .partitionBy("source", "subRelType") - .orderBy(col("target").desc_nulls_last()); + .partitionBy("source", "subRelType") + .orderBy(col("target").desc_nulls_last()); WindowSpec target_w = Window - .partitionBy("target", "subRelType") - .orderBy(col("source").desc_nulls_last()); + .partitionBy("target", "subRelType") + .orderBy(col("source").desc_nulls_last()); - spark.read().schema(Encoders.bean(Relation.class).schema()).json(inputRelationsPath) - .where("source NOT LIKE 'unresolved%' AND target NOT LIKE 'unresolved%'") - .where("datainfo.deletedbyinference != true") - .where(relationFilter.isEmpty() ? "" : "lower(relClass) NOT IN ("+ Joiner.on(',').join(relationFilter) +")") - .withColumn("source_w_pos", functions.row_number().over(source_w)) - .where("source_w_pos < " + sourceMaxRelations ) - .drop("source_w_pos") - .withColumn("target_w_pos", functions.row_number().over(target_w)) - .where("target_w_pos < " + targetMaxRelations) - .drop( "target_w_pos") - .coalesce(relPartitions) - .write() - .mode(SaveMode.Overwrite) - .parquet(outputPath); + spark + .read() + .schema(Encoders.bean(Relation.class).schema()) + .json(inputRelationsPath) + .where("source NOT LIKE 'unresolved%' AND target NOT LIKE 'unresolved%'") + .where("datainfo.deletedbyinference != true") + .where( + relationFilter.isEmpty() ? "" + : "lower(relClass) NOT IN (" + + relationFilter.stream().map(s -> "'" + s + "'").collect(Collectors.joining(",")) + ")") + .withColumn("source_w_pos", functions.row_number().over(source_w)) + .where("source_w_pos < " + sourceMaxRelations) + .drop("source_w_pos") + .withColumn("target_w_pos", functions.row_number().over(target_w)) + .where("target_w_pos < " + targetMaxRelations) + .drop("target_w_pos") + .coalesce(relPartitions) + .write() + .mode(SaveMode.Overwrite) + .parquet(outputPath); } private static void removeOutputDir(SparkSession spark, String path) { diff --git a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml index eb446ddd8..434b4c9af 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml @@ -144,21 +144,23 @@ eu.dnetlib.dhp.oa.provision.PrepareRelationsJob dhp-graph-provision-${projectVersion}.jar - --executor-cores=${sparkExecutorCoresForJoining} - --executor-memory=${sparkExecutorMemoryForJoining} + --executor-cores=4 + --executor-memory=6G --driver-memory=${sparkDriverMemoryForJoining} + --conf spark.executor.memoryOverhead=6G --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.sql.shuffle.partitions=3840 + --conf spark.sql.shuffle.partitions=15000 + --conf spark.network.timeout=${sparkNetworkTimeout} --inputRelationsPath${inputGraphRootPath}/relation --outputPath${workingDir}/relation --sourceMaxRelations${sourceMaxRelations} --targetMaxRelations${targetMaxRelations} --relationFilter${relationFilter} - --relPartitions5000 + --relPartitions15000 From e234848af8b0a313a0c8b3988d2ceb4f425edc78 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Wed, 8 May 2024 10:00:53 +0200 Subject: [PATCH 48/97] oaf record: xpath for root --- .../main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java index e2ae890e5..fff5d015d 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java @@ -105,7 +105,7 @@ public class IrishOaiExporterJob { if (isValid(doc)) { r.setId(doc.valueOf("//*[local-name()='objIdentifier']").trim()); - r.setBody(gzip(xml)); + r.setBody(gzip(doc.selectSingleNode("//*[local-name()='entity']").asXML())); r.setDate(LocalDateTime.now()); r.setSets(new ArrayList<>()); } From c9a327bc5094f48c08f4d7d0b3274378d8d8c63f Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Wed, 8 May 2024 11:34:08 +0200 Subject: [PATCH 49/97] refactoring of gzip method --- .../dhp/oa/oaipmh/IrishOaiExporterJob.java | 14 +++++------ .../oa/oaipmh/IrishOaiExporterJobTest.java | 24 +++++-------------- 2 files changed, 13 insertions(+), 25 deletions(-) diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java index fff5d015d..b59f0ae73 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java @@ -4,6 +4,7 @@ import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.nio.charset.Charset; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Optional; @@ -145,15 +146,14 @@ public class IrishOaiExporterJob { protected static byte[] gzip(final String str) { if (StringUtils.isBlank(str)) { return null; } - try { - final ByteArrayOutputStream obj = new ByteArrayOutputStream(); - final GZIPOutputStream gzip = new GZIPOutputStream(obj); - gzip.write(str.getBytes("UTF-8")); - gzip.flush(); - gzip.close(); - return obj.toByteArray(); + try (final ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + try (final GZIPOutputStream gzip = new GZIPOutputStream(baos)) { + IOUtils.write(str.getBytes(Charset.defaultCharset()), gzip); + } + return baos.toByteArray(); } catch (final IOException e) { throw new RuntimeException("error in gzip", e); } } + } diff --git a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJobTest.java b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJobTest.java index 6140b0907..e33c701c5 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJobTest.java +++ b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJobTest.java @@ -6,10 +6,9 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.IOException; -import java.io.InputStreamReader; +import java.nio.charset.Charset; import java.util.zip.GZIPInputStream; import org.apache.commons.io.IOUtils; @@ -57,7 +56,7 @@ class IrishOaiExporterJobTest { final byte[] bytes = IrishOaiExporterJob.gzip(message); assertNotNull(bytes); assertTrue(bytes.length > 0); - assertEquals(message, decompress(bytes)); + assertEquals(message, gunzip(bytes)); } @Test @@ -66,22 +65,11 @@ class IrishOaiExporterJobTest { assertNull(IrishOaiExporterJob.gzip(null)); } - private static String decompress(final byte[] compressed) { - final StringBuilder outStr = new StringBuilder(); + public static String gunzip(final byte[] compressed) { if ((compressed == null) || (compressed.length == 0)) { return null; } - try { - if (isCompressed(compressed)) { - final GZIPInputStream gis = new GZIPInputStream(new ByteArrayInputStream(compressed)); - final BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(gis, "UTF-8")); - - String line; - while ((line = bufferedReader.readLine()) != null) { - outStr.append(line); - } - } else { - outStr.append(compressed); - } - return outStr.toString(); + if (!isCompressed(compressed)) { return new String(compressed); } + try (final GZIPInputStream gis = new GZIPInputStream(new ByteArrayInputStream(compressed))) { + return IOUtils.toString(gis, Charset.defaultCharset()); } catch (final IOException e) { throw new RuntimeException("error in gunzip", e); } From 18aa323ee972c8b0565273ada553892f0568f83e Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Wed, 8 May 2024 11:36:46 +0200 Subject: [PATCH 50/97] cleanup unused classes, adjustments in the oozie wf definition --- .../dhp/oa/provision/RelationComparator.java | 44 ---------- .../dhp/oa/provision/RelationList.java | 25 ------ .../dhp/oa/provision/SortableRelation.java | 81 ------------------- .../model/ProvisionModelSupport.java | 10 +-- .../dhp/oa/provision/oozie_app/workflow.xml | 11 +-- 5 files changed, 7 insertions(+), 164 deletions(-) delete mode 100644 dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/RelationComparator.java delete mode 100644 dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/RelationList.java delete mode 100644 dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/SortableRelation.java diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/RelationComparator.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/RelationComparator.java deleted file mode 100644 index e13bc60eb..000000000 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/RelationComparator.java +++ /dev/null @@ -1,44 +0,0 @@ - -package eu.dnetlib.dhp.oa.provision; - -import java.util.Comparator; -import java.util.Map; -import java.util.Optional; - -import com.google.common.collect.ComparisonChain; -import com.google.common.collect.Maps; - -import eu.dnetlib.dhp.schema.common.ModelConstants; -import eu.dnetlib.dhp.schema.oaf.Relation; - -public class RelationComparator implements Comparator { - - private static final Map weights = Maps.newHashMap(); - - static { - weights.put(ModelConstants.OUTCOME, 0); - weights.put(ModelConstants.SUPPLEMENT, 1); - weights.put(ModelConstants.REVIEW, 2); - weights.put(ModelConstants.CITATION, 3); - weights.put(ModelConstants.AFFILIATION, 4); - weights.put(ModelConstants.RELATIONSHIP, 5); - weights.put(ModelConstants.PUBLICATION_DATASET, 6); - weights.put(ModelConstants.SIMILARITY, 7); - - weights.put(ModelConstants.PROVISION, 8); - weights.put(ModelConstants.PARTICIPATION, 9); - weights.put(ModelConstants.DEDUP, 10); - } - - private Integer getWeight(Relation o) { - return Optional.ofNullable(weights.get(o.getSubRelType())).orElse(Integer.MAX_VALUE); - } - - @Override - public int compare(Relation o1, Relation o2) { - return ComparisonChain - .start() - .compare(getWeight(o1), getWeight(o2)) - .result(); - } -} diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/RelationList.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/RelationList.java deleted file mode 100644 index 6e5fd7dba..000000000 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/RelationList.java +++ /dev/null @@ -1,25 +0,0 @@ - -package eu.dnetlib.dhp.oa.provision; - -import java.io.Serializable; -import java.util.PriorityQueue; -import java.util.Queue; - -import eu.dnetlib.dhp.schema.oaf.Relation; - -public class RelationList implements Serializable { - - private Queue relations; - - public RelationList() { - this.relations = new PriorityQueue<>(new RelationComparator()); - } - - public Queue getRelations() { - return relations; - } - - public void setRelations(Queue relations) { - this.relations = relations; - } -} diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/SortableRelation.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/SortableRelation.java deleted file mode 100644 index 8740b47fc..000000000 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/SortableRelation.java +++ /dev/null @@ -1,81 +0,0 @@ - -package eu.dnetlib.dhp.oa.provision; - -import java.io.Serializable; -import java.util.Map; -import java.util.Optional; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.google.common.collect.ComparisonChain; -import com.google.common.collect.Maps; - -import eu.dnetlib.dhp.schema.common.ModelConstants; -import eu.dnetlib.dhp.schema.oaf.Relation; - -public class SortableRelation extends Relation implements Comparable, Serializable { - - private static final Map weights = Maps.newHashMap(); - - static { - weights.put(ModelConstants.OUTCOME, 0); - weights.put(ModelConstants.SUPPLEMENT, 1); - weights.put(ModelConstants.REVIEW, 2); - weights.put(ModelConstants.CITATION, 3); - weights.put(ModelConstants.AFFILIATION, 4); - weights.put(ModelConstants.RELATIONSHIP, 5); - weights.put(ModelConstants.PUBLICATION_RESULTTYPE_CLASSID, 6); - weights.put(ModelConstants.SIMILARITY, 7); - - weights.put(ModelConstants.PROVISION, 8); - weights.put(ModelConstants.PARTICIPATION, 9); - weights.put(ModelConstants.DEDUP, 10); - } - - private static final long serialVersionUID = 34753984579L; - - private String groupingKey; - - public static SortableRelation create(Relation r, String groupingKey) { - SortableRelation sr = new SortableRelation(); - sr.setGroupingKey(groupingKey); - sr.setSource(r.getSource()); - sr.setTarget(r.getTarget()); - sr.setRelType(r.getRelType()); - sr.setSubRelType(r.getSubRelType()); - sr.setRelClass(r.getRelClass()); - sr.setDataInfo(r.getDataInfo()); - sr.setCollectedfrom(r.getCollectedfrom()); - sr.setLastupdatetimestamp(r.getLastupdatetimestamp()); - sr.setProperties(r.getProperties()); - sr.setValidated(r.getValidated()); - sr.setValidationDate(r.getValidationDate()); - - return sr; - } - - @JsonIgnore - public Relation asRelation() { - return this; - } - - @Override - public int compareTo(SortableRelation o) { - return ComparisonChain - .start() - .compare(getGroupingKey(), o.getGroupingKey()) - .compare(getWeight(this), getWeight(o)) - .result(); - } - - private Integer getWeight(SortableRelation o) { - return Optional.ofNullable(weights.get(o.getSubRelType())).orElse(Integer.MAX_VALUE); - } - - public String getGroupingKey() { - return groupingKey; - } - - public void setGroupingKey(String groupingKey) { - this.groupingKey = groupingKey; - } -} diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/ProvisionModelSupport.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/ProvisionModelSupport.java index 0e6e95de5..10a99704c 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/ProvisionModelSupport.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/ProvisionModelSupport.java @@ -1,8 +1,6 @@ package eu.dnetlib.dhp.oa.provision.model; -import static org.apache.commons.lang3.StringUtils.substringBefore; - import java.io.StringReader; import java.util.*; import java.util.stream.Collectors; @@ -16,12 +14,9 @@ import org.jetbrains.annotations.Nullable; import com.google.common.base.Splitter; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.collect.Sets; import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup; import eu.dnetlib.dhp.common.vocabulary.VocabularyTerm; -import eu.dnetlib.dhp.oa.provision.RelationList; -import eu.dnetlib.dhp.oa.provision.SortableRelation; import eu.dnetlib.dhp.oa.provision.utils.ContextDef; import eu.dnetlib.dhp.oa.provision.utils.ContextMapper; import eu.dnetlib.dhp.schema.common.ModelSupport; @@ -55,10 +50,7 @@ public class ProvisionModelSupport { .newArrayList( RelatedEntityWrapper.class, JoinedEntity.class, - RelatedEntity.class, - SortableRelationKey.class, - SortableRelation.class, - RelationList.class)); + RelatedEntity.class)); return modelClasses.toArray(new Class[] {}); } diff --git a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml index 434b4c9af..1fc28e7ca 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml @@ -125,7 +125,7 @@ ${wf:conf('resumeFrom') eq 'prepare_relations'} ${wf:conf('resumeFrom') eq 'fork_join_related_entities'} ${wf:conf('resumeFrom') eq 'fork_join_all_entities'} - ${wf:conf('resumeFrom') eq 'convert_to_xml'} + ${wf:conf('resumeFrom') eq 'create_payloads'} ${wf:conf('resumeFrom') eq 'drop_solr_collection'} ${wf:conf('resumeFrom') eq 'to_solr_index'} @@ -587,19 +587,20 @@ - + - + yarn cluster - convert_to_xml + create_payloads eu.dnetlib.dhp.oa.provision.XmlConverterJob dhp-graph-provision-${projectVersion}.jar --executor-cores=${sparkExecutorCores} --executor-memory=${sparkExecutorMemory} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -607,7 +608,7 @@ --conf spark.sql.shuffle.partitions=3840 --conf spark.network.timeout=${sparkNetworkTimeout} - --inputPath${workingDir}/join_entities + --inputPath/user/claudio.atzori/data/beta_provision/join_entities --outputPath${workingDir}/xml_json --contextApiBaseUrl${contextApiBaseUrl} --isLookupUrl${isLookupUrl} From 39a2afe8b538c45b1e4d20ed31d3eee1c9dbdd7b Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Thu, 9 May 2024 13:54:42 +0200 Subject: [PATCH 51/97] [graph provision] fixed XML serialization of the usage counts measures, renamed workflow actions to better reflect their role --- ...erterJob.java => PayloadConverterJob.java} | 16 +-- .../model/ProvisionModelSupport.java | 11 +- .../oa/provision/utils/XmlRecordFactory.java | 110 ++++++++++-------- .../utils/XmlSerializationUtils.java | 33 ++++++ ...on => input_params_payload_converter.json} | 0 .../dhp/oa/provision/oozie_app/workflow.xml | 2 +- .../dhp/oa/provision/EOSCFuture_Test.java | 2 +- .../provision/IndexRecordTransformerTest.java | 6 +- .../oa/provision/XmlRecordFactoryTest.java | 14 +-- 9 files changed, 120 insertions(+), 74 deletions(-) rename dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/{XmlConverterJob.java => PayloadConverterJob.java} (92%) rename dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/{input_params_xml_converter.json => input_params_payload_converter.json} (100%) diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/XmlConverterJob.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/PayloadConverterJob.java similarity index 92% rename from dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/XmlConverterJob.java rename to dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/PayloadConverterJob.java index 4353e863f..f34caad75 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/XmlConverterJob.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/PayloadConverterJob.java @@ -3,24 +3,16 @@ package eu.dnetlib.dhp.oa.provision; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; import static eu.dnetlib.dhp.utils.DHPUtils.toSeq; -import static org.apache.spark.sql.functions.*; import java.util.List; import java.util.Map; import java.util.Optional; import org.apache.commons.io.IOUtils; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.compress.GzipCodec; -import org.apache.hadoop.mapred.SequenceFileOutputFormat; import org.apache.spark.SparkConf; import org.apache.spark.SparkContext; -import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.function.MapFunction; -import org.apache.spark.api.java.function.PairFunction; import org.apache.spark.sql.*; -import org.apache.spark.sql.expressions.UserDefinedFunction; -import org.apache.spark.sql.types.DataTypes; import org.apache.spark.util.LongAccumulator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,9 +37,9 @@ import scala.Tuple2; /** * XmlConverterJob converts the JoinedEntities as XML records */ -public class XmlConverterJob { +public class PayloadConverterJob { - private static final Logger log = LoggerFactory.getLogger(XmlConverterJob.class); + private static final Logger log = LoggerFactory.getLogger(PayloadConverterJob.class); public static final String schemaLocation = "https://www.openaire.eu/schema/1.0/oaf-1.0.xsd"; @@ -56,8 +48,8 @@ public class XmlConverterJob { final ArgumentApplicationParser parser = new ArgumentApplicationParser( IOUtils .toString( - XmlConverterJob.class - .getResourceAsStream("/eu/dnetlib/dhp/oa/provision/input_params_xml_converter.json"))); + PayloadConverterJob.class + .getResourceAsStream("/eu/dnetlib/dhp/oa/provision/input_params_payload_converter.json"))); parser.parseArgument(args); final Boolean isSparkSessionManaged = Optional diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/ProvisionModelSupport.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/ProvisionModelSupport.java index 10a99704c..a085a72e0 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/ProvisionModelSupport.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/ProvisionModelSupport.java @@ -19,8 +19,10 @@ import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup; import eu.dnetlib.dhp.common.vocabulary.VocabularyTerm; import eu.dnetlib.dhp.oa.provision.utils.ContextDef; import eu.dnetlib.dhp.oa.provision.utils.ContextMapper; +import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.oaf.*; +import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; import eu.dnetlib.dhp.schema.solr.*; import eu.dnetlib.dhp.schema.solr.AccessRight; import eu.dnetlib.dhp.schema.solr.Author; @@ -66,7 +68,11 @@ public class ProvisionModelSupport { .setHeader( SolrRecordHeader .newInstance( - e.getId(), e.getOriginalId(), type, deletedbyinference)); + StringUtils + .substringAfter( + e.getId(), + IdentifierFactory.ID_PREFIX_SEPARATOR), + e.getOriginalId(), type, deletedbyinference)); r.setCollectedfrom(asProvenance(e.getCollectedfrom())); r.setContext(asContext(e.getContext(), contextMapper)); r.setPid(asPid(e.getPid())); @@ -106,7 +112,8 @@ public class ProvisionModelSupport { .newInstance( relation.getRelType(), relation.getRelClass(), - relation.getTarget(), relatedRecordType)); + StringUtils.substringAfter(relation.getTarget(), IdentifierFactory.ID_PREFIX_SEPARATOR), + relatedRecordType)); rr.setAcronym(re.getAcronym()); rr.setCode(re.getCode()); diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java index 63597c61e..65fa122c8 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java @@ -1,25 +1,23 @@ package eu.dnetlib.dhp.oa.provision.utils; -import static eu.dnetlib.dhp.oa.provision.utils.GraphMappingUtils.authorPidTypes; -import static eu.dnetlib.dhp.oa.provision.utils.GraphMappingUtils.getRelDescriptor; -import static org.apache.commons.lang3.StringUtils.isNotBlank; -import static org.apache.commons.lang3.StringUtils.substringBefore; - -import java.io.IOException; -import java.io.Serializable; -import java.io.StringReader; -import java.io.StringWriter; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import javax.xml.transform.*; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; - +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.mycila.xmltool.XMLDoc; +import com.mycila.xmltool.XMLTag; +import eu.dnetlib.dhp.oa.provision.model.JoinedEntity; +import eu.dnetlib.dhp.oa.provision.model.RelatedEntity; +import eu.dnetlib.dhp.oa.provision.model.RelatedEntityWrapper; +import eu.dnetlib.dhp.oa.provision.model.XmlInstance; +import eu.dnetlib.dhp.schema.common.*; +import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.dhp.schema.oaf.*; +import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; +import eu.dnetlib.dhp.schema.oaf.utils.ModelHardLimits; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; @@ -31,27 +29,26 @@ import org.dom4j.Node; import org.dom4j.io.OutputFormat; import org.dom4j.io.SAXReader; import org.dom4j.io.XMLWriter; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Joiner; -import com.google.common.base.Splitter; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import com.mycila.xmltool.XMLDoc; -import com.mycila.xmltool.XMLTag; - -import eu.dnetlib.dhp.oa.provision.model.JoinedEntity; -import eu.dnetlib.dhp.oa.provision.model.RelatedEntity; -import eu.dnetlib.dhp.oa.provision.model.RelatedEntityWrapper; -import eu.dnetlib.dhp.oa.provision.model.XmlInstance; -import eu.dnetlib.dhp.schema.common.*; -import eu.dnetlib.dhp.schema.oaf.*; -import eu.dnetlib.dhp.schema.oaf.Result; -import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; -import eu.dnetlib.dhp.schema.oaf.utils.ModelHardLimits; import scala.Tuple2; +import javax.xml.transform.*; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import java.io.IOException; +import java.io.Serializable; +import java.io.StringReader; +import java.io.StringWriter; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static eu.dnetlib.dhp.oa.provision.utils.GraphMappingUtils.authorPidTypes; +import static eu.dnetlib.dhp.oa.provision.utils.GraphMappingUtils.getRelDescriptor; +import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.apache.commons.lang3.StringUtils.substringBefore; + public class XmlRecordFactory implements Serializable { /** @@ -93,10 +90,13 @@ public class XmlRecordFactory implements Serializable { } public String build(final JoinedEntity je) { + return build(je, false); + } + + public String build(final JoinedEntity je, final Boolean validate) { final Set contexts = Sets.newHashSet(); - // final OafEntity entity = toOafEntity(je.getEntity()); final OafEntity entity = je.getEntity(); final TemplateFactory templateFactory = new TemplateFactory(); try { @@ -122,8 +122,14 @@ public class XmlRecordFactory implements Serializable { .buildBody( mainType, metadata, relations, listChildren(entity, je, templateFactory), listExtraInfo(entity)); - return templateFactory.buildRecord(entity, schemaLocation, body); - // return printXML(templateFactory.buildRecord(entity, schemaLocation, body), indent); + String xmlRecord = templateFactory.buildRecord(entity, schemaLocation, body); + + if (Boolean.TRUE.equals(validate)) { + // rise an exception when an invalid record was built + new SAXReader().read(new StringReader(xmlRecord)); + } + return xmlRecord; + // return printXML(templateFactory.buildRecord(entity, schemaLocation, body), indent); } catch (final Throwable e) { throw new RuntimeException(String.format("error building record '%s'", entity.getId()), e); } @@ -1038,13 +1044,21 @@ public class XmlRecordFactory implements Serializable { } private List measuresAsXml(List measures) { - return measures - .stream() - .map(m -> { - List> l = Lists.newArrayList(new Tuple2<>("id", m.getId())); - m.getUnit().forEach(kv -> l.add(new Tuple2<>(kv.getKey(), kv.getValue()))); - return XmlSerializationUtils.asXmlElement("measure", l); - }) + return Stream + .concat( + measures + .stream() + .filter(m -> !"downloads".equals(m.getId()) && !"views".equals(m.getId())) + .map(m -> { + List> l = Lists.newArrayList(new Tuple2<>("id", m.getId())); + m.getUnit().forEach(kv -> l.add(new Tuple2<>(kv.getKey(), kv.getValue()))); + return XmlSerializationUtils.asXmlElement("measure", l); + }), + measures + .stream() + .filter(m -> "downloads".equals(m.getId()) || "views".equals(m.getId())) + .filter(m -> m.getUnit().stream().anyMatch(u -> Integer.parseInt(u.getValue()) > 0)) + .map(m -> XmlSerializationUtils.usageMeasureAsXmlElement("measure", m))) .collect(Collectors.toList()); } diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlSerializationUtils.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlSerializationUtils.java index deacac3ad..31763ace3 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlSerializationUtils.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlSerializationUtils.java @@ -5,7 +5,11 @@ import static eu.dnetlib.dhp.oa.provision.utils.GraphMappingUtils.removePrefix; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; +import java.util.HashSet; import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; @@ -166,6 +170,35 @@ public class XmlSerializationUtils { return sb.toString(); } + // infrastruct_::f66f1bd369679b5b077dcdf006089556||OpenAIRE + public static String usageMeasureAsXmlElement(String name, Measure measure) { + HashSet dsIds = Optional + .ofNullable(measure.getUnit()) + .map( + m -> m + .stream() + .map(KeyValue::getKey) + .collect(Collectors.toCollection(HashSet::new))) + .orElse(new HashSet<>()); + + StringBuilder sb = new StringBuilder(); + dsIds.forEach(dsId -> { + sb + .append("<") + .append(name); + for (KeyValue kv : measure.getUnit()) { + sb.append(" ").append(attr(measure.getId(), kv.getValue())); + } + sb + .append(">") + .append(dsId) + .append(""); + }); + return sb.toString(); + } + public static String mapEoscIf(EoscIfGuidelines e) { return asXmlElement( "eoscifguidelines", Lists diff --git a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/input_params_xml_converter.json b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/input_params_payload_converter.json similarity index 100% rename from dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/input_params_xml_converter.json rename to dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/input_params_payload_converter.json diff --git a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml index 1fc28e7ca..59058d467 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml @@ -594,7 +594,7 @@ yarn cluster create_payloads - eu.dnetlib.dhp.oa.provision.XmlConverterJob + eu.dnetlib.dhp.oa.provision.PayloadConverterJob dhp-graph-provision-${projectVersion}.jar --executor-cores=${sparkExecutorCores} diff --git a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/EOSCFuture_Test.java b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/EOSCFuture_Test.java index 1a982ca39..4c43de25c 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/EOSCFuture_Test.java +++ b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/EOSCFuture_Test.java @@ -50,7 +50,7 @@ public class EOSCFuture_Test { final ContextMapper contextMapper = new ContextMapper(); final XmlRecordFactory xmlRecordFactory = new XmlRecordFactory(contextMapper, false, - XmlConverterJob.schemaLocation); + PayloadConverterJob.schemaLocation); final OtherResearchProduct p = OBJECT_MAPPER .readValue( diff --git a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/IndexRecordTransformerTest.java b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/IndexRecordTransformerTest.java index 8d5aa3f3a..718b43f03 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/IndexRecordTransformerTest.java +++ b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/IndexRecordTransformerTest.java @@ -57,7 +57,7 @@ public class IndexRecordTransformerTest { public void testPublicationRecordTransformation() throws IOException, TransformerException { final XmlRecordFactory xmlRecordFactory = new XmlRecordFactory(contextMapper, false, - XmlConverterJob.schemaLocation); + PayloadConverterJob.schemaLocation); final Publication p = load("publication.json", Publication.class); final Project pj = load("project.json", Project.class); @@ -82,7 +82,7 @@ public class IndexRecordTransformerTest { void testPeerReviewed() throws IOException, TransformerException { final XmlRecordFactory xmlRecordFactory = new XmlRecordFactory(contextMapper, false, - XmlConverterJob.schemaLocation); + PayloadConverterJob.schemaLocation); final Publication p = load("publication.json", Publication.class); @@ -98,7 +98,7 @@ public class IndexRecordTransformerTest { public void testRiunet() throws IOException, TransformerException { final XmlRecordFactory xmlRecordFactory = new XmlRecordFactory(contextMapper, false, - XmlConverterJob.schemaLocation); + PayloadConverterJob.schemaLocation); final Publication p = load("riunet.json", Publication.class); diff --git a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java index f26c384d2..d617991a1 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java +++ b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java @@ -37,7 +37,7 @@ public class XmlRecordFactoryTest { final ContextMapper contextMapper = new ContextMapper(); final XmlRecordFactory xmlRecordFactory = new XmlRecordFactory(contextMapper, false, - XmlConverterJob.schemaLocation); + PayloadConverterJob.schemaLocation); final Publication p = OBJECT_MAPPER .readValue(IOUtils.toString(getClass().getResourceAsStream("publication.json")), Publication.class); @@ -105,7 +105,7 @@ public class XmlRecordFactoryTest { final ContextMapper contextMapper = new ContextMapper(); final XmlRecordFactory xmlRecordFactory = new XmlRecordFactory(contextMapper, false, - XmlConverterJob.schemaLocation); + PayloadConverterJob.schemaLocation); final Publication p = OBJECT_MAPPER .readValue(IOUtils.toString(getClass().getResourceAsStream("publication.json")), Publication.class); @@ -136,7 +136,7 @@ public class XmlRecordFactoryTest { final ContextMapper contextMapper = new ContextMapper(); final XmlRecordFactory xmlRecordFactory = new XmlRecordFactory(contextMapper, false, - XmlConverterJob.schemaLocation); + PayloadConverterJob.schemaLocation); final Publication p = OBJECT_MAPPER .readValue(IOUtils.toString(getClass().getResourceAsStream("publication.json")), Publication.class); @@ -166,7 +166,7 @@ public class XmlRecordFactoryTest { final ContextMapper contextMapper = new ContextMapper(); final XmlRecordFactory xmlRecordFactory = new XmlRecordFactory(contextMapper, false, - XmlConverterJob.schemaLocation); + PayloadConverterJob.schemaLocation); final Datasource d = OBJECT_MAPPER .readValue(IOUtils.toString(getClass().getResourceAsStream("datasource.json")), Datasource.class); @@ -203,7 +203,7 @@ public class XmlRecordFactoryTest { final ContextMapper contextMapper = new ContextMapper(); final XmlRecordFactory xmlRecordFactory = new XmlRecordFactory(contextMapper, false, - XmlConverterJob.schemaLocation); + PayloadConverterJob.schemaLocation); final OtherResearchProduct p = OBJECT_MAPPER .readValue( @@ -226,7 +226,7 @@ public class XmlRecordFactoryTest { final ContextMapper contextMapper = new ContextMapper(); final XmlRecordFactory xmlRecordFactory = new XmlRecordFactory(contextMapper, false, - XmlConverterJob.schemaLocation); + PayloadConverterJob.schemaLocation); final OtherResearchProduct p = OBJECT_MAPPER .readValue( @@ -249,7 +249,7 @@ public class XmlRecordFactoryTest { final ContextMapper contextMapper = new ContextMapper(); final XmlRecordFactory xmlRecordFactory = new XmlRecordFactory(contextMapper, false, - XmlConverterJob.schemaLocation); + PayloadConverterJob.schemaLocation); final Publication p = OBJECT_MAPPER .readValue( From 55f39f785094f6500171d06945b3e5fcfc479a4c Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Thu, 9 May 2024 14:06:04 +0200 Subject: [PATCH 52/97] [graph provision] adds the possibility to validate the XML records before storing them via the validateXML parameter --- .../dhp/oa/provision/PayloadConverterJob.java | 17 ++++++++++++----- .../input_params_payload_converter.json | 6 ++++++ .../dhp/oa/provision/oozie_app/workflow.xml | 6 ++++++ 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/PayloadConverterJob.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/PayloadConverterJob.java index f34caad75..d7e22e557 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/PayloadConverterJob.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/PayloadConverterJob.java @@ -64,6 +64,12 @@ public class PayloadConverterJob { final String outputPath = parser.get("outputPath"); log.info("outputPath: {}", outputPath); + final Boolean validateXML = Optional + .ofNullable(parser.get("validateXML")) + .map(Boolean::valueOf) + .orElse(Boolean.FALSE); + log.info("validateXML: {}", validateXML); + final String contextApiBaseUrl = parser.get("contextApiBaseUrl"); log.info("contextApiBaseUrl: {}", contextApiBaseUrl); @@ -78,18 +84,19 @@ public class PayloadConverterJob { runWithSparkSession(conf, isSparkSessionManaged, spark -> { removeOutputDir(spark, outputPath); - convertToXml( + createPayloads( spark, inputPath, outputPath, ContextMapper.fromAPI(contextApiBaseUrl), - VocabularyGroup.loadVocsFromIS(isLookup)); + VocabularyGroup.loadVocsFromIS(isLookup), validateXML); }); } - private static void convertToXml( + private static void createPayloads( final SparkSession spark, final String inputPath, final String outputPath, final ContextMapper contextMapper, - final VocabularyGroup vocabularies) { + final VocabularyGroup vocabularies, + final Boolean validateXML) { final XmlRecordFactory recordFactory = new XmlRecordFactory( prepareAccumulators(spark.sparkContext()), @@ -110,7 +117,7 @@ public class PayloadConverterJob { .as(Encoders.kryo(JoinedEntity.class)) .map( (MapFunction>) je -> new Tuple2<>( - recordFactory.build(je), + recordFactory.build(je, validateXML), ProvisionModelSupport.transform(je, contextMapper, vocabularies)), Encoders.tuple(Encoders.STRING(), Encoders.bean(SolrRecord.class))) .map( diff --git a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/input_params_payload_converter.json b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/input_params_payload_converter.json index 4509eb9de..1b43ca5fd 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/input_params_payload_converter.json +++ b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/input_params_payload_converter.json @@ -22,5 +22,11 @@ "paramLongName": "isLookupUrl", "paramDescription": "URL of the context ISLookup Service", "paramRequired": true + }, + { + "paramName": "val", + "paramLongName": "validateXML", + "paramDescription": "should the process check the XML validity", + "paramRequired": false } ] diff --git a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml index 59058d467..1682f2ed5 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml @@ -13,6 +13,11 @@ contextApiBaseUrl context API URL + + validateXML + should the payload converter validate the XMLs + false + relPartitions number or partitions for the relations Dataset @@ -610,6 +615,7 @@ --inputPath/user/claudio.atzori/data/beta_provision/join_entities --outputPath${workingDir}/xml_json + --validateXML${validateXML} --contextApiBaseUrl${contextApiBaseUrl} --isLookupUrl${isLookupUrl} From 1efe7f7e39ea10d9c010cdefd40e1439b5bb52dd Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 14 May 2024 12:39:31 +0200 Subject: [PATCH 53/97] [graph provision] upgrade to dhp-schema:6.1.2, included project.oamandatepublications in the JSON payload mapping, fixed serialisation of the usageCounts measures --- .../dhp/oa/provision/PayloadConverterJob.java | 6 +- .../model/ProvisionModelSupport.java | 1 + .../oa/provision/utils/XmlRecordFactory.java | 79 ++++++++++--------- .../utils/XmlSerializationUtils.java | 8 +- pom.xml | 2 +- 5 files changed, 49 insertions(+), 47 deletions(-) diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/PayloadConverterJob.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/PayloadConverterJob.java index d7e22e557..d46ab1404 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/PayloadConverterJob.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/PayloadConverterJob.java @@ -65,9 +65,9 @@ public class PayloadConverterJob { log.info("outputPath: {}", outputPath); final Boolean validateXML = Optional - .ofNullable(parser.get("validateXML")) - .map(Boolean::valueOf) - .orElse(Boolean.FALSE); + .ofNullable(parser.get("validateXML")) + .map(Boolean::valueOf) + .orElse(Boolean.FALSE); log.info("validateXML: {}", validateXML); final String contextApiBaseUrl = parser.get("contextApiBaseUrl"); diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/ProvisionModelSupport.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/ProvisionModelSupport.java index a085a72e0..48e6b3ec9 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/ProvisionModelSupport.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/ProvisionModelSupport.java @@ -146,6 +146,7 @@ public class ProvisionModelSupport { ps.setContracttype(mapCodeLabel(p.getContracttype())); ps.setCurrency(mapField(p.getCurrency())); ps.setDuration(mapField(p.getDuration())); + ps.setOamandatepublications(mapField(p.getOamandatepublications())); ps.setCallidentifier(mapField(p.getCallidentifier())); ps.setEcarticle29_3(mapField(p.getEcarticle29_3())); ps.setEnddate(mapField(p.getEnddate())); diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java index 65fa122c8..ec322dbd4 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java @@ -1,23 +1,25 @@ package eu.dnetlib.dhp.oa.provision.utils; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Joiner; -import com.google.common.base.Splitter; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import com.mycila.xmltool.XMLDoc; -import com.mycila.xmltool.XMLTag; -import eu.dnetlib.dhp.oa.provision.model.JoinedEntity; -import eu.dnetlib.dhp.oa.provision.model.RelatedEntity; -import eu.dnetlib.dhp.oa.provision.model.RelatedEntityWrapper; -import eu.dnetlib.dhp.oa.provision.model.XmlInstance; -import eu.dnetlib.dhp.schema.common.*; -import eu.dnetlib.dhp.schema.oaf.Result; -import eu.dnetlib.dhp.schema.oaf.*; -import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; -import eu.dnetlib.dhp.schema.oaf.utils.ModelHardLimits; +import static eu.dnetlib.dhp.oa.provision.utils.GraphMappingUtils.authorPidTypes; +import static eu.dnetlib.dhp.oa.provision.utils.GraphMappingUtils.getRelDescriptor; +import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.apache.commons.lang3.StringUtils.substringBefore; + +import java.io.IOException; +import java.io.Serializable; +import java.io.StringReader; +import java.io.StringWriter; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import javax.xml.transform.*; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; + import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; @@ -29,26 +31,27 @@ import org.dom4j.Node; import org.dom4j.io.OutputFormat; import org.dom4j.io.SAXReader; import org.dom4j.io.XMLWriter; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.mycila.xmltool.XMLDoc; +import com.mycila.xmltool.XMLTag; + +import eu.dnetlib.dhp.oa.provision.model.JoinedEntity; +import eu.dnetlib.dhp.oa.provision.model.RelatedEntity; +import eu.dnetlib.dhp.oa.provision.model.RelatedEntityWrapper; +import eu.dnetlib.dhp.oa.provision.model.XmlInstance; +import eu.dnetlib.dhp.schema.common.*; +import eu.dnetlib.dhp.schema.oaf.*; +import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; +import eu.dnetlib.dhp.schema.oaf.utils.ModelHardLimits; import scala.Tuple2; -import javax.xml.transform.*; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; -import java.io.IOException; -import java.io.Serializable; -import java.io.StringReader; -import java.io.StringWriter; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static eu.dnetlib.dhp.oa.provision.utils.GraphMappingUtils.authorPidTypes; -import static eu.dnetlib.dhp.oa.provision.utils.GraphMappingUtils.getRelDescriptor; -import static org.apache.commons.lang3.StringUtils.isNotBlank; -import static org.apache.commons.lang3.StringUtils.substringBefore; - public class XmlRecordFactory implements Serializable { /** @@ -127,9 +130,9 @@ public class XmlRecordFactory implements Serializable { if (Boolean.TRUE.equals(validate)) { // rise an exception when an invalid record was built new SAXReader().read(new StringReader(xmlRecord)); - } - return xmlRecord; - // return printXML(templateFactory.buildRecord(entity, schemaLocation, body), indent); + } + return xmlRecord; + // return printXML(templateFactory.buildRecord(entity, schemaLocation, body), indent); } catch (final Throwable e) { throw new RuntimeException(String.format("error building record '%s'", entity.getId()), e); } diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlSerializationUtils.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlSerializationUtils.java index 31763ace3..b4d021b68 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlSerializationUtils.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlSerializationUtils.java @@ -190,11 +190,9 @@ public class XmlSerializationUtils { sb.append(" ").append(attr(measure.getId(), kv.getValue())); } sb - .append(">") - .append(dsId) - .append(""); + .append(" ") + .append(attr("datasource", dsId)) + .append("/>"); }); return sb.toString(); } diff --git a/pom.xml b/pom.xml index 892382b9d..bd19bda49 100644 --- a/pom.xml +++ b/pom.xml @@ -888,7 +888,7 @@ 3.3.3 3.4.2 [2.12,3.0) - [6.1.1] + [6.1.2] [4.0.3] [6.0.5] [3.1.6] From 2b3b5fe9a172bb1fafb4815a5c52aa9fcaff6644 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Wed, 15 May 2024 14:13:16 +0200 Subject: [PATCH 54/97] oai finalization and test --- .../dhp/oa/oaipmh/IrishOaiExporterJob.java | 26 ++++- .../dhp/oa/oaipmh/OaiRecordWrapper.java | 11 ++- .../input_params_irish_oai_exporter.json | 6 -- .../eu/dnetlib/dhp/oa/oaipmh/oai-finalize.sql | 12 +++ .../dhp/oa/oaipmh/DbSerializationTest.java | 97 +++++++++++++++++++ .../oa/oaipmh/IrishOaiExporterJobTest.java | 14 ++- 6 files changed, 146 insertions(+), 20 deletions(-) create mode 100644 dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/oaipmh/oai-finalize.sql create mode 100644 dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/oaipmh/DbSerializationTest.java diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java index b59f0ae73..433baf272 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java @@ -1,3 +1,4 @@ + package eu.dnetlib.dhp.oa.oaipmh; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; @@ -5,6 +6,9 @@ import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.charset.Charset; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Optional; @@ -37,6 +41,8 @@ public class IrishOaiExporterJob { protected static final int NUM_CONNECTIONS = 20; + public static final String TMP_OAI_TABLE = "temp_oai_data"; + public static void main(final String[] args) throws Exception { final ArgumentApplicationParser parser = new ArgumentApplicationParser( @@ -53,7 +59,6 @@ public class IrishOaiExporterJob { final String inputPath = parser.get("inputPath"); final String dbUrl = parser.get("dbUrl"); - final String dbTable = parser.get("dbTable"); final String dbUser = parser.get("dbUser"); final String dbPwd = parser.get("dbPwd"); final int numConnections = Optional @@ -64,7 +69,6 @@ public class IrishOaiExporterJob { log.info("inputPath: '{}'", inputPath); log.info("dbUrl: '{}'", dbUrl); log.info("dbUser: '{}'", dbUser); - log.info("table: '{}'", dbTable); log.info("dbPwd: '{}'", "xxx"); log.info("numPartitions: '{}'", numConnections); @@ -80,6 +84,7 @@ public class IrishOaiExporterJob { final Encoder encoderTuple = Encoders.bean(TupleWrapper.class); final Encoder encoderOaiRecord = Encoders.bean(OaiRecordWrapper.class); + log.info("Creating temporary table..."); runWithSparkSession(conf, isSparkSessionManaged, spark -> { final Dataset docs = spark @@ -91,12 +96,23 @@ public class IrishOaiExporterJob { .map((MapFunction) IrishOaiExporterJob::asIrishOaiResult, encoderOaiRecord) .filter((FilterFunction) obj -> (obj != null) && StringUtils.isNotBlank(obj.getId())); - docs.repartition(numConnections) + docs + .repartition(numConnections) .write() .mode(SaveMode.Overwrite) - .jdbc(dbUrl, dbTable, connectionProperties); + .jdbc(dbUrl, TMP_OAI_TABLE, connectionProperties); }); + log.info("Temporary table created."); + + log.info("Updating OAI records..."); + try (final Connection con = DriverManager.getConnection(dbUrl, dbUser, dbPwd)) { + try (final Statement st = con.createStatement()) { + final String query = IOUtils.toString(IrishOaiExporterJob.class.getResourceAsStream("oai-finalize.sql")); + st.execute(query); + } + } + log.info("DONE."); } protected static OaiRecordWrapper asIrishOaiResult(final String xml) { @@ -107,7 +123,7 @@ public class IrishOaiExporterJob { if (isValid(doc)) { r.setId(doc.valueOf("//*[local-name()='objIdentifier']").trim()); r.setBody(gzip(doc.selectSingleNode("//*[local-name()='entity']").asXML())); - r.setDate(LocalDateTime.now()); + r.setDate(LocalDateTime.now().toString()); r.setSets(new ArrayList<>()); } return r; diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/OaiRecordWrapper.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/OaiRecordWrapper.java index 4c2766754..2fdf32c96 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/OaiRecordWrapper.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/OaiRecordWrapper.java @@ -1,7 +1,7 @@ + package eu.dnetlib.dhp.oa.oaipmh; import java.io.Serializable; -import java.time.LocalDateTime; import java.util.List; public class OaiRecordWrapper implements Serializable { @@ -10,10 +10,11 @@ public class OaiRecordWrapper implements Serializable { private String id; private byte[] body; - private LocalDateTime date; + private String date; private List sets; - public OaiRecordWrapper() {} + public OaiRecordWrapper() { + } public String getId() { return this.id; @@ -31,11 +32,11 @@ public class OaiRecordWrapper implements Serializable { this.body = body; } - public LocalDateTime getDate() { + public String getDate() { return this.date; } - public void setDate(final LocalDateTime date) { + public void setDate(final String date) { this.date = date; } diff --git a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/oaipmh/input_params_irish_oai_exporter.json b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/oaipmh/input_params_irish_oai_exporter.json index 99a12927b..86b2bb0d3 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/oaipmh/input_params_irish_oai_exporter.json +++ b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/oaipmh/input_params_irish_oai_exporter.json @@ -23,12 +23,6 @@ "paramDescription": "the user of the database", "paramRequired": true }, - { - "paramName": "t", - "paramLongName": "dbTable", - "paramDescription": "the name of the table in the database", - "paramRequired": true - }, { "paramName": "dpwd", "paramLongName": "dbPwd", diff --git a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/oaipmh/oai-finalize.sql b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/oaipmh/oai-finalize.sql new file mode 100644 index 000000000..1ec0dfee0 --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/oaipmh/oai-finalize.sql @@ -0,0 +1,12 @@ +BEGIN; + +DELETE FROM oai_data; + +INSERT INTO oai_data(id, body, date, sets) SELECT + id, + body, + date::timestamp, + sets +FROM temp_oai_data; + +COMMIT; diff --git a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/oaipmh/DbSerializationTest.java b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/oaipmh/DbSerializationTest.java new file mode 100644 index 000000000..f33708f86 --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/oaipmh/DbSerializationTest.java @@ -0,0 +1,97 @@ + +package eu.dnetlib.dhp.oa.oaipmh; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; + +import org.apache.commons.io.IOUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SaveMode; +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +@Disabled +public class DbSerializationTest { + + private static SparkSession spark; + + public static final String dbUrl = "jdbc:postgresql://localhost:5432/db_test"; + public static final String dbUser = null; + public static final String dbPwd = null; + + @BeforeAll + public static void beforeAll() throws IOException { + + final SparkConf conf = new SparkConf(); + conf.setAppName("TEST"); + conf.setMaster("local[*]"); + conf.set("spark.driver.host", "localhost"); + + spark = SparkSession + .builder() + .appName("TEST") + .config(conf) + .getOrCreate(); + } + + @AfterAll + public static void afterAll() throws IOException { + spark.stop(); + } + + @Test + public void testDatabaseSerialization() throws Exception { + final Properties connectionProperties = new Properties(); + if (dbUser != null) { + connectionProperties.put("user", dbUser); + } + if (dbPwd != null) { + connectionProperties.put("password", dbPwd); + } + + runWithSparkSession(new SparkConf(), false, spark -> { + + final List list = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + final OaiRecordWrapper r = new OaiRecordWrapper(); + r.setId("record_" + i); + r.setBody("jsahdjkahdjahdajad".getBytes()); + r.setDate(LocalDateTime.now().toString()); + r.setSets(Arrays.asList()); + list.add(r); + } + + final Dataset docs = spark.createDataset(list, Encoders.bean(OaiRecordWrapper.class)); + + docs + .write() + .mode(SaveMode.Overwrite) + .jdbc(dbUrl, IrishOaiExporterJob.TMP_OAI_TABLE, connectionProperties); + + }); + + try (final Connection con = DriverManager.getConnection(dbUrl, dbUser, dbPwd)) { + try (final Statement st = con.createStatement()) { + final String query = IOUtils.toString(getClass().getResourceAsStream("oai-finalize.sql")); + st.execute(query); + } + } + + } + +} diff --git a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJobTest.java b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJobTest.java index e33c701c5..57a32e246 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJobTest.java +++ b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJobTest.java @@ -1,3 +1,4 @@ + package eu.dnetlib.dhp.oa.oaipmh; import static org.junit.Assert.assertNull; @@ -17,7 +18,7 @@ import org.dom4j.DocumentException; import org.dom4j.io.SAXReader; import org.junit.jupiter.api.Test; -class IrishOaiExporterJobTest { +public class IrishOaiExporterJobTest { @Test void testAsIrishOaiResult() throws Exception { @@ -66,8 +67,12 @@ class IrishOaiExporterJobTest { } public static String gunzip(final byte[] compressed) { - if ((compressed == null) || (compressed.length == 0)) { return null; } - if (!isCompressed(compressed)) { return new String(compressed); } + if ((compressed == null) || (compressed.length == 0)) { + return null; + } + if (!isCompressed(compressed)) { + return new String(compressed); + } try (final GZIPInputStream gis = new GZIPInputStream(new ByteArrayInputStream(compressed))) { return IOUtils.toString(gis, Charset.defaultCharset()); } catch (final IOException e) { @@ -76,6 +81,7 @@ class IrishOaiExporterJobTest { } private static boolean isCompressed(final byte[] compressed) { - return (compressed[0] == (byte) GZIPInputStream.GZIP_MAGIC) && (compressed[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8)); + return (compressed[0] == (byte) GZIPInputStream.GZIP_MAGIC) + && (compressed[1] == (byte) (GZIPInputStream.GZIP_MAGIC >> 8)); } } From 0611c81a2fcdb769974dc35c412774c76a1921bb Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Wed, 15 May 2024 15:33:10 +0200 Subject: [PATCH 55/97] [graph provision] using Qualifier.classNames to populate the correponsing fields in the JSON payload --- .../provision/model/ProvisionModelSupport.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/ProvisionModelSupport.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/ProvisionModelSupport.java index 48e6b3ec9..f46aebdcf 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/ProvisionModelSupport.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/model/ProvisionModelSupport.java @@ -387,7 +387,7 @@ public class ProvisionModelSupport { .equals( Optional .ofNullable(t.getQualifier()) - .map(Qualifier::getClassid) + .map(Qualifier::getClassname) .orElse(null))) .map(StructuredProperty::getValue) .collect(Collectors.toList())) @@ -405,7 +405,7 @@ public class ProvisionModelSupport { .equals( Optional .ofNullable(t.getQualifier()) - .map(Qualifier::getClassid) + .map(Qualifier::getClassname) .orElse(null))) .map(StructuredProperty::getValue) .findFirst()) @@ -472,7 +472,7 @@ public class ProvisionModelSupport { } private static String mapQualifier(eu.dnetlib.dhp.schema.oaf.Qualifier q) { - return Optional.ofNullable(q).map(Qualifier::getClassid).orElse(null); + return Optional.ofNullable(q).map(Qualifier::getClassname).orElse(null); } private static Journal mapJournal(eu.dnetlib.dhp.schema.oaf.Journal joaf) { @@ -581,7 +581,7 @@ public class ProvisionModelSupport { .map( pids -> pids .stream() - .map(p -> Pid.newInstance(p.getQualifier().getClassid(), p.getValue())) + .map(p -> Pid.newInstance(p.getQualifier().getClassname(), p.getValue())) .collect(Collectors.toList())) .orElse(null); } @@ -606,8 +606,8 @@ public class ProvisionModelSupport { subjects -> subjects .stream() .filter(s -> Objects.nonNull(s.getQualifier())) - .filter(s -> Objects.nonNull(s.getQualifier().getClassid())) - .map(s -> Subject.newInstance(s.getValue(), s.getQualifier().getClassid())) + .filter(s -> Objects.nonNull(s.getQualifier().getClassname())) + .map(s -> Subject.newInstance(s.getValue(), s.getQualifier().getClassname())) .collect(Collectors.toList())) .orElse(null); } @@ -619,8 +619,8 @@ public class ProvisionModelSupport { subjects -> subjects .stream() .filter(s -> Objects.nonNull(s.getQualifier())) - .filter(s -> Objects.nonNull(s.getQualifier().getClassid())) - .map(s -> Subject.newInstance(s.getValue(), s.getQualifier().getClassid())) + .filter(s -> Objects.nonNull(s.getQualifier().getClassname())) + .map(s -> Subject.newInstance(s.getValue(), s.getQualifier().getClassname())) .collect(Collectors.toList())) .orElse(null); } From 92f018d1962c964f4c15ac18a9d33b2fe6ae5301 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Wed, 15 May 2024 15:39:18 +0200 Subject: [PATCH 56/97] [graph provision] fixed path pointing to an intermediate data store in the working directory --- .../eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml index 1682f2ed5..50acb4526 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml @@ -613,7 +613,7 @@ --conf spark.sql.shuffle.partitions=3840 --conf spark.network.timeout=${sparkNetworkTimeout} - --inputPath/user/claudio.atzori/data/beta_provision/join_entities + --inputPath${workingDir}/join_entities --outputPath${workingDir}/xml_json --validateXML${validateXML} --contextApiBaseUrl${contextApiBaseUrl} From 6efab4d88e7ce481896e5569e1801daf81c96777 Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Thu, 16 May 2024 16:19:18 +0200 Subject: [PATCH 57/97] fixed scholexplorer bug --- .../dhp/sx/graph/scholix/ScholixUtils.scala | 2 +- dhp-shade-package/pom.xml | 150 +++++++++--------- .../dhp/sx/graph/ScholexplorerUtils.scala | 15 +- .../graph/SparkCreateScholexplorerDump.scala | 23 ++- .../graph/scholix/ScholixGenerationTest.scala | 17 +- 5 files changed, 112 insertions(+), 95 deletions(-) diff --git a/dhp-common/src/main/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixUtils.scala b/dhp-common/src/main/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixUtils.scala index f256ca1a1..72a17777e 100644 --- a/dhp-common/src/main/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixUtils.scala +++ b/dhp-common/src/main/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixUtils.scala @@ -389,7 +389,7 @@ object ScholixUtils extends Serializable { if (persistentIdentifiers.isEmpty) return null s.setLocalIdentifier(persistentIdentifiers.asJava) - s.setTypology(r.getResulttype.getClassid) +// s.setTypology(r.getResulttype.getClassid) s.setSubType(r.getInstance().get(0).getInstancetype.getClassname) diff --git a/dhp-shade-package/pom.xml b/dhp-shade-package/pom.xml index 128a57116..fd9c04066 100644 --- a/dhp-shade-package/pom.xml +++ b/dhp-shade-package/pom.xml @@ -31,86 +31,86 @@ dhp-actionmanager ${project.version} - - eu.dnetlib.dhp - dhp-aggregation - ${project.version} - - - eu.dnetlib.dhp - dhp-blacklist - ${project.version} - - - eu.dnetlib.dhp - dhp-broker-events - ${project.version} - - - eu.dnetlib.dhp - dhp-dedup-openaire - ${project.version} - - - eu.dnetlib.dhp - dhp-enrichment - ${project.version} - + + + + + + + + + + + + + + + + + + + + + + + + + eu.dnetlib.dhp dhp-graph-mapper ${project.version} - - eu.dnetlib.dhp - dhp-graph-provision - ${project.version} - - - eu.dnetlib.dhp - dhp-impact-indicators - ${project.version} - - - eu.dnetlib.dhp - dhp-stats-actionsets - ${project.version} - - - eu.dnetlib.dhp - dhp-stats-hist-snaps - ${project.version} - - - eu.dnetlib.dhp - dhp-stats-monitor-irish - ${project.version} - - - eu.dnetlib.dhp - dhp-stats-promote - ${project.version} - - - eu.dnetlib.dhp - dhp-stats-update - ${project.version} - - - eu.dnetlib.dhp - dhp-swh - ${project.version} - - - eu.dnetlib.dhp - dhp-usage-raw-data-update - ${project.version} - - - eu.dnetlib.dhp - dhp-usage-stats-build - ${project.version} - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/ScholexplorerUtils.scala b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/ScholexplorerUtils.scala index 95564d523..f62f271e3 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/ScholexplorerUtils.scala +++ b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/ScholexplorerUtils.scala @@ -1,14 +1,8 @@ package eu.dnetlib.dhp.sx.graph +import com.fasterxml.jackson.databind.ObjectMapper import eu.dnetlib.dhp.schema.oaf.{KeyValue, Result, StructuredProperty} -import eu.dnetlib.dhp.schema.sx.scholix.{ - Scholix, - ScholixCollectedFrom, - ScholixEntityId, - ScholixIdentifier, - ScholixRelationship, - ScholixResource -} +import eu.dnetlib.dhp.schema.sx.scholix.{Scholix, ScholixCollectedFrom, ScholixEntityId, ScholixIdentifier, ScholixRelationship, ScholixResource} import org.json4s import org.json4s.DefaultFormats import org.json4s.jackson.JsonMethods.parse @@ -28,6 +22,7 @@ case class RelKeyValue(key: String, value: String) {} object ScholexplorerUtils { val OPENAIRE_IDENTIFIER_SCHEMA: String = "OpenAIRE Identifier" + val mapper= new ObjectMapper() case class RelationVocabulary(original: String, inverse: String) {} @@ -242,7 +237,7 @@ object ScholexplorerUtils { s } - def updateTarget(s: Scholix, t: ScholixResource): Scholix = { + def updateTarget(s: Scholix, t: ScholixResource): String = { s.setTarget(t) val spublishers: Seq[ScholixEntityId] = @@ -251,6 +246,6 @@ object ScholexplorerUtils { if (t.getPublisher != null && !t.getPublisher.isEmpty) t.getPublisher.asScala else List() val mergedPublishers = spublishers.union(tpublishers).distinct.take(10).toList s.setPublisher(mergedPublishers.asJava) - s + mapper.writeValueAsString(s) } } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkCreateScholexplorerDump.scala b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkCreateScholexplorerDump.scala index 1211dcc78..32aa68665 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkCreateScholexplorerDump.scala +++ b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkCreateScholexplorerDump.scala @@ -11,7 +11,7 @@ import eu.dnetlib.dhp.schema.oaf.{ Dataset => OafDataset } import eu.dnetlib.dhp.schema.sx.scholix.{Scholix, ScholixResource} -import org.apache.spark.sql.functions.{col, concat, expr, md5} +import org.apache.spark.sql.functions.{col, concat, expr, first, md5} import org.apache.spark.sql.types.StructType import org.apache.spark.sql._ import org.slf4j.{Logger, LoggerFactory} @@ -89,7 +89,13 @@ class SparkCreateScholexplorerDump(propertyPath: String, args: Array[String], lo .withColumn("cf", expr("transform(collectedfrom, x -> struct(x.key, x.value))")) .drop("collectedfrom") .withColumnRenamed("cf", "collectedfrom") - .distinct() + .groupBy(col("id")) + .agg( + first("source").alias("source"), + first("target").alias("target"), + first("relClass").alias("relClass"), + first("collectedfrom").alias("collectedfrom") + ) bidRel.write.mode(SaveMode.Overwrite).save(s"$otuputPath/relation") @@ -97,27 +103,32 @@ class SparkCreateScholexplorerDump(propertyPath: String, args: Array[String], lo def generateScholix(outputPath: String, spark: SparkSession): Unit = { implicit val scholixResourceEncoder: Encoder[ScholixResource] = Encoders.bean(classOf[ScholixResource]) - implicit val scholixEncoder: Encoder[Scholix] = Encoders.bean(classOf[Scholix]) + implicit val scholixEncoder: Encoder[Scholix] = Encoders.kryo(classOf[Scholix]) import spark.implicits._ val relations = spark.read.load(s"$outputPath/relation").as[RelationInfo] val resource = spark.read.load(s"$outputPath/resource").as[ScholixResource] + + val scholix_one_verse = relations .joinWith(resource, relations("source") === resource("dnetIdentifier"), "inner") .map(res => ScholexplorerUtils.generateScholix(res._1, res._2)) + .map(s=> (s.getIdentifier, s))(Encoders.tuple(Encoders.STRING, Encoders.kryo(classOf[Scholix]))) + val resourceTarget = relations .joinWith(resource, relations("target") === resource("dnetIdentifier"), "inner") .map(res => (res._1.id, res._2))(Encoders.tuple(Encoders.STRING, Encoders.kryo(classOf[ScholixResource]))) + scholix_one_verse - .joinWith(resourceTarget, scholix_one_verse("identifier") === resourceTarget("_1"), "inner") - .map(k => ScholexplorerUtils.updateTarget(k._1, k._2._2)) + .joinWith(resourceTarget, scholix_one_verse("_1") === resourceTarget("_1"), "inner") + .map(k => ScholexplorerUtils.updateTarget(k._1._2, k._2._2)) .write .mode(SaveMode.Overwrite) .option("compression", "gzip") - .json(s"$outputPath/scholix") + .text(s"$outputPath/scholix") } } diff --git a/dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixGenerationTest.scala b/dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixGenerationTest.scala index 0a2872cb4..67d40dcf1 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixGenerationTest.scala +++ b/dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixGenerationTest.scala @@ -1,17 +1,28 @@ package eu.dnetlib.dhp.sx.graph.scholix +import eu.dnetlib.dhp.schema.sx.scholix.ScholixResource import eu.dnetlib.dhp.sx.graph.SparkCreateScholexplorerDump -import org.apache.spark.sql.SparkSession +import org.apache.spark.SparkConf +import org.apache.spark.sql.{Encoder, Encoders, SparkSession} import org.junit.jupiter.api.Test +import org.objenesis.strategy.StdInstantiatorStrategy class ScholixGenerationTest { @Test def generateScholix(): Unit = { + val spark: SparkSession = SparkSession.builder().master("local[*]").getOrCreate() val app = new SparkCreateScholexplorerDump(null, null, null) -// app.generateScholixResource("/home/sandro/Downloads/scholix_sample/", "/home/sandro/Downloads/scholix/", spark) -// app.generateBidirectionalRelations("/home/sandro/Downloads/scholix_sample/", "/home/sandro/Downloads/scholix/", spark) +// app.generateScholixResource("/home/sandro/Downloads/scholix_sample/", "/home/sandro/Downloads/scholix/", spark) +// app.generateBidirectionalRelations( +// "/home/sandro/Downloads/scholix_sample/", +// "/home/sandro/Downloads/scholix/", +// spark +// ) app.generateScholix("/home/sandro/Downloads/scholix/", spark) + + + } } From a87f9ea64317dff7afac5045a4c64bb9c8a26954 Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Fri, 17 May 2024 14:16:43 +0200 Subject: [PATCH 58/97] fixed scholexplorer bug --- .../eu/dnetlib/dhp/sx/graph/ScholexplorerUtils.scala | 11 +++++++++-- .../dhp/sx/graph/SparkCreateScholexplorerDump.scala | 6 +----- .../dhp/sx/graph/scholix/ScholixGenerationTest.scala | 2 -- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/ScholexplorerUtils.scala b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/ScholexplorerUtils.scala index f62f271e3..d171d96d9 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/ScholexplorerUtils.scala +++ b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/ScholexplorerUtils.scala @@ -2,7 +2,14 @@ package eu.dnetlib.dhp.sx.graph import com.fasterxml.jackson.databind.ObjectMapper import eu.dnetlib.dhp.schema.oaf.{KeyValue, Result, StructuredProperty} -import eu.dnetlib.dhp.schema.sx.scholix.{Scholix, ScholixCollectedFrom, ScholixEntityId, ScholixIdentifier, ScholixRelationship, ScholixResource} +import eu.dnetlib.dhp.schema.sx.scholix.{ + Scholix, + ScholixCollectedFrom, + ScholixEntityId, + ScholixIdentifier, + ScholixRelationship, + ScholixResource +} import org.json4s import org.json4s.DefaultFormats import org.json4s.jackson.JsonMethods.parse @@ -22,7 +29,7 @@ case class RelKeyValue(key: String, value: String) {} object ScholexplorerUtils { val OPENAIRE_IDENTIFIER_SCHEMA: String = "OpenAIRE Identifier" - val mapper= new ObjectMapper() + val mapper = new ObjectMapper() case class RelationVocabulary(original: String, inverse: String) {} diff --git a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkCreateScholexplorerDump.scala b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkCreateScholexplorerDump.scala index 32aa68665..dd420ab95 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkCreateScholexplorerDump.scala +++ b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/sx/graph/SparkCreateScholexplorerDump.scala @@ -109,19 +109,15 @@ class SparkCreateScholexplorerDump(propertyPath: String, args: Array[String], lo val relations = spark.read.load(s"$outputPath/relation").as[RelationInfo] val resource = spark.read.load(s"$outputPath/resource").as[ScholixResource] - - val scholix_one_verse = relations .joinWith(resource, relations("source") === resource("dnetIdentifier"), "inner") .map(res => ScholexplorerUtils.generateScholix(res._1, res._2)) - .map(s=> (s.getIdentifier, s))(Encoders.tuple(Encoders.STRING, Encoders.kryo(classOf[Scholix]))) - + .map(s => (s.getIdentifier, s))(Encoders.tuple(Encoders.STRING, Encoders.kryo(classOf[Scholix]))) val resourceTarget = relations .joinWith(resource, relations("target") === resource("dnetIdentifier"), "inner") .map(res => (res._1.id, res._2))(Encoders.tuple(Encoders.STRING, Encoders.kryo(classOf[ScholixResource]))) - scholix_one_verse .joinWith(resourceTarget, scholix_one_verse("_1") === resourceTarget("_1"), "inner") .map(k => ScholexplorerUtils.updateTarget(k._1._2, k._2._2)) diff --git a/dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixGenerationTest.scala b/dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixGenerationTest.scala index 67d40dcf1..204fe9794 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixGenerationTest.scala +++ b/dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/sx/graph/scholix/ScholixGenerationTest.scala @@ -22,7 +22,5 @@ class ScholixGenerationTest { // ) app.generateScholix("/home/sandro/Downloads/scholix/", spark) - - } } From 032bcc8279849cfa498bc8227f8a96c4e1a48525 Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Mon, 20 May 2024 09:24:15 +0200 Subject: [PATCH 59/97] since last beta workflow we decide to introduce in the graph only MAG item with DOI and set them invisible ( this should be the same behaviour of the previous DOIBoost mapping). This commit apply this type of mapping --- .../dhp/collection/mag/MagUtility.scala | 41 +++++-------------- .../dhp/collection/mag/SparkMAGtoOAF.scala | 3 ++ .../dhp/collection/mag/MAGMappingTest.scala | 12 ++++-- 3 files changed, 22 insertions(+), 34 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/mag/MagUtility.scala b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/mag/MagUtility.scala index df22a6b84..c415dd9a4 100644 --- a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/mag/MagUtility.scala +++ b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/mag/MagUtility.scala @@ -79,23 +79,6 @@ object MagUtility extends Serializable { private val MAGCollectedFrom = keyValue(ModelConstants.MAG_ID, ModelConstants.MAG_NAME) private val MAGDataInfo: DataInfo = { - val di = new DataInfo - di.setDeletedbyinference(false) - di.setInferred(false) - di.setInvisible(false) - di.setTrust("0.9") - di.setProvenanceaction( - OafMapperUtils.qualifier( - ModelConstants.SYSIMPORT_ACTIONSET, - ModelConstants.SYSIMPORT_ACTIONSET, - ModelConstants.DNET_PROVENANCE_ACTIONS, - ModelConstants.DNET_PROVENANCE_ACTIONS - ) - ) - di - } - - private val MAGDataInfoInvisible: DataInfo = { val di = new DataInfo di.setDeletedbyinference(false) di.setInferred(false) @@ -111,8 +94,7 @@ object MagUtility extends Serializable { ) di } - - val datatypedict = Map( +val datatypedict = Map( "bool" -> BooleanType, "int" -> IntegerType, "uint" -> IntegerType, @@ -453,7 +435,6 @@ object MagUtility extends Serializable { case "repository" => result = new Publication() - result.setDataInfo(MAGDataInfoInvisible) qualifier( "0038", "Other literature type", @@ -488,8 +469,7 @@ object MagUtility extends Serializable { } if (result != null) { - if (result.getDataInfo == null) - result.setDataInfo(MAGDataInfo) + result.setDataInfo(MAGDataInfo) val i = new Instance i.setInstancetype(tp) i.setInstanceTypeMapping( @@ -512,7 +492,7 @@ object MagUtility extends Serializable { return null result.setCollectedfrom(List(MAGCollectedFrom).asJava) - val pidList = List( + var pidList = List( structuredProperty( paper.paperId.get.toString, qualifier( @@ -525,7 +505,7 @@ object MagUtility extends Serializable { ) ) - result.setPid(pidList.asJava) + result.setOriginalId(pidList.map(s => s.getValue).asJava) @@ -618,10 +598,9 @@ object MagUtility extends Serializable { } val instance = result.getInstance().get(0) - instance.setPid(pidList.asJava) - if (paper.doi.orNull != null) - instance.setAlternateIdentifier( - List( + + if (paper.doi.orNull != null) { + pidList = pidList ::: List( structuredProperty( paper.doi.get, qualifier( @@ -632,8 +611,10 @@ object MagUtility extends Serializable { ), null ) - ).asJava - ) + ) + } + instance.setPid(pidList.asJava) + result.setPid(pidList.asJava) instance.setUrl(paper.urls.get.asJava) instance.setHostedby(ModelConstants.UNKNOWN_REPOSITORY) instance.setCollectedfrom(MAGCollectedFrom) diff --git a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/mag/SparkMAGtoOAF.scala b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/mag/SparkMAGtoOAF.scala index 5dd38970d..123d8e0f8 100644 --- a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/mag/SparkMAGtoOAF.scala +++ b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/mag/SparkMAGtoOAF.scala @@ -35,9 +35,12 @@ class SparkMAGtoOAF(propertyPath: String, args: Array[String], log: Logger) def convertMAG(spark: SparkSession, magBasePath: String, mdStorePath: String): Unit = { import spark.implicits._ + + spark.read .load(s"$magBasePath/mag_denormalized") .as[MAGPaper] + .filter(col("doi").isNotNull) .map(s => MagUtility.convertMAGtoOAF(s)) .filter(s => s != null) .write diff --git a/dhp-workflows/dhp-aggregation/src/test/scala/eu/dnetlib/dhp/collection/mag/MAGMappingTest.scala b/dhp-workflows/dhp-aggregation/src/test/scala/eu/dnetlib/dhp/collection/mag/MAGMappingTest.scala index 59b91d66b..3ae25decb 100644 --- a/dhp-workflows/dhp-aggregation/src/test/scala/eu/dnetlib/dhp/collection/mag/MAGMappingTest.scala +++ b/dhp-workflows/dhp-aggregation/src/test/scala/eu/dnetlib/dhp/collection/mag/MAGMappingTest.scala @@ -3,13 +3,17 @@ package eu.dnetlib.dhp.collection.mag import com.fasterxml.jackson.databind.ObjectMapper import eu.dnetlib.dhp.schema.oaf.{Dataset, Publication, Result} import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.functions.col import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.Test + + class MAGMappingTest { val mapper = new ObjectMapper() + def mappingTest(): Unit = { val spark = SparkSession @@ -18,12 +22,12 @@ class MAGMappingTest { .master("local[*]") .getOrCreate() - val s = new SparkMagOrganizationAS(null, null, null) - - s.generateAS(spark, "/home/sandro/Downloads/mag_test", "/home/sandro/Downloads/mag_AS") - + val s = new SparkMAGtoOAF(null, null, null) + s.convertMAG(spark, "/Users/sandro/Downloads/", "/Users/sandro/Downloads/mag_OAF") } + + @Test def mappingMagType(): Unit = { From ca9414b737a1841eacf5cc7cea6c48f065ab3afc Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Tue, 21 May 2024 09:11:13 +0200 Subject: [PATCH 60/97] Implement multiple node name splitter on GZipCollectorPlugin and all nodes that use XMLIterator. If the splitter name contains is a comma separated values it splits for all the values --- .../plugin/gzip/GzipCollectorPlugin.java | 16 +++++ .../collection/plugin/utils/XMLIterator.java | 47 ++++++++++--- .../plugin/file/FileGZipMultipleNodeTest.java | 63 ++++++++++++++++++ .../dhp/collection/plugin/file/dblp.gz | Bin 0 -> 1097 bytes 4 files changed, 117 insertions(+), 9 deletions(-) create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/gzip/GzipCollectorPlugin.java create mode 100644 dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/file/FileGZipMultipleNodeTest.java create mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/plugin/file/dblp.gz diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/gzip/GzipCollectorPlugin.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/gzip/GzipCollectorPlugin.java new file mode 100644 index 000000000..44b1eeb18 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/gzip/GzipCollectorPlugin.java @@ -0,0 +1,16 @@ +package eu.dnetlib.dhp.collection.plugin.gzip; + +import eu.dnetlib.dhp.collection.ApiDescriptor; +import eu.dnetlib.dhp.collection.plugin.CollectorPlugin; +import eu.dnetlib.dhp.common.aggregation.AggregatorReport; +import eu.dnetlib.dhp.common.collection.CollectorException; + +import java.util.stream.Stream; + +public class GzipCollectorPlugin implements CollectorPlugin { + + @Override + public Stream collect(ApiDescriptor api, AggregatorReport report) throws CollectorException { + return Stream.empty(); + } +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/utils/XMLIterator.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/utils/XMLIterator.java index e05fe263a..ca351346c 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/utils/XMLIterator.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/utils/XMLIterator.java @@ -8,7 +8,10 @@ import java.io.StringWriter; import java.nio.charset.Charset; import java.nio.charset.CharsetDecoder; import java.nio.charset.CodingErrorAction; +import java.util.Arrays; import java.util.Iterator; +import java.util.List; +import java.util.stream.Collectors; import javax.xml.stream.XMLEventFactory; import javax.xml.stream.XMLEventReader; @@ -19,6 +22,7 @@ import javax.xml.stream.XMLStreamException; import javax.xml.stream.events.StartElement; import javax.xml.stream.events.XMLEvent; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -58,13 +62,22 @@ public class XMLIterator implements Iterator { private String element; + private List elements; + private InputStream inputStream; public XMLIterator(final String element, final InputStream inputStream) { super(); this.element = element; + if (element.contains(",")) { + elements= Arrays.stream(element.split(",")) + .filter(StringUtils::isNoneBlank) + .map(String::toLowerCase) + .collect(Collectors.toList()); + } this.inputStream = inputStream; this.parser = getParser(); + try { this.current = findElement(parser); } catch (XMLStreamException e) { @@ -113,7 +126,7 @@ public class XMLIterator implements Iterator { final XMLEvent event = parser.nextEvent(); // TODO: replace with depth tracking instead of close tag tracking. - if (event.isEndElement() && event.asEndElement().getName().getLocalPart().equals(element)) { + if (event.isEndElement() && isCheckTag(event.asEndElement().getName().getLocalPart())) { writer.add(event); break; } @@ -142,18 +155,16 @@ public class XMLIterator implements Iterator { XMLEvent peek = parser.peek(); if (peek != null && peek.isStartElement()) { String name = peek.asStartElement().getName().getLocalPart(); - if (element.equals(name)) { - return peek; - } + if( isCheckTag(name)) + return peek; } while (parser.hasNext()) { - final XMLEvent event = parser.nextEvent(); + XMLEvent event= parser.nextEvent(); if (event != null && event.isStartElement()) { String name = event.asStartElement().getName().getLocalPart(); - if (element.equals(name)) { - return event; - } + if( isCheckTag(name)) + return event; } } return null; @@ -161,12 +172,30 @@ public class XMLIterator implements Iterator { private XMLEventReader getParser() { try { - return inputFactory.get().createXMLEventReader(sanitize(inputStream)); + XMLInputFactory xif = inputFactory.get(); + xif.setProperty(XMLInputFactory.SUPPORT_DTD, false); + return xif.createXMLEventReader(sanitize(inputStream)); } catch (XMLStreamException e) { throw new RuntimeException(e); } } + private boolean isCheckTag(final String tagName) { + if (elements!= null) { + final String found =elements.stream() + .filter(e -> e.equalsIgnoreCase(tagName)) + .findFirst() + .orElse(null); + if (found!= null) + return true; + } else { + if (element.equalsIgnoreCase(tagName)) { + return true; + } + } + return false; + } + private Reader sanitize(final InputStream in) { final CharsetDecoder charsetDecoder = Charset.forName(UTF_8).newDecoder(); charsetDecoder.onMalformedInput(CodingErrorAction.REPLACE); diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/file/FileGZipMultipleNodeTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/file/FileGZipMultipleNodeTest.java new file mode 100644 index 000000000..2ed199156 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/file/FileGZipMultipleNodeTest.java @@ -0,0 +1,63 @@ +package eu.dnetlib.dhp.collection.plugin.file; + + +import eu.dnetlib.dhp.collection.ApiDescriptor; +import eu.dnetlib.dhp.common.aggregation.AggregatorReport; +import eu.dnetlib.dhp.common.collection.CollectorException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.junit.jupiter.api.*; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Objects; +import java.util.stream.Stream; + +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@ExtendWith(MockitoExtension.class) +public class FileGZipMultipleNodeTest { + + private static final Logger log = LoggerFactory.getLogger(FileGZipCollectorPluginTest.class); + + private final ApiDescriptor api = new ApiDescriptor(); + + private FileGZipCollectorPlugin plugin; + + private static final String SPLIT_ON_ELEMENT = "incollection,article"; + + @BeforeEach + public void setUp() throws IOException { + + final String gzipFile = Objects + .requireNonNull( + this + .getClass() + .getResource("/eu/dnetlib/dhp/collection/plugin/file/dblp.gz")) + .getFile(); + + api.setBaseUrl(gzipFile); + + HashMap params = new HashMap<>(); + params.put("splitOnElement", SPLIT_ON_ELEMENT); + + api.setParams(params); + + FileSystem fs = FileSystem.get(new Configuration()); + plugin = new FileGZipCollectorPlugin(fs); + } + + @Test + void test() throws CollectorException { + + final Stream stream = plugin.collect(api, new AggregatorReport()); + + stream.limit(10).forEach(s -> { + Assertions.assertTrue(s.length() > 0); + log.info(s); + }); + } +} diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/plugin/file/dblp.gz b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/plugin/file/dblp.gz new file mode 100644 index 0000000000000000000000000000000000000000..979bcbed2845c7c7995db4f1fdc10069f89f1bcb GIT binary patch literal 1097 zcmV-P1h)GhiwFotJWOT)17u=sZ~)DhT~pgQ6hPnmE41#+&h8AhtbCBTYutfADHIAr zQrJH2$hKmu$dX5vv(5hcy|P0?0RyFF1|EVfTUW|CS652-_G=-~g6WDYx#bYgcMy|_ zO1aFooP+6!J01@MmpE^uSK)8FCzILd(>;{NqD0fr)7joJa_o(ln$&?e>~-`C!C%NB ziV!R@5EvCHHEhf2`DEn!BbW3Xbjg-mj%IVFVLXNx+|B_1O$Kr5pt6pIS(eZR9xfB1 zN|ti!w4s~uh8w}6Nv#(wm8`0~5VuY1S4x_(Dk8%m#O>w!(UMUe!E6R0ZZ6kPVx=xu zu~kvlhG}%(Ol<>gqqD2&qWzkuEX3=#*AW zWr~^xQ75*c)8m*7-7$H| zE{8VLo+NTb`-v(aUQ(Pteo0MhJ*PnBL~|w$ZN_@fE2A<^3-re>uM*7+L+5SHKx&Tm z_7DE-^fnrk7win0d%dS>YsfyEwBqs3w#yPIQVV99P$?muI(_svU$zc1{s|q$^Vqmh zGW(%oID@t0&MGdE?b=xh4+;?B?$C;*=!j>zxmTR!6!0{HE{|AKlFcPRX#YAvaV&R7MCk{sxlN*jdH*&%!!>>R`AluC4{DzMnaHaL~bR-LW#Oy(b-|pABK2!w<5m6`y|-E zj`cOb5RuBPhHOXgR4r4$4~y314B&}oW1_;6N@`Kxb^dz>=7+Ud-sb&{>Nx0sPqkae zgS-K@J)!k)@P1E)7!B@VdrFPw$tBwM(7A`~Yn_at7{FJsfZ5kl!{ON$!-EvIdp_q$ zyX8aa+Gmn!K)fw77Hi5AG+kDP75A~8u_TueLs-}z`t)Hsfy3aYztQ0SB)I{LAE3;o zs(`(Uuf{|ZxX|opl6~+)bnnUL&EOYiGt+!QV=lN^ZUB-L(wJTydnYg=9ITpSFr1>U zlDdM@qBLLv1xHjv_OI52ynOa7|CY(aAh;uwTlU*^R#&Fe#Aa}->?EmHMa1J`c82vA zi0zf>Jj8ADB4Pi%Y84Mrk>zF4wWdcE;ZecsW)1v(A=+svAi;D}i!(+ki z*|*G>w)|KZF_fpOQFx>>NZwQ39F7ZaxB4zW%?>P7hCZ@?E5VGh$$Z Date: Tue, 21 May 2024 13:45:29 +0200 Subject: [PATCH 61/97] removed plugin, use only FileGZip plugin --- .../plugin/gzip/GzipCollectorPlugin.java | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/gzip/GzipCollectorPlugin.java diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/gzip/GzipCollectorPlugin.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/gzip/GzipCollectorPlugin.java deleted file mode 100644 index 44b1eeb18..000000000 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/gzip/GzipCollectorPlugin.java +++ /dev/null @@ -1,16 +0,0 @@ -package eu.dnetlib.dhp.collection.plugin.gzip; - -import eu.dnetlib.dhp.collection.ApiDescriptor; -import eu.dnetlib.dhp.collection.plugin.CollectorPlugin; -import eu.dnetlib.dhp.common.aggregation.AggregatorReport; -import eu.dnetlib.dhp.common.collection.CollectorException; - -import java.util.stream.Stream; - -public class GzipCollectorPlugin implements CollectorPlugin { - - @Override - public Stream collect(ApiDescriptor api, AggregatorReport report) throws CollectorException { - return Stream.empty(); - } -} From 834461ba26a92b98ac18e2c0206ba0cb15a2c598 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 21 May 2024 13:47:05 +0200 Subject: [PATCH 62/97] [graph provision]fixed wf definition, revised serialization of the usage counts measures --- .../utils/XmlSerializationUtils.java | 27 ++++++------------- .../dhp/oa/provision/oozie_app/workflow.xml | 2 +- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlSerializationUtils.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlSerializationUtils.java index b4d021b68..fbd647ae4 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlSerializationUtils.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlSerializationUtils.java @@ -170,30 +170,19 @@ public class XmlSerializationUtils { return sb.toString(); } - // infrastruct_::f66f1bd369679b5b077dcdf006089556||OpenAIRE + // + // public static String usageMeasureAsXmlElement(String name, Measure measure) { - HashSet dsIds = Optional - .ofNullable(measure.getUnit()) - .map( - m -> m - .stream() - .map(KeyValue::getKey) - .collect(Collectors.toCollection(HashSet::new))) - .orElse(new HashSet<>()); - StringBuilder sb = new StringBuilder(); - dsIds.forEach(dsId -> { + for (KeyValue kv : measure.getUnit()) { sb .append("<") - .append(name); - for (KeyValue kv : measure.getUnit()) { - sb.append(" ").append(attr(measure.getId(), kv.getValue())); - } - sb + .append(name) .append(" ") - .append(attr("datasource", dsId)) - .append("/>"); - }); + .append(attr(measure.getId(), kv.getValue())) + .append(attr("datasource", kv.getKey())) + .append(" />"); + } return sb.toString(); } diff --git a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml index 50acb4526..a754c7a5d 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/provision/oozie_app/workflow.xml @@ -15,8 +15,8 @@ validateXML - should the payload converter validate the XMLs false + should the payload converter validate the XMLs relPartitions From 1af4224d3dc5ab9eac7f197b58afa4e5d06af87a Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Sun, 26 May 2024 15:43:24 +0200 Subject: [PATCH 63/97] [org dedup] avoid NPEs in SparkPrepareOrgRels --- .../main/java/eu/dnetlib/dhp/oa/dedup/SparkPrepareOrgRels.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPrepareOrgRels.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPrepareOrgRels.java index 61325ab50..4fea61c18 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPrepareOrgRels.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPrepareOrgRels.java @@ -217,7 +217,7 @@ public class SparkPrepareOrgRels extends AbstractSparkAction { final Organization o = r._2()._2(); return new OrgSimRel( r._1()._1(), - o.getOriginalId().get(0), + Optional.ofNullable(o.getOriginalId()).map(oid -> oid.get(0)).orElse(null), Optional.ofNullable(o.getLegalname()).map(Field::getValue).orElse(""), Optional.ofNullable(o.getLegalshortname()).map(Field::getValue).orElse(""), Optional.ofNullable(o.getCountry()).map(Qualifier::getClassid).orElse(""), From 3a7a6ecc32fd443ed1a23b8f9ffaadee285db2c1 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Sun, 26 May 2024 16:48:11 +0200 Subject: [PATCH 64/97] [org dedup] avoid NPEs in SparkPrepareOrgRels --- .../java/eu/dnetlib/dhp/oa/dedup/SparkPrepareOrgRels.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPrepareOrgRels.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPrepareOrgRels.java index 4fea61c18..83ec7e522 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPrepareOrgRels.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPrepareOrgRels.java @@ -249,7 +249,9 @@ public class SparkPrepareOrgRels extends AbstractSparkAction { .map( (MapFunction, Tuple2>, OrgSimRel>) r -> { OrgSimRel orgSimRel = r._1()._2(); - orgSimRel.setLocal_id(r._2()._2().getOriginalId().get(0)); + orgSimRel + .setLocal_id( + Optional.ofNullable(r._2()._2().getOriginalId()).map(oid -> oid.get(0)).orElse(null)); return orgSimRel; }, Encoders.bean(OrgSimRel.class)); From 107d958b8949e756271083b94182e42667be3d3b Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Sun, 26 May 2024 21:23:30 +0200 Subject: [PATCH 65/97] [org dedup] avoid NPEs in SparkPrepareNewOrgs --- .../dnetlib/dhp/oa/dedup/SparkPrepareNewOrgs.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPrepareNewOrgs.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPrepareNewOrgs.java index d12048b02..0507b7b9a 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPrepareNewOrgs.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPrepareNewOrgs.java @@ -22,7 +22,9 @@ import eu.dnetlib.dhp.oa.dedup.model.OrgSimRel; import eu.dnetlib.dhp.schema.common.EntityType; import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.common.ModelSupport; +import eu.dnetlib.dhp.schema.oaf.Field; import eu.dnetlib.dhp.schema.oaf.Organization; +import eu.dnetlib.dhp.schema.oaf.Qualifier; import eu.dnetlib.dhp.schema.oaf.Relation; import eu.dnetlib.dhp.utils.ISLookupClientFactory; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; @@ -164,12 +166,12 @@ public class SparkPrepareNewOrgs extends AbstractSparkAction { .map( (MapFunction, Tuple2>, OrgSimRel>) r -> new OrgSimRel( "", - r._1()._2().getOriginalId().get(0), - r._1()._2().getLegalname() != null ? r._1()._2().getLegalname().getValue() : "", - r._1()._2().getLegalshortname() != null ? r._1()._2().getLegalshortname().getValue() : "", - r._1()._2().getCountry() != null ? r._1()._2().getCountry().getClassid() : "", - r._1()._2().getWebsiteurl() != null ? r._1()._2().getWebsiteurl().getValue() : "", - r._1()._2().getCollectedfrom().get(0).getValue(), + Optional.ofNullable(r._1()._2().getOriginalId()).map(oid -> oid.get(0)).orElse(null), + Optional.ofNullable(r._1()._2().getLegalname()).map(Field::getValue).orElse(""), + Optional.ofNullable(r._1()._2().getLegalshortname()).map(Field::getValue).orElse(""), + Optional.ofNullable(r._1()._2().getCountry()).map(Qualifier::getClassid).orElse(""), + Optional.ofNullable(r._1()._2().getWebsiteurl()).map(Field::getValue).orElse(""), + Optional.ofNullable(r._1()._2().getCollectedfrom()).map(cf -> cf.get(0).getValue()).orElse(null), "", structuredPropertyListToString(r._1()._2().getPid()), parseECField(r._1()._2().getEclegalbody()), From b55fed09f8a0e88b432b93da9d265cfd9533ae59 Mon Sep 17 00:00:00 2001 From: Miriam Baglioni Date: Fri, 24 May 2024 12:28:24 +0200 Subject: [PATCH 66/97] Update to include a blackList that filters out the results we know are wrongly associated to IE --- .../CreateActionSetFromWebEntries.java | 56 ++++++++++++------- .../actionmanager/webcrawl/as_parameters.json | 7 ++- .../actionmanager/webcrawl/CreateASTest.java | 28 +++++++--- .../webcrawl/{ => input}/part-00000 | 0 .../webcrawl/{ => input}/part-00001 | 0 .../webcrawl/{ => input}/part-00002 | 0 6 files changed, 62 insertions(+), 29 deletions(-) rename dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/webcrawl/{ => input}/part-00000 (100%) rename dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/webcrawl/{ => input}/part-00001 (100%) rename dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/webcrawl/{ => input}/part-00002 (100%) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateActionSetFromWebEntries.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateActionSetFromWebEntries.java index eb370e981..541ed8e10 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateActionSetFromWebEntries.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateActionSetFromWebEntries.java @@ -12,6 +12,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.io.compress.GzipCodec; import org.apache.hadoop.mapred.SequenceFileOutputFormat; import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.FilterFunction; import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.sql.*; import org.apache.spark.sql.types.StructType; @@ -70,6 +71,9 @@ public class CreateActionSetFromWebEntries implements Serializable { final String outputPath = parser.get("outputPath"); log.info("outputPath: {}", outputPath); + final String blackListInputPath = parser.get("blackListPath"); + log.info("blackListInputPath: {}", blackListInputPath); + SparkConf conf = new SparkConf(); runWithSparkSession( @@ -77,35 +81,40 @@ public class CreateActionSetFromWebEntries implements Serializable { isSparkSessionManaged, spark -> { - createActionSet(spark, inputPath, outputPath); + createActionSet(spark, inputPath, outputPath, blackListInputPath); }); } public static void createActionSet(SparkSession spark, String inputPath, - String outputPath) { + String outputPath, String blackListInputPath) { final Dataset dataset = readWebCrawl(spark, inputPath) - .filter("publication_year <= 2020 or country_code=='IE'") + .filter("country_code=='IE'") .drop("publication_year"); - dataset.flatMap((FlatMapFunction) row -> { - List ret = new ArrayList<>(); - final String ror = ROR_PREFIX - + IdentifierFactory.md5(PidCleaner.normalizePidValue("ROR", row.getAs("ror"))); - ret.addAll(createAffiliationRelationPairDOI(row.getAs("doi"), ror)); - ret.addAll(createAffiliationRelationPairPMID(row.getAs("pmid"), ror)); - ret.addAll(createAffiliationRelationPairPMCID(row.getAs("pmcid"), ror)); + final Dataset blackList = readBlackList(spark, blackListInputPath); - return ret - .iterator(); - }, Encoders.bean(Relation.class)) - .toJavaRDD() - .map(p -> new AtomicAction(p.getClass(), p)) - .mapToPair( - aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()), - new Text(OBJECT_MAPPER.writeValueAsString(aa)))) - .saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, GzipCodec.class); + dataset.join(blackList, dataset.col("id").equalTo(blackList.col("OpenAlexId")), "left") + .filter((FilterFunction) r -> r.getAs("OpenAlexId") == null) + .drop("OpenAlexId") + .flatMap((FlatMapFunction) row -> { + List ret = new ArrayList<>(); + final String ror = ROR_PREFIX + + IdentifierFactory.md5(PidCleaner.normalizePidValue("ROR", row.getAs("ror"))); + ret.addAll(createAffiliationRelationPairDOI(row.getAs("doi"), ror)); + ret.addAll(createAffiliationRelationPairPMID(row.getAs("pmid"), ror)); + ret.addAll(createAffiliationRelationPairPMCID(row.getAs("pmcid"), ror)); + + return ret + .iterator(); + }, Encoders.bean(Relation.class)) + .toJavaRDD() + .map(p -> new AtomicAction(p.getClass(), p)) + .mapToPair( + aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()), + new Text(OBJECT_MAPPER.writeValueAsString(aa)))) + .saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class);//, GzipCodec.class); } @@ -136,6 +145,15 @@ public class CreateActionSetFromWebEntries implements Serializable { } + private static Dataset readBlackList(SparkSession spark, String inputPath){ + + return spark + .read() + .option("header", true) + .csv(inputPath) + .select("OpenAlexId"); + } + private static List createAffiliationRelationPairPMCID(String pmcid, String ror) { if (pmcid == null) return new ArrayList<>(); diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/as_parameters.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/as_parameters.json index 3f056edf7..b79140b3a 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/as_parameters.json +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/as_parameters.json @@ -16,5 +16,10 @@ "paramLongName": "isSparkSessionManaged", "paramDescription": "the hdfs name node", "paramRequired": false - } + },{ + "paramName": "bl", + "paramLongName": "blackListPath", + "paramDescription": "the working path", + "paramRequired": true +} ] diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateASTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateASTest.java index 402f07d4d..c574a5812 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateASTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateASTest.java @@ -75,8 +75,12 @@ public class CreateASTest { String inputPath = getClass() .getResource( - "/eu/dnetlib/dhp/actionmanager/webcrawl/") + "/eu/dnetlib/dhp/actionmanager/webcrawl/input/") .getPath(); + String blackListPath = getClass() + .getResource( + "/eu/dnetlib/dhp/actionmanager/webcrawl/blackList/") + .getPath(); CreateActionSetFromWebEntries .main( @@ -86,7 +90,8 @@ public class CreateASTest { "-sourcePath", inputPath, "-outputPath", - workingDir.toString() + "/actionSet1" + workingDir.toString() + "/actionSet1", + "-blackListPath", blackListPath }); final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); @@ -96,7 +101,7 @@ public class CreateASTest { .map(value -> OBJECT_MAPPER.readValue(value._2().toString(), AtomicAction.class)) .map(aa -> ((Relation) aa.getPayload())); - Assertions.assertEquals(64, tmp.count()); + Assertions.assertEquals(58, tmp.count()); } @@ -109,6 +114,10 @@ public class CreateASTest { .getResource( "/eu/dnetlib/dhp/actionmanager/webcrawl/") .getPath(); + String blackListPath = getClass() + .getResource( + "/eu/dnetlib/dhp/actionmanager/webcrawl/blackList/") + .getPath(); CreateActionSetFromWebEntries .main( @@ -118,7 +127,8 @@ public class CreateASTest { "-sourcePath", inputPath, "-outputPath", - workingDir.toString() + "/actionSet1" + workingDir.toString() + "/actionSet1", + "-blackListPath", blackListPath }); final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); @@ -184,7 +194,7 @@ public class CreateASTest { Assertions .assertEquals( - 5, tmp + 2, tmp .filter( r -> r .getSource() @@ -197,7 +207,7 @@ public class CreateASTest { Assertions .assertEquals( - 5, tmp + 2, tmp .filter( r -> r .getTarget() @@ -210,7 +220,7 @@ public class CreateASTest { Assertions .assertEquals( - 2, tmp + 1, tmp .filter( r -> r .getTarget() @@ -224,7 +234,7 @@ public class CreateASTest { Assertions .assertEquals( - 2, tmp + 1, tmp .filter( r -> r .getTarget() @@ -238,7 +248,7 @@ public class CreateASTest { Assertions .assertEquals( - 1, tmp + 0, tmp .filter( r -> r .getTarget() diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/webcrawl/part-00000 b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/webcrawl/input/part-00000 similarity index 100% rename from dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/webcrawl/part-00000 rename to dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/webcrawl/input/part-00000 diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/webcrawl/part-00001 b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/webcrawl/input/part-00001 similarity index 100% rename from dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/webcrawl/part-00001 rename to dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/webcrawl/input/part-00001 diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/webcrawl/part-00002 b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/webcrawl/input/part-00002 similarity index 100% rename from dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/webcrawl/part-00002 rename to dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/webcrawl/input/part-00002 From 87c9c61b414657777eb5d65323851d4ec70bd18d Mon Sep 17 00:00:00 2001 From: Miriam Baglioni Date: Fri, 24 May 2024 15:23:42 +0200 Subject: [PATCH 67/97] Update to include a blackList that filters out the results we know are wrongly associated to IE - refactoring --- .../CreateActionSetFromWebEntries.java | 49 ++++++++++--------- .../actionmanager/webcrawl/CreateASTest.java | 16 +++--- 2 files changed, 33 insertions(+), 32 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateActionSetFromWebEntries.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateActionSetFromWebEntries.java index 541ed8e10..27970f2c3 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateActionSetFromWebEntries.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateActionSetFromWebEntries.java @@ -95,26 +95,27 @@ public class CreateActionSetFromWebEntries implements Serializable { final Dataset blackList = readBlackList(spark, blackListInputPath); - dataset.join(blackList, dataset.col("id").equalTo(blackList.col("OpenAlexId")), "left") - .filter((FilterFunction) r -> r.getAs("OpenAlexId") == null) - .drop("OpenAlexId") - .flatMap((FlatMapFunction) row -> { - List ret = new ArrayList<>(); - final String ror = ROR_PREFIX - + IdentifierFactory.md5(PidCleaner.normalizePidValue("ROR", row.getAs("ror"))); - ret.addAll(createAffiliationRelationPairDOI(row.getAs("doi"), ror)); - ret.addAll(createAffiliationRelationPairPMID(row.getAs("pmid"), ror)); - ret.addAll(createAffiliationRelationPairPMCID(row.getAs("pmcid"), ror)); + dataset + .join(blackList, dataset.col("id").equalTo(blackList.col("OpenAlexId")), "left") + .filter((FilterFunction) r -> r.getAs("OpenAlexId") == null) + .drop("OpenAlexId") + .flatMap((FlatMapFunction) row -> { + List ret = new ArrayList<>(); + final String ror = ROR_PREFIX + + IdentifierFactory.md5(PidCleaner.normalizePidValue("ROR", row.getAs("ror"))); + ret.addAll(createAffiliationRelationPairDOI(row.getAs("doi"), ror)); + ret.addAll(createAffiliationRelationPairPMID(row.getAs("pmid"), ror)); + ret.addAll(createAffiliationRelationPairPMCID(row.getAs("pmcid"), ror)); - return ret - .iterator(); - }, Encoders.bean(Relation.class)) - .toJavaRDD() - .map(p -> new AtomicAction(p.getClass(), p)) - .mapToPair( - aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()), - new Text(OBJECT_MAPPER.writeValueAsString(aa)))) - .saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class);//, GzipCodec.class); + return ret + .iterator(); + }, Encoders.bean(Relation.class)) + .toJavaRDD() + .map(p -> new AtomicAction(p.getClass(), p)) + .mapToPair( + aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()), + new Text(OBJECT_MAPPER.writeValueAsString(aa)))) + .saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, GzipCodec.class); } @@ -145,13 +146,13 @@ public class CreateActionSetFromWebEntries implements Serializable { } - private static Dataset readBlackList(SparkSession spark, String inputPath){ + private static Dataset readBlackList(SparkSession spark, String inputPath) { return spark - .read() - .option("header", true) - .csv(inputPath) - .select("OpenAlexId"); + .read() + .option("header", true) + .csv(inputPath) + .select("OpenAlexId"); } private static List createAffiliationRelationPairPMCID(String pmcid, String ror) { diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateASTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateASTest.java index c574a5812..e9291f93c 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateASTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/webcrawl/CreateASTest.java @@ -78,9 +78,9 @@ public class CreateASTest { "/eu/dnetlib/dhp/actionmanager/webcrawl/input/") .getPath(); String blackListPath = getClass() - .getResource( - "/eu/dnetlib/dhp/actionmanager/webcrawl/blackList/") - .getPath(); + .getResource( + "/eu/dnetlib/dhp/actionmanager/webcrawl/blackList/") + .getPath(); CreateActionSetFromWebEntries .main( @@ -91,7 +91,7 @@ public class CreateASTest { inputPath, "-outputPath", workingDir.toString() + "/actionSet1", - "-blackListPath", blackListPath + "-blackListPath", blackListPath }); final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); @@ -115,9 +115,9 @@ public class CreateASTest { "/eu/dnetlib/dhp/actionmanager/webcrawl/") .getPath(); String blackListPath = getClass() - .getResource( - "/eu/dnetlib/dhp/actionmanager/webcrawl/blackList/") - .getPath(); + .getResource( + "/eu/dnetlib/dhp/actionmanager/webcrawl/blackList/") + .getPath(); CreateActionSetFromWebEntries .main( @@ -128,7 +128,7 @@ public class CreateASTest { inputPath, "-outputPath", workingDir.toString() + "/actionSet1", - "-blackListPath", blackListPath + "-blackListPath", blackListPath }); final JavaSparkContext sc = new JavaSparkContext(spark.sparkContext()); From 75d5ddb999fe9a061b8211f828ffa35e6b8c763d Mon Sep 17 00:00:00 2001 From: Miriam Baglioni Date: Fri, 24 May 2024 16:01:19 +0200 Subject: [PATCH 68/97] Update to include a blackList that filters out the results we know are wrongly associated to IE - update workflow definition - the blacklist parameter --- .../eu/dnetlib/dhp/actionmanager/webcrawl/job.properties | 1 + .../eu/dnetlib/dhp/actionmanager/webcrawl/oozie_app/workflow.xml | 1 + 2 files changed, 2 insertions(+) diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/job.properties b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/job.properties index f616baea7..d7bd709fc 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/job.properties +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/job.properties @@ -1,2 +1,3 @@ sourcePath=/user/miriam.baglioni/openalex-snapshot/data/works/ outputPath=/tmp/miriam/webcrawlComplete/ +blackListPath=/user/miriam.baglioni/openalex-blackList diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/oozie_app/workflow.xml index 653a7d384..b9394c7e6 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/webcrawl/oozie_app/workflow.xml @@ -45,6 +45,7 @@ --sourcePath${sourcePath} --outputPath${outputPath} + --blackListPath${blackListPath} From 73316d8c8353408670ef07c837cb626fdc508e90 Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Tue, 28 May 2024 14:14:51 +0200 Subject: [PATCH 69/97] Add jaxb and jaxws dependencies when compiling with spark-34 profile as they are required to run with jdk > 8 --- dhp-common/pom.xml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/dhp-common/pom.xml b/dhp-common/pom.xml index 2c7a0ef8c..bfec019af 100644 --- a/dhp-common/pom.xml +++ b/dhp-common/pom.xml @@ -169,4 +169,23 @@ + + + + spark-34 + + + javax.xml.bind + jaxb-api + 2.2.11 + + + com.sun.xml.ws + jaxws-ri + 2.3.3 + pom + + + + From e3f28338c147571f54c81fa9996b0c03f8f95455 Mon Sep 17 00:00:00 2001 From: LSmyrnaios Date: Tue, 28 May 2024 17:51:45 +0300 Subject: [PATCH 70/97] Miscellaneous updates to the copying operation to Impala Cluster: - Assign the WRITE and EXECUTE permissions to the DBs' HDFS-directories, in order to be able to create tables on top of them, in the Impala Cluster. - Make sure the "copydb" function returns early, when it encounters a fatal error, while respecting the "SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR" config. --- .../oozie_app/copyDataToImpalaCluster.sh | 31 +++++++++++++------ .../oozie_app/copyDataToImpalaCluster.sh | 31 +++++++++++++------ .../oozie_app/copyDataToImpalaCluster.sh | 31 +++++++++++++------ .../oozie_app/copyDataToImpalaCluster.sh | 31 +++++++++++++------ 4 files changed, 88 insertions(+), 36 deletions(-) diff --git a/dhp-workflows/dhp-stats-hist-snaps/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-hist-snaps/oozie_app/copyDataToImpalaCluster.sh b/dhp-workflows/dhp-stats-hist-snaps/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-hist-snaps/oozie_app/copyDataToImpalaCluster.sh index 26760d650..ca0f7a643 100644 --- a/dhp-workflows/dhp-stats-hist-snaps/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-hist-snaps/oozie_app/copyDataToImpalaCluster.sh +++ b/dhp-workflows/dhp-stats-hist-snaps/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-hist-snaps/oozie_app/copyDataToImpalaCluster.sh @@ -72,6 +72,8 @@ function copydb() { rm -f error.log if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then exit 2 + else + return 2 fi fi @@ -90,19 +92,30 @@ function copydb() { -pb \ ${OCEAN_HDFS_NODE}/user/hive/warehouse/${db}.db ${IMPALA_HDFS_DB_BASE_PATH} - # Check the exit status of the "hadoop distcp" command. - if [ $? -eq 0 ]; then - echo -e "\nSuccessfully copied the files of '${db}'.\n" + if [ $? -eq 0 ]; then # Check the exit status of the "hadoop distcp" command. + echo -e "\nSuccessfully copied the files of '${db}' from Ocean to Impala cluster.\n" else echo -e "\n\nERROR: FAILED TO TRANSFER THE FILES OF '${db}', WITH 'hadoop distcp'. GOT EXIT STATUS: $?\n\n" rm -f error.log if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then exit 3 + else + return 3 fi fi - # In case we ever use this script for a writable DB (using inserts/updates), we should perform the following costly operation as well.. - #hdfs dfs -conf ${IMPALA_CONFIG_FILE} -chmod -R 777 ${TEMP_SUBDIR_FULLPATH}/${db}.db + # Give WRITE and EXECUTE permissions to the DBs' directory only, in order to be able to create more tables later, on top of that DB. + hdfs dfs -conf ${IMPALA_CONFIG_FILE} -chmod u+wx ${IMPALA_HDFS_DB_BASE_PATH}/${db}.db + # In case we ever use this script for a writable DB (using inserts/updates), we should perform the costly recursive operation as well, using the "-R" param. + if [ $? -ne 0 ]; then # Check the exit status.. + echo -e "\n\nERROR: FAILED TO ASSIGN WRITE AND EXECUTE PERMISSIONS TO THE DIRECTORY OF DB: '${db}'. GOT EXIT STATUS: $?\n\n" + rm -f error.log + if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then + exit 4 + else + return 4 + fi + fi echo -e "\nCreating schema for db: '${db}'\n" @@ -131,7 +144,7 @@ function copydb() { if [ -z "$CURRENT_PRQ_FILE" ]; then # If there is not parquet-file inside. echo -e "\nERROR: THE TABLE \"${i}\" HAD NO FILES TO GET THE SCHEMA FROM! IT'S EMPTY!\n\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then - exit 4 + exit 5 fi else impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "create table ${db}.${i} like parquet '${CURRENT_PRQ_FILE}' stored as parquet;" |& tee error.log @@ -139,7 +152,7 @@ function copydb() { if [ -n "$log_errors" ]; then echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN CREATING TABLE '${i}'!\n\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then - exit 5 + exit 6 fi fi fi @@ -185,7 +198,7 @@ function copydb() { if [[ $new_num_of_views_to_retry -eq $previous_num_of_views_to_retry ]]; then echo -e "\n\nERROR: THE NUMBER OF VIEWS TO RETRY HAS NOT BEEN REDUCED! THE SCRIPT IS LIKELY GOING TO AN INFINITE-LOOP! EXITING..\n\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then - exit 6 + exit 7 fi elif [[ $new_num_of_views_to_retry -gt 0 ]]; then echo -e "\nTo be retried \"create_view_statements\" (${new_num_of_views_to_retry}):\n\n${all_create_view_statements[@]}\n" @@ -215,7 +228,7 @@ function copydb() { echo -e "\n\nERROR: 1 OR MORE ENTITIES OF DB '${db}' FAILED TO BE COPIED TO IMPALA CLUSTER!\n\n" rm -f error.log if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then - exit 7 + exit 8 fi fi diff --git a/dhp-workflows/dhp-stats-monitor-irish/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor-irish/oozie_app/copyDataToImpalaCluster.sh b/dhp-workflows/dhp-stats-monitor-irish/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor-irish/oozie_app/copyDataToImpalaCluster.sh index 26760d650..ca0f7a643 100644 --- a/dhp-workflows/dhp-stats-monitor-irish/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor-irish/oozie_app/copyDataToImpalaCluster.sh +++ b/dhp-workflows/dhp-stats-monitor-irish/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor-irish/oozie_app/copyDataToImpalaCluster.sh @@ -72,6 +72,8 @@ function copydb() { rm -f error.log if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then exit 2 + else + return 2 fi fi @@ -90,19 +92,30 @@ function copydb() { -pb \ ${OCEAN_HDFS_NODE}/user/hive/warehouse/${db}.db ${IMPALA_HDFS_DB_BASE_PATH} - # Check the exit status of the "hadoop distcp" command. - if [ $? -eq 0 ]; then - echo -e "\nSuccessfully copied the files of '${db}'.\n" + if [ $? -eq 0 ]; then # Check the exit status of the "hadoop distcp" command. + echo -e "\nSuccessfully copied the files of '${db}' from Ocean to Impala cluster.\n" else echo -e "\n\nERROR: FAILED TO TRANSFER THE FILES OF '${db}', WITH 'hadoop distcp'. GOT EXIT STATUS: $?\n\n" rm -f error.log if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then exit 3 + else + return 3 fi fi - # In case we ever use this script for a writable DB (using inserts/updates), we should perform the following costly operation as well.. - #hdfs dfs -conf ${IMPALA_CONFIG_FILE} -chmod -R 777 ${TEMP_SUBDIR_FULLPATH}/${db}.db + # Give WRITE and EXECUTE permissions to the DBs' directory only, in order to be able to create more tables later, on top of that DB. + hdfs dfs -conf ${IMPALA_CONFIG_FILE} -chmod u+wx ${IMPALA_HDFS_DB_BASE_PATH}/${db}.db + # In case we ever use this script for a writable DB (using inserts/updates), we should perform the costly recursive operation as well, using the "-R" param. + if [ $? -ne 0 ]; then # Check the exit status.. + echo -e "\n\nERROR: FAILED TO ASSIGN WRITE AND EXECUTE PERMISSIONS TO THE DIRECTORY OF DB: '${db}'. GOT EXIT STATUS: $?\n\n" + rm -f error.log + if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then + exit 4 + else + return 4 + fi + fi echo -e "\nCreating schema for db: '${db}'\n" @@ -131,7 +144,7 @@ function copydb() { if [ -z "$CURRENT_PRQ_FILE" ]; then # If there is not parquet-file inside. echo -e "\nERROR: THE TABLE \"${i}\" HAD NO FILES TO GET THE SCHEMA FROM! IT'S EMPTY!\n\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then - exit 4 + exit 5 fi else impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "create table ${db}.${i} like parquet '${CURRENT_PRQ_FILE}' stored as parquet;" |& tee error.log @@ -139,7 +152,7 @@ function copydb() { if [ -n "$log_errors" ]; then echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN CREATING TABLE '${i}'!\n\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then - exit 5 + exit 6 fi fi fi @@ -185,7 +198,7 @@ function copydb() { if [[ $new_num_of_views_to_retry -eq $previous_num_of_views_to_retry ]]; then echo -e "\n\nERROR: THE NUMBER OF VIEWS TO RETRY HAS NOT BEEN REDUCED! THE SCRIPT IS LIKELY GOING TO AN INFINITE-LOOP! EXITING..\n\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then - exit 6 + exit 7 fi elif [[ $new_num_of_views_to_retry -gt 0 ]]; then echo -e "\nTo be retried \"create_view_statements\" (${new_num_of_views_to_retry}):\n\n${all_create_view_statements[@]}\n" @@ -215,7 +228,7 @@ function copydb() { echo -e "\n\nERROR: 1 OR MORE ENTITIES OF DB '${db}' FAILED TO BE COPIED TO IMPALA CLUSTER!\n\n" rm -f error.log if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then - exit 7 + exit 8 fi fi diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh index 1ab3e417a..dd2203eef 100644 --- a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh +++ b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh @@ -72,6 +72,8 @@ function copydb() { rm -f error.log if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then exit 2 + else + return 2 fi fi @@ -90,19 +92,30 @@ function copydb() { -pb \ ${OCEAN_HDFS_NODE}/user/hive/warehouse/${db}.db ${IMPALA_HDFS_DB_BASE_PATH} - # Check the exit status of the "hadoop distcp" command. - if [ $? -eq 0 ]; then - echo -e "\nSuccessfully copied the files of '${db}'.\n" + if [ $? -eq 0 ]; then # Check the exit status of the "hadoop distcp" command. + echo -e "\nSuccessfully copied the files of '${db}' from Ocean to Impala cluster.\n" else echo -e "\n\nERROR: FAILED TO TRANSFER THE FILES OF '${db}', WITH 'hadoop distcp'. GOT EXIT STATUS: $?\n\n" rm -f error.log if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then exit 3 + else + return 3 fi fi - # In case we ever use this script for a writable DB (using inserts/updates), we should perform the following costly operation as well.. - #hdfs dfs -conf ${IMPALA_CONFIG_FILE} -chmod -R 777 ${TEMP_SUBDIR_FULLPATH}/${db}.db + # Give WRITE and EXECUTE permissions to the DBs' directory only, in order to be able to create more tables later, on top of that DB. + hdfs dfs -conf ${IMPALA_CONFIG_FILE} -chmod u+wx ${IMPALA_HDFS_DB_BASE_PATH}/${db}.db + # In case we ever use this script for a writable DB (using inserts/updates), we should perform the costly recursive operation as well, using the "-R" param. + if [ $? -ne 0 ]; then # Check the exit status.. + echo -e "\n\nERROR: FAILED TO ASSIGN WRITE AND EXECUTE PERMISSIONS TO THE DIRECTORY OF DB: '${db}'. GOT EXIT STATUS: $?\n\n" + rm -f error.log + if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then + exit 4 + else + return 4 + fi + fi echo -e "\nCreating schema for db: '${db}'\n" @@ -131,7 +144,7 @@ function copydb() { if [ -z "$CURRENT_PRQ_FILE" ]; then # If there is not parquet-file inside. echo -e "\nERROR: THE TABLE \"${i}\" HAD NO FILES TO GET THE SCHEMA FROM! IT'S EMPTY!\n\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then - exit 4 + exit 5 fi else impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "create table ${db}.${i} like parquet '${CURRENT_PRQ_FILE}' stored as parquet;" |& tee error.log @@ -139,7 +152,7 @@ function copydb() { if [ -n "$log_errors" ]; then echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN CREATING TABLE '${i}'!\n\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then - exit 5 + exit 6 fi fi fi @@ -185,7 +198,7 @@ function copydb() { if [[ $new_num_of_views_to_retry -eq $previous_num_of_views_to_retry ]]; then echo -e "\n\nERROR: THE NUMBER OF VIEWS TO RETRY HAS NOT BEEN REDUCED! THE SCRIPT IS LIKELY GOING TO AN INFINITE-LOOP! EXITING..\n\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then - exit 6 + exit 7 fi elif [[ $new_num_of_views_to_retry -gt 0 ]]; then echo -e "\nTo be retried \"create_view_statements\" (${new_num_of_views_to_retry}):\n\n${all_create_view_statements[@]}\n" @@ -215,7 +228,7 @@ function copydb() { echo -e "\n\nERROR: 1 OR MORE ENTITIES OF DB '${db}' FAILED TO BE COPIED TO IMPALA CLUSTER!\n\n" rm -f error.log if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then - exit 7 + exit 8 fi fi diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh index 7957a659c..918775f49 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh @@ -74,6 +74,8 @@ function copydb() { rm -f error.log if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then exit 2 + else + return 2 fi fi @@ -92,19 +94,30 @@ function copydb() { -pb \ ${OCEAN_HDFS_NODE}/user/hive/warehouse/${db}.db ${IMPALA_HDFS_DB_BASE_PATH} - # Check the exit status of the "hadoop distcp" command. - if [ $? -eq 0 ]; then - echo -e "\nSuccessfully copied the files of '${db}'.\n" + if [ $? -eq 0 ]; then # Check the exit status of the "hadoop distcp" command. + echo -e "\nSuccessfully copied the files of '${db}' from Ocean to Impala cluster.\n" else echo -e "\n\nERROR: FAILED TO TRANSFER THE FILES OF '${db}', WITH 'hadoop distcp'. GOT EXIT STATUS: $?\n\n" rm -f error.log if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then exit 3 + else + return 3 fi fi - # In case we ever use this script for a writable DB (using inserts/updates), we should perform the following costly operation as well.. - #hdfs dfs -conf ${IMPALA_CONFIG_FILE} -chmod -R 777 ${TEMP_SUBDIR_FULLPATH}/${db}.db + # Give WRITE and EXECUTE permissions to the DBs' directory only, in order to be able to create more tables later, on top of that DB. + hdfs dfs -conf ${IMPALA_CONFIG_FILE} -chmod u+wx ${IMPALA_HDFS_DB_BASE_PATH}/${db}.db + # In case we ever use this script for a writable DB (using inserts/updates), we should perform the costly recursive operation as well, using the "-R" param. + if [ $? -ne 0 ]; then # Check the exit status.. + echo -e "\n\nERROR: FAILED TO ASSIGN WRITE AND EXECUTE PERMISSIONS TO THE DIRECTORY OF DB: '${db}'. GOT EXIT STATUS: $?\n\n" + rm -f error.log + if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then + exit 4 + else + return 4 + fi + fi echo -e "\nCreating schema for db: '${db}'\n" @@ -133,7 +146,7 @@ function copydb() { if [ -z "$CURRENT_PRQ_FILE" ]; then # If there is not parquet-file inside. echo -e "\nERROR: THE TABLE \"${i}\" HAD NO FILES TO GET THE SCHEMA FROM! IT'S EMPTY!\n\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then - exit 4 + exit 5 fi else impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "create table ${db}.${i} like parquet '${CURRENT_PRQ_FILE}' stored as parquet;" |& tee error.log @@ -141,7 +154,7 @@ function copydb() { if [ -n "$log_errors" ]; then echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN CREATING TABLE '${i}'!\n\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then - exit 5 + exit 6 fi fi fi @@ -187,7 +200,7 @@ function copydb() { if [[ $new_num_of_views_to_retry -eq $previous_num_of_views_to_retry ]]; then echo -e "\n\nERROR: THE NUMBER OF VIEWS TO RETRY HAS NOT BEEN REDUCED! THE SCRIPT IS LIKELY GOING TO AN INFINITE-LOOP! EXITING..\n\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then - exit 6 + exit 7 fi elif [[ $new_num_of_views_to_retry -gt 0 ]]; then echo -e "\nTo be retried \"create_view_statements\" (${new_num_of_views_to_retry}):\n\n${all_create_view_statements[@]}\n" @@ -217,7 +230,7 @@ function copydb() { echo -e "\n\nERROR: 1 OR MORE ENTITIES OF DB '${db}' FAILED TO BE COPIED TO IMPALA CLUSTER!\n\n" rm -f error.log if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then - exit 7 + exit 8 fi fi From 5d85b70e1fbd724ab34648e3ace3282ce40bae58 Mon Sep 17 00:00:00 2001 From: Miriam Baglioni Date: Wed, 29 May 2024 11:55:00 +0200 Subject: [PATCH 71/97] [NOAMI] removed Ireland funder id 501100011103. ticket 9635 --- .../eu/dnetlib/dhp/collection/crossref/irish_funder.json | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/crossref/irish_funder.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/crossref/irish_funder.json index f0275e06b..e4f491e5c 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/crossref/irish_funder.json +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/crossref/irish_funder.json @@ -625,12 +625,6 @@ "name": "Alimentary Health", "synonym": [] }, - { - "id": "501100011103", - "uri": "http://dx.doi.org/10.13039/501100011103", - "name": "Rann\u00eds", - "synonym": [] - }, { "id": "501100012354", "uri": "http://dx.doi.org/10.13039/501100012354", From e996787be232c0a4f214712d6fffc0884ab2c400 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Wed, 29 May 2024 15:05:17 +0200 Subject: [PATCH 72/97] OSF test --- .../collection/plugin/rest/OsfPreprintCollectorTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/OsfPreprintCollectorTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/OsfPreprintCollectorTest.java index 90f4c7f25..0e64f8bab 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/OsfPreprintCollectorTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/OsfPreprintCollectorTest.java @@ -39,8 +39,8 @@ public class OsfPreprintCollectorTest { private final String resumptionType = "page"; private final String resumptionXpath = "/*/*[local-name()='links']/*[local-name()='next']"; - private final String resultSizeParam = ""; - private final String resultSizeValue = ""; + private final String resultSizeParam = "page[size]"; + private final String resultSizeValue = "100"; private final String resultFormatParam = "format"; private final String resultFormatValue = "json"; @@ -74,7 +74,7 @@ public class OsfPreprintCollectorTest { final AtomicInteger i = new AtomicInteger(0); final Stream stream = this.rcp.collect(this.api, new AggregatorReport()); - stream.limit(200).forEach(s -> { + stream.limit(2000).forEach(s -> { Assertions.assertTrue(s.length() > 0); i.incrementAndGet(); log.info(s); From 1b165a14a09394adda40aca0eb3df238d471a448 Mon Sep 17 00:00:00 2001 From: Alessia Date: Wed, 29 May 2024 15:41:36 +0200 Subject: [PATCH 73/97] Rest collector plugin on hadoop supports a new param to pass request headers --- .../plugin/rest/RestCollectorPlugin.java | 11 ++-- .../collection/plugin/rest/RestIterator.java | 62 +++++++++++++------ .../plugin/rest/RestCollectorPluginTest.java | 42 ++++++++++--- .../plugin/rest/RestIteratorTest.java | 2 +- 4 files changed, 83 insertions(+), 34 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestCollectorPlugin.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestCollectorPlugin.java index 997948687..8445e49e0 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestCollectorPlugin.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestCollectorPlugin.java @@ -1,12 +1,14 @@ package eu.dnetlib.dhp.collection.plugin.rest; +import java.util.Map; import java.util.Optional; import java.util.Spliterator; import java.util.Spliterators; import java.util.stream.Stream; import java.util.stream.StreamSupport; +import com.google.gson.Gson; import org.apache.commons.lang3.StringUtils; import eu.dnetlib.dhp.collection.ApiDescriptor; @@ -47,6 +49,9 @@ public class RestCollectorPlugin implements CollectorPlugin { final String entityXpath = api.getParams().get("entityXpath"); final String authMethod = api.getParams().get("authMethod"); final String authToken = api.getParams().get("authToken"); + final String requestHeaderMap = api.getParams().get("requestHeaderMap"); + Gson gson = new Gson(); + Map requestHeaders = gson.fromJson(requestHeaderMap, Map.class); final String resultSizeValue = Optional .ofNullable(api.getParams().get("resultSizeValue")) .filter(StringUtils::isNotBlank) @@ -64,9 +69,6 @@ public class RestCollectorPlugin implements CollectorPlugin { if (StringUtils.isBlank(resultFormatValue)) { throw new CollectorException("Param 'resultFormatValue' is null or empty"); } - if (StringUtils.isBlank(queryParams)) { - throw new CollectorException("Param 'queryParams' is null or empty"); - } if (StringUtils.isBlank(entityXpath)) { throw new CollectorException("Param 'entityXpath' is null or empty"); } @@ -92,7 +94,8 @@ public class RestCollectorPlugin implements CollectorPlugin { entityXpath, authMethod, authToken, - resultOutputFormat); + resultOutputFormat, + requestHeaders); return StreamSupport .stream( diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java index 76af6cff1..e51c9eb1b 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java @@ -9,6 +9,7 @@ import java.net.URL; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.Iterator; +import java.util.Map; import java.util.Queue; import java.util.concurrent.PriorityBlockingQueue; @@ -24,6 +25,7 @@ import javax.xml.xpath.XPathExpression; import javax.xml.xpath.XPathExpressionException; import javax.xml.xpath.XPathFactory; +import com.google.common.collect.Maps; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpHeaders; @@ -49,13 +51,14 @@ import eu.dnetlib.dhp.common.collection.HttpClientParams; */ public class RestIterator implements Iterator { + private static final Logger log = LoggerFactory.getLogger(RestIterator.class); public static final String UTF_8 = "UTF-8"; private static final int MAX_ATTEMPTS = 5; private final HttpClientParams clientParams; - private final String BASIC = "basic"; + private final String AUTHBASIC = "basic"; private final String baseUrl; private final String resumptionType; @@ -89,6 +92,12 @@ public class RestIterator implements Iterator { */ private final String resultOutputFormat; + /* + Can be used to set additional request headers, like for content negotiation + */ + private Map requestHeaders; + + /** * RestIterator class compatible to version 1.3.33 */ @@ -107,7 +116,8 @@ public class RestIterator implements Iterator { final String entityXpath, final String authMethod, final String authToken, - final String resultOutputFormat) { + final String resultOutputFormat, + final Map requestHeaders) { this.clientParams = clientParams; this.baseUrl = baseUrl; @@ -119,6 +129,7 @@ public class RestIterator implements Iterator { this.authMethod = authMethod; this.authToken = authToken; this.resultOutputFormat = resultOutputFormat; + this.requestHeaders = requestHeaders != null ? requestHeaders : Maps.newHashMap(); this.queryFormat = StringUtils.isNotBlank(resultFormatParam) ? "&" + resultFormatParam + "=" + resultFormatValue : ""; @@ -231,25 +242,20 @@ public class RestIterator implements Iterator { final URL qUrl = new URL(query); log.debug("authMethod: {}", this.authMethod); - if ("bearer".equalsIgnoreCase(this.authMethod)) { - log.trace("authMethod before inputStream: {}", resultXml); - final HttpURLConnection conn = (HttpURLConnection) qUrl.openConnection(); - conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + this.authToken); - conn.setRequestProperty(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.getMimeType()); - conn.setRequestMethod("GET"); - theHttpInputStream = conn.getInputStream(); - } else if (this.BASIC.equalsIgnoreCase(this.authMethod)) { - log.trace("authMethod before inputStream: {}", resultXml); - final HttpURLConnection conn = (HttpURLConnection) qUrl.openConnection(); - conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Basic " + this.authToken); - conn.setRequestProperty(HttpHeaders.ACCEPT, ContentType.APPLICATION_XML.getMimeType()); - conn.setRequestMethod("GET"); - theHttpInputStream = conn.getInputStream(); - } else { - theHttpInputStream = qUrl.openStream(); + if (this.authMethod == "bearer") { + log.trace("RestIterator.downloadPage():: authMethod before inputStream: " + resultXml); + requestHeaders.put("Authorization", "Bearer " + authToken); + //requestHeaders.put("Content-Type", "application/json"); + } else if (AUTHBASIC.equalsIgnoreCase(this.authMethod)) { + log.trace("RestIterator.downloadPage():: authMethod before inputStream: " + resultXml); + requestHeaders.put("Authorization", "Basic " + authToken); + //requestHeaders.put("accept", "application/xml"); } + HttpURLConnection conn = (HttpURLConnection) qUrl.openConnection(); + conn.setRequestMethod("GET"); + this.setRequestHeader(conn); + resultStream = conn.getInputStream(); - this.resultStream = theHttpInputStream; if ("json".equals(this.resultOutputFormat)) { resultJson = IOUtils.toString(this.resultStream, StandardCharsets.UTF_8); resultXml = JsonUtils.convertToXML(resultJson); @@ -380,7 +386,7 @@ public class RestIterator implements Iterator { try { if (this.resultTotal == -1) { this.resultTotal = Integer.parseInt(this.xprResultTotalPath.evaluate(resultNode)); - if ("page".equalsIgnoreCase(this.resumptionType) && !this.BASIC.equalsIgnoreCase(this.authMethod)) { + if ("page".equalsIgnoreCase(this.resumptionType) && !this.AUTHBASIC.equalsIgnoreCase(this.authMethod)) { this.resultTotal += 1; } // to correct the upper bound log.info("resultTotal was -1 is now: " + this.resultTotal); @@ -433,6 +439,22 @@ public class RestIterator implements Iterator { } } + /** + * setRequestHeader + * + * setRequestProperty: Sets the general request property. If a property with the key already exists, overwrite its value with the new value. + * @param conn + */ + private void setRequestHeader(HttpURLConnection conn) { + if (requestHeaders != null) { + for (String key : requestHeaders.keySet()) { + conn.setRequestProperty(key, requestHeaders.get(key)); + } + log.debug("Set Request Header with: " + requestHeaders); + } + + } + public String getResultFormatValue() { return this.resultFormatValue; } diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/RestCollectorPluginTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/RestCollectorPluginTest.java index f708c367b..a9fc325c3 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/RestCollectorPluginTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/RestCollectorPluginTest.java @@ -4,10 +4,16 @@ package eu.dnetlib.dhp.collection.plugin.rest; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; import java.util.HashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; +import com.google.gson.Gson; import org.junit.jupiter.api.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,18 +31,18 @@ class RestCollectorPluginTest { private static final Logger log = LoggerFactory.getLogger(RestCollectorPluginTest.class); - private final String baseUrl = "https://share.osf.io/api/v2/search/creativeworks/_search"; - private final String resumptionType = "count"; - private final String resumptionParam = "from"; - private final String entityXpath = "//hits/hits"; - private final String resumptionXpath = "//hits"; - private final String resultTotalXpath = "//hits/total"; - private final String resultFormatParam = "format"; + private final String baseUrl = "https://ddh-openapi.worldbank.org/search"; + private final String resumptionType = "discover"; + private final String resumptionParam = "skip"; + private final String entityXpath = "//*[local-name()='data']"; + private final String resumptionXpath = ""; + private final String resultTotalXpath = "//*[local-name()='count']"; + private final String resultFormatParam = ""; private final String resultFormatValue = "json"; - private final String resultSizeParam = "size"; + private final String resultSizeParam = "top"; private final String resultSizeValue = "10"; // private String query = "q=%28sources%3ASocArXiv+AND+type%3Apreprint%29"; - private final String query = "q=%28sources%3AengrXiv+AND+type%3Apreprint%29"; + private final String query = ""; // private String query = "=(sources:engrXiv AND type:preprint)"; private final String protocolDescriptor = "rest_json2xml"; @@ -56,10 +62,12 @@ class RestCollectorPluginTest { params.put("resultSizeValue", resultSizeValue); params.put("queryParams", query); params.put("entityXpath", entityXpath); + params.put("requestHeaderMap", "{\"User-Agent\": \"OpenAIRE DEV\"}"); api.setBaseUrl(baseUrl); api.setParams(params); + rcp = new RestCollectorPlugin(new HttpClientParams()); } @@ -78,4 +86,20 @@ class RestCollectorPluginTest { log.info("{}", i.intValue()); Assertions.assertTrue(i.intValue() > 0); } + + @Disabled + @Test + void testUrl() throws IOException { + String url_s = "https://ddh-openapi.worldbank.org/search?&top=10"; + URL url = new URL(url_s); + final HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + conn.setRequestProperty("User-Agent", "OpenAIRE"); + Gson gson = new Gson(); + System.out.println("Request header"); + System.out.println(gson.toJson(conn.getHeaderFields())); + InputStream inputStream = conn.getInputStream(); + + + } } diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/RestIteratorTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/RestIteratorTest.java index e2d6ad3e7..ed31c2b7e 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/RestIteratorTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/RestIteratorTest.java @@ -44,7 +44,7 @@ public class RestIteratorTest { final RestIterator iterator = new RestIterator(clientParams, baseUrl, resumptionType, resumptionParam, resumptionXpath, resultTotalXpath, resultFormatParam, resultFormatValue, resultSizeParam, resultSizeValue, - query, entityXpath, authMethod, authToken, resultOffsetParam); + query, entityXpath, authMethod, authToken, resultOffsetParam, null); int i = 20; while (iterator.hasNext() && i > 0) { String result = iterator.next(); From c272c4ad68255820fe6d9fd3d4aac182da1f5678 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Wed, 29 May 2024 15:50:07 +0200 Subject: [PATCH 74/97] code formatting --- .../plugin/rest/RestCollectorPlugin.java | 5 +++-- .../dhp/collection/plugin/rest/RestIterator.java | 16 ++++++++-------- .../plugin/rest/RestCollectorPluginTest.java | 5 ++--- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestCollectorPlugin.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestCollectorPlugin.java index 8445e49e0..f4ba09f72 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestCollectorPlugin.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestCollectorPlugin.java @@ -8,9 +8,10 @@ import java.util.Spliterators; import java.util.stream.Stream; import java.util.stream.StreamSupport; -import com.google.gson.Gson; import org.apache.commons.lang3.StringUtils; +import com.google.gson.Gson; + import eu.dnetlib.dhp.collection.ApiDescriptor; import eu.dnetlib.dhp.collection.plugin.CollectorPlugin; import eu.dnetlib.dhp.common.aggregation.AggregatorReport; @@ -95,7 +96,7 @@ public class RestCollectorPlugin implements CollectorPlugin { authMethod, authToken, resultOutputFormat, - requestHeaders); + requestHeaders); return StreamSupport .stream( diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java index e51c9eb1b..2518fd92f 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java @@ -25,7 +25,6 @@ import javax.xml.xpath.XPathExpression; import javax.xml.xpath.XPathExpressionException; import javax.xml.xpath.XPathFactory; -import com.google.common.collect.Maps; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpHeaders; @@ -36,6 +35,8 @@ import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.InputSource; +import com.google.common.collect.Maps; + import eu.dnetlib.dhp.collection.plugin.utils.JsonUtils; import eu.dnetlib.dhp.common.collection.CollectorException; import eu.dnetlib.dhp.common.collection.HttpClientParams; @@ -51,7 +52,6 @@ import eu.dnetlib.dhp.common.collection.HttpClientParams; */ public class RestIterator implements Iterator { - private static final Logger log = LoggerFactory.getLogger(RestIterator.class); public static final String UTF_8 = "UTF-8"; private static final int MAX_ATTEMPTS = 5; @@ -93,11 +93,10 @@ public class RestIterator implements Iterator { private final String resultOutputFormat; /* - Can be used to set additional request headers, like for content negotiation - */ + * Can be used to set additional request headers, like for content negotiation + */ private Map requestHeaders; - /** * RestIterator class compatible to version 1.3.33 */ @@ -245,11 +244,11 @@ public class RestIterator implements Iterator { if (this.authMethod == "bearer") { log.trace("RestIterator.downloadPage():: authMethod before inputStream: " + resultXml); requestHeaders.put("Authorization", "Bearer " + authToken); - //requestHeaders.put("Content-Type", "application/json"); + // requestHeaders.put("Content-Type", "application/json"); } else if (AUTHBASIC.equalsIgnoreCase(this.authMethod)) { log.trace("RestIterator.downloadPage():: authMethod before inputStream: " + resultXml); requestHeaders.put("Authorization", "Basic " + authToken); - //requestHeaders.put("accept", "application/xml"); + // requestHeaders.put("accept", "application/xml"); } HttpURLConnection conn = (HttpURLConnection) qUrl.openConnection(); conn.setRequestMethod("GET"); @@ -386,7 +385,8 @@ public class RestIterator implements Iterator { try { if (this.resultTotal == -1) { this.resultTotal = Integer.parseInt(this.xprResultTotalPath.evaluate(resultNode)); - if ("page".equalsIgnoreCase(this.resumptionType) && !this.AUTHBASIC.equalsIgnoreCase(this.authMethod)) { + if ("page".equalsIgnoreCase(this.resumptionType) + && !this.AUTHBASIC.equalsIgnoreCase(this.authMethod)) { this.resultTotal += 1; } // to correct the upper bound log.info("resultTotal was -1 is now: " + this.resultTotal); diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/RestCollectorPluginTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/RestCollectorPluginTest.java index a9fc325c3..99b95d9e3 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/RestCollectorPluginTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/RestCollectorPluginTest.java @@ -13,11 +13,12 @@ import java.util.HashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; -import com.google.gson.Gson; import org.junit.jupiter.api.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.gson.Gson; + import eu.dnetlib.dhp.collection.ApiDescriptor; import eu.dnetlib.dhp.common.aggregation.AggregatorReport; import eu.dnetlib.dhp.common.collection.CollectorException; @@ -67,7 +68,6 @@ class RestCollectorPluginTest { api.setBaseUrl(baseUrl); api.setParams(params); - rcp = new RestCollectorPlugin(new HttpClientParams()); } @@ -100,6 +100,5 @@ class RestCollectorPluginTest { System.out.println(gson.toJson(conn.getHeaderFields())); InputStream inputStream = conn.getInputStream(); - } } From a02f3f0d2b5b87fe6cf2b2b37f0e54832b16ce8c Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Thu, 30 May 2024 10:21:18 +0200 Subject: [PATCH 75/97] code formatting --- .../collection/plugin/utils/XMLIterator.java | 32 +++++---- .../plugin/file/FileGZipMultipleNodeTest.java | 69 ++++++++++--------- 2 files changed, 52 insertions(+), 49 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/utils/XMLIterator.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/utils/XMLIterator.java index ca351346c..7e5c5e3c3 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/utils/XMLIterator.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/utils/XMLIterator.java @@ -70,10 +70,11 @@ public class XMLIterator implements Iterator { super(); this.element = element; if (element.contains(",")) { - elements= Arrays.stream(element.split(",")) - .filter(StringUtils::isNoneBlank) - .map(String::toLowerCase) - .collect(Collectors.toList()); + elements = Arrays + .stream(element.split(",")) + .filter(StringUtils::isNoneBlank) + .map(String::toLowerCase) + .collect(Collectors.toList()); } this.inputStream = inputStream; this.parser = getParser(); @@ -155,16 +156,16 @@ public class XMLIterator implements Iterator { XMLEvent peek = parser.peek(); if (peek != null && peek.isStartElement()) { String name = peek.asStartElement().getName().getLocalPart(); - if( isCheckTag(name)) - return peek; + if (isCheckTag(name)) + return peek; } while (parser.hasNext()) { - XMLEvent event= parser.nextEvent(); + XMLEvent event = parser.nextEvent(); if (event != null && event.isStartElement()) { String name = event.asStartElement().getName().getLocalPart(); - if( isCheckTag(name)) - return event; + if (isCheckTag(name)) + return event; } } return null; @@ -181,12 +182,13 @@ public class XMLIterator implements Iterator { } private boolean isCheckTag(final String tagName) { - if (elements!= null) { - final String found =elements.stream() - .filter(e -> e.equalsIgnoreCase(tagName)) - .findFirst() - .orElse(null); - if (found!= null) + if (elements != null) { + final String found = elements + .stream() + .filter(e -> e.equalsIgnoreCase(tagName)) + .findFirst() + .orElse(null); + if (found != null) return true; } else { if (element.equalsIgnoreCase(tagName)) { diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/file/FileGZipMultipleNodeTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/file/FileGZipMultipleNodeTest.java index 2ed199156..2b5e90ab2 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/file/FileGZipMultipleNodeTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/file/FileGZipMultipleNodeTest.java @@ -1,9 +1,11 @@ + package eu.dnetlib.dhp.collection.plugin.file; +import java.io.IOException; +import java.util.HashMap; +import java.util.Objects; +import java.util.stream.Stream; -import eu.dnetlib.dhp.collection.ApiDescriptor; -import eu.dnetlib.dhp.common.aggregation.AggregatorReport; -import eu.dnetlib.dhp.common.collection.CollectorException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.junit.jupiter.api.*; @@ -12,52 +14,51 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.HashMap; -import java.util.Objects; -import java.util.stream.Stream; +import eu.dnetlib.dhp.collection.ApiDescriptor; +import eu.dnetlib.dhp.common.aggregation.AggregatorReport; +import eu.dnetlib.dhp.common.collection.CollectorException; @TestMethodOrder(MethodOrderer.OrderAnnotation.class) @ExtendWith(MockitoExtension.class) public class FileGZipMultipleNodeTest { - private static final Logger log = LoggerFactory.getLogger(FileGZipCollectorPluginTest.class); + private static final Logger log = LoggerFactory.getLogger(FileGZipCollectorPluginTest.class); - private final ApiDescriptor api = new ApiDescriptor(); + private final ApiDescriptor api = new ApiDescriptor(); - private FileGZipCollectorPlugin plugin; + private FileGZipCollectorPlugin plugin; - private static final String SPLIT_ON_ELEMENT = "incollection,article"; + private static final String SPLIT_ON_ELEMENT = "incollection,article"; - @BeforeEach - public void setUp() throws IOException { + @BeforeEach + public void setUp() throws IOException { - final String gzipFile = Objects - .requireNonNull( - this - .getClass() - .getResource("/eu/dnetlib/dhp/collection/plugin/file/dblp.gz")) - .getFile(); + final String gzipFile = Objects + .requireNonNull( + this + .getClass() + .getResource("/eu/dnetlib/dhp/collection/plugin/file/dblp.gz")) + .getFile(); - api.setBaseUrl(gzipFile); + api.setBaseUrl(gzipFile); - HashMap params = new HashMap<>(); - params.put("splitOnElement", SPLIT_ON_ELEMENT); + HashMap params = new HashMap<>(); + params.put("splitOnElement", SPLIT_ON_ELEMENT); - api.setParams(params); + api.setParams(params); - FileSystem fs = FileSystem.get(new Configuration()); - plugin = new FileGZipCollectorPlugin(fs); - } + FileSystem fs = FileSystem.get(new Configuration()); + plugin = new FileGZipCollectorPlugin(fs); + } - @Test - void test() throws CollectorException { + @Test + void test() throws CollectorException { - final Stream stream = plugin.collect(api, new AggregatorReport()); + final Stream stream = plugin.collect(api, new AggregatorReport()); - stream.limit(10).forEach(s -> { - Assertions.assertTrue(s.length() > 0); - log.info(s); - }); - } + stream.limit(10).forEach(s -> { + Assertions.assertTrue(s.length() > 0); + log.info(s); + }); + } } From 81090ad593b1bb1572c033989c86e79f795670e6 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Wed, 5 Jun 2024 10:03:33 +0200 Subject: [PATCH 76/97] [IE OAIPHM] added oozie workflow, minor changes, code formatting --- .../dhp/oa/oaipmh/IrishOaiExporterJob.java | 68 ++++++----- .../dhp/oa/oaipmh/oozie_app/workflow.xml | 106 ++++++++++++++++++ .../dhp/oa/oaipmh/DbSerializationTest.java | 14 +-- .../oa/oaipmh/IrishOaiExporterJobTest.java | 3 +- 4 files changed, 155 insertions(+), 36 deletions(-) create mode 100644 dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/oaipmh/oozie_app/workflow.xml diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java index 433baf272..57f180fa0 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java @@ -46,15 +46,16 @@ public class IrishOaiExporterJob { public static void main(final String[] args) throws Exception { final ArgumentApplicationParser parser = new ArgumentApplicationParser( - IOUtils - .toString(XmlConverterJob.class - .getResourceAsStream("/eu/dnetlib/dhp/oa/oaipmh/input_params_irish_oai_exporter.json"))); + IOUtils + .toString( + XmlConverterJob.class + .getResourceAsStream("/eu/dnetlib/dhp/oa/oaipmh/input_params_irish_oai_exporter.json"))); parser.parseArgument(args); final Boolean isSparkSessionManaged = Optional - .ofNullable(parser.get("isSparkSessionManaged")) - .map(Boolean::valueOf) - .orElse(Boolean.TRUE); + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); log.info("isSparkSessionManaged: {}", isSparkSessionManaged); final String inputPath = parser.get("inputPath"); @@ -62,9 +63,9 @@ public class IrishOaiExporterJob { final String dbUser = parser.get("dbUser"); final String dbPwd = parser.get("dbPwd"); final int numConnections = Optional - .ofNullable(parser.get("numConnections")) - .map(Integer::valueOf) - .orElse(NUM_CONNECTIONS); + .ofNullable(parser.get("numConnections")) + .map(Integer::valueOf) + .orElse(NUM_CONNECTIONS); log.info("inputPath: '{}'", inputPath); log.info("dbUrl: '{}'", dbUrl); @@ -78,29 +79,31 @@ public class IrishOaiExporterJob { final SparkConf conf = new SparkConf(); conf.registerKryoClasses(new Class[] { - SerializableSolrInputDocument.class + SerializableSolrInputDocument.class }); final Encoder encoderTuple = Encoders.bean(TupleWrapper.class); final Encoder encoderOaiRecord = Encoders.bean(OaiRecordWrapper.class); + final String date = LocalDateTime.now().toString(); + log.info("Creating temporary table..."); runWithSparkSession(conf, isSparkSessionManaged, spark -> { final Dataset docs = spark - .read() - .schema(encoderTuple.schema()) - .json(inputPath) - .as(encoderTuple) - .map((MapFunction) TupleWrapper::getXml, Encoders.STRING()) - .map((MapFunction) IrishOaiExporterJob::asIrishOaiResult, encoderOaiRecord) - .filter((FilterFunction) obj -> (obj != null) && StringUtils.isNotBlank(obj.getId())); + .read() + .schema(encoderTuple.schema()) + .json(inputPath) + .as(encoderTuple) + .map((MapFunction) TupleWrapper::getXml, Encoders.STRING()) + .map((MapFunction) r -> asIrishOaiResult(r, date), encoderOaiRecord) + .filter((FilterFunction) obj -> (obj != null) && StringUtils.isNotBlank(obj.getId())); docs - .repartition(numConnections) - .write() - .mode(SaveMode.Overwrite) - .jdbc(dbUrl, TMP_OAI_TABLE, connectionProperties); + .repartition(numConnections) + .write() + .mode(SaveMode.Overwrite) + .jdbc(dbUrl, TMP_OAI_TABLE, connectionProperties); }); log.info("Temporary table created."); @@ -108,14 +111,15 @@ public class IrishOaiExporterJob { log.info("Updating OAI records..."); try (final Connection con = DriverManager.getConnection(dbUrl, dbUser, dbPwd)) { try (final Statement st = con.createStatement()) { - final String query = IOUtils.toString(IrishOaiExporterJob.class.getResourceAsStream("oai-finalize.sql")); + final String query = IOUtils + .toString(IrishOaiExporterJob.class.getResourceAsStream("oai-finalize.sql")); st.execute(query); } } log.info("DONE."); } - protected static OaiRecordWrapper asIrishOaiResult(final String xml) { + protected static OaiRecordWrapper asIrishOaiResult(final String xml, final String date) { try { final Document doc = DocumentHelper.parseText(xml); final OaiRecordWrapper r = new OaiRecordWrapper(); @@ -123,7 +127,7 @@ public class IrishOaiExporterJob { if (isValid(doc)) { r.setId(doc.valueOf("//*[local-name()='objIdentifier']").trim()); r.setBody(gzip(doc.selectSingleNode("//*[local-name()='entity']").asXML())); - r.setDate(LocalDateTime.now().toString()); + r.setDate(date); r.setSets(new ArrayList<>()); } return r; @@ -140,19 +144,25 @@ public class IrishOaiExporterJob { if (n != null) { for (final Object o : n.selectNodes(".//*[local-name()='datainfo']/*[local-name()='deletedbyinference']")) { - if ("true".equals(((Node) o).getText().trim())) { return false; } + if ("true".equals(((Node) o).getText().trim())) { + return false; + } } // verify the main country of the result for (final Object o : n.selectNodes("./*[local-name()='country']")) { - if ("IE".equals(((Node) o).valueOf("@classid").trim())) { return true; } + if ("IE".equals(((Node) o).valueOf("@classid").trim())) { + return true; + } } // verify the countries of the related organizations for (final Object o : n.selectNodes(".//*[local-name()='rel']")) { final String relType = ((Node) o).valueOf("./*[local-name() = 'to']/@type").trim(); final String relCountry = ((Node) o).valueOf("./*[local-name() = 'country']/@classid").trim(); - if ("organization".equals(relType) && "IE".equals(relCountry)) { return true; } + if ("organization".equals(relType) && "IE".equals(relCountry)) { + return true; + } } } return false; @@ -160,7 +170,9 @@ public class IrishOaiExporterJob { } protected static byte[] gzip(final String str) { - if (StringUtils.isBlank(str)) { return null; } + if (StringUtils.isBlank(str)) { + return null; + } try (final ByteArrayOutputStream baos = new ByteArrayOutputStream()) { try (final GZIPOutputStream gzip = new GZIPOutputStream(baos)) { diff --git a/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/oaipmh/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/oaipmh/oozie_app/workflow.xml new file mode 100644 index 000000000..c4caad91e --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/main/resources/eu/dnetlib/dhp/oa/oaipmh/oozie_app/workflow.xml @@ -0,0 +1,106 @@ + + + + + inputPath + The path of the input records on HDFS + + + numConnections + number of connections to the postgres db (for the write operation) + + + dbUrl + the url of the database + + + dbUser + the user of the database + + + dbPwd + the password for the user of the database + + + + sparkDriverMemory + memory for driver process + + + sparkExecutorMemory + memory for individual executor + + + sparkExecutorCores + number of cores used by single executor + + + oozieActionShareLibForSpark2 + oozie action sharelib for spark 2.* + + + spark2ExtraListeners + com.cloudera.spark.lineage.NavigatorAppListener + spark 2.* extra listeners classname + + + spark2SqlQueryExecutionListeners + com.cloudera.spark.lineage.NavigatorQueryListener + spark 2.* sql query execution listeners classname + + + spark2YarnHistoryServerAddress + spark 2.* yarn history server address + + + spark2EventLogDir + spark 2.* event log dir location + + + + + ${jobTracker} + ${nameNode} + + + oozie.action.sharelib.for.spark + ${oozieActionShareLibForSpark2} + + + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + yarn + cluster + Irish OAI-PHM provision + eu.dnetlib.dhp.oa.oaipmh.IrishOaiExporterJob + dhp-graph-provision-${projectVersion}.jar + + --executor-cores=${sparkExecutorCores} + --executor-memory=${sparkExecutorMemory} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.shuffle.partitions=8000 + + --inputPath${inputPath} + --numConnections${numConnections} + --dbUrl${dbUrl} + --dbUser${dbUser} + --dbPwd${dbPwd} + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/oaipmh/DbSerializationTest.java b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/oaipmh/DbSerializationTest.java index f33708f86..d487fda94 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/oaipmh/DbSerializationTest.java +++ b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/oaipmh/DbSerializationTest.java @@ -42,10 +42,10 @@ public class DbSerializationTest { conf.set("spark.driver.host", "localhost"); spark = SparkSession - .builder() - .appName("TEST") - .config(conf) - .getOrCreate(); + .builder() + .appName("TEST") + .config(conf) + .getOrCreate(); } @AfterAll @@ -79,9 +79,9 @@ public class DbSerializationTest { final Dataset docs = spark.createDataset(list, Encoders.bean(OaiRecordWrapper.class)); docs - .write() - .mode(SaveMode.Overwrite) - .jdbc(dbUrl, IrishOaiExporterJob.TMP_OAI_TABLE, connectionProperties); + .write() + .mode(SaveMode.Overwrite) + .jdbc(dbUrl, IrishOaiExporterJob.TMP_OAI_TABLE, connectionProperties); }); diff --git a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJobTest.java b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJobTest.java index 57a32e246..c16f75e1d 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJobTest.java +++ b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJobTest.java @@ -10,6 +10,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.charset.Charset; +import java.time.LocalDateTime; import java.util.zip.GZIPInputStream; import org.apache.commons.io.IOUtils; @@ -23,7 +24,7 @@ public class IrishOaiExporterJobTest { @Test void testAsIrishOaiResult() throws Exception { final String xml = IOUtils.toString(getClass().getResourceAsStream("record_IE.xml")); - final OaiRecordWrapper res = IrishOaiExporterJob.asIrishOaiResult(xml); + final OaiRecordWrapper res = IrishOaiExporterJob.asIrishOaiResult(xml, LocalDateTime.now().toString()); assertNotNull(res.getId()); assertNotNull(res.getBody()); assertNotNull(res.getSets()); From 73bd1938a5a8dfead4035f50fded1108cf45a281 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Wed, 5 Jun 2024 12:17:35 +0200 Subject: [PATCH 77/97] [graph2hive] use sparkExecutorMemory to define also the memoryOverhead --- .../dhp/oa/graph/hive/oozie_app/workflow.xml | 30 ++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/hive/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/hive/oozie_app/workflow.xml index 4468382be..eec67fc5c 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/hive/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/hive/oozie_app/workflow.xml @@ -116,17 +116,19 @@ --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} + --conf spark.sql.shuffle.partitions=10000 --inputPath${inputPath}/publication --hiveDbName${hiveDbName} --classNameeu.dnetlib.dhp.schema.oaf.Publication --hiveMetastoreUris${hiveMetastoreUris} - --numPartitions8000 + --numPartitions10000 @@ -143,17 +145,19 @@ --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} + --conf spark.sql.shuffle.partitions=4000 --inputPath${inputPath}/dataset --hiveDbName${hiveDbName} --classNameeu.dnetlib.dhp.schema.oaf.Dataset --hiveMetastoreUris${hiveMetastoreUris} - --numPartitions4000 + --numPartitions8000 @@ -170,11 +174,13 @@ --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} + --conf spark.sql.shuffle.partitions=8000 --inputPath${inputPath}/otherresearchproduct --hiveDbName${hiveDbName} @@ -197,17 +203,19 @@ --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} + --conf spark.sql.shuffle.partitions=1000 --inputPath${inputPath}/software --hiveDbName${hiveDbName} --classNameeu.dnetlib.dhp.schema.oaf.Software --hiveMetastoreUris${hiveMetastoreUris} - --numPartitions300 + --numPartitions1000 @@ -224,17 +232,19 @@ --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} + --conf spark.sql.shuffle.partitions=200 --inputPath${inputPath}/datasource --hiveDbName${hiveDbName} --classNameeu.dnetlib.dhp.schema.oaf.Datasource --hiveMetastoreUris${hiveMetastoreUris} - --numPartitions100 + --numPartitions200 @@ -251,17 +261,19 @@ --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} + --conf spark.sql.shuffle.partitions=1000 --inputPath${inputPath}/organization --hiveDbName${hiveDbName} --classNameeu.dnetlib.dhp.schema.oaf.Organization --hiveMetastoreUris${hiveMetastoreUris} - --numPartitions400 + --numPartitions1000 @@ -278,17 +290,19 @@ --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} + --conf spark.sql.shuffle.partitions=1000 --inputPath${inputPath}/project --hiveDbName${hiveDbName} --classNameeu.dnetlib.dhp.schema.oaf.Project --hiveMetastoreUris${hiveMetastoreUris} - --numPartitions100 + --numPartitions1000 @@ -305,17 +319,19 @@ --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} + --conf spark.sql.shuffle.partitions=15000 --inputPath${inputPath}/relation --hiveDbName${hiveDbName} --classNameeu.dnetlib.dhp.schema.oaf.Relation --hiveMetastoreUris${hiveMetastoreUris} - --numPartitions10000 + --numPartitions15000 From f70dc76b61a2597ed1a62d90a5f8394cf8710f19 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Thu, 6 Jun 2024 10:43:10 +0200 Subject: [PATCH 78/97] minor --- .../oaf/utils/ResultTypeComparator.java | 87 ------------------- .../dhp/oa/oaipmh/IrishOaiExporterJob.java | 3 +- 2 files changed, 1 insertion(+), 89 deletions(-) delete mode 100644 dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/ResultTypeComparator.java diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/ResultTypeComparator.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/ResultTypeComparator.java deleted file mode 100644 index e10b281b8..000000000 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/ResultTypeComparator.java +++ /dev/null @@ -1,87 +0,0 @@ - -package eu.dnetlib.dhp.schema.oaf.utils; - -import static eu.dnetlib.dhp.schema.common.ModelConstants.CROSSREF_ID; - -import java.util.Comparator; -import java.util.HashSet; -import java.util.Optional; -import java.util.stream.Collectors; - -import eu.dnetlib.dhp.schema.common.ModelConstants; -import eu.dnetlib.dhp.schema.oaf.KeyValue; -import eu.dnetlib.dhp.schema.oaf.Result; - -public class ResultTypeComparator implements Comparator { - - public static final ResultTypeComparator INSTANCE = new ResultTypeComparator(); - - @Override - public int compare(Result left, Result right) { - - if (left == null && right == null) - return 0; - if (left == null) - return 1; - if (right == null) - return -1; - - HashSet lCf = getCollectedFromIds(left); - HashSet rCf = getCollectedFromIds(right); - - if (lCf.contains(CROSSREF_ID) && !rCf.contains(CROSSREF_ID)) { - return -1; - } - if (!lCf.contains(CROSSREF_ID) && rCf.contains(CROSSREF_ID)) { - return 1; - } - - if (left.getResulttype() == null || left.getResulttype().getClassid() == null) { - if (right.getResulttype() == null || right.getResulttype().getClassid() == null) { - return 0; - } - return 1; - } else if (right.getResulttype() == null || right.getResulttype().getClassid() == null) { - return -1; - } - - String lClass = left.getResulttype().getClassid(); - String rClass = right.getResulttype().getClassid(); - - if (!lClass.equals(rClass)) { - if (lClass.equals(ModelConstants.PUBLICATION_RESULTTYPE_CLASSID)) - return -1; - if (rClass.equals(ModelConstants.PUBLICATION_RESULTTYPE_CLASSID)) - return 1; - - if (lClass.equals(ModelConstants.DATASET_RESULTTYPE_CLASSID)) - return -1; - if (rClass.equals(ModelConstants.DATASET_RESULTTYPE_CLASSID)) - return 1; - - if (lClass.equals(ModelConstants.SOFTWARE_RESULTTYPE_CLASSID)) - return -1; - if (rClass.equals(ModelConstants.SOFTWARE_RESULTTYPE_CLASSID)) - return 1; - - if (lClass.equals(ModelConstants.ORP_RESULTTYPE_CLASSID)) - return -1; - if (rClass.equals(ModelConstants.ORP_RESULTTYPE_CLASSID)) - return 1; - } - - // Else (but unlikely), lexicographical ordering will do. - return lClass.compareTo(rClass); - } - - protected HashSet getCollectedFromIds(Result left) { - return Optional - .ofNullable(left.getCollectedfrom()) - .map( - cf -> cf - .stream() - .map(KeyValue::getKey) - .collect(Collectors.toCollection(HashSet::new))) - .orElse(new HashSet<>()); - } -} diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java index 57f180fa0..3d69370af 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/oaipmh/IrishOaiExporterJob.java @@ -31,7 +31,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import eu.dnetlib.dhp.application.ArgumentApplicationParser; -import eu.dnetlib.dhp.oa.provision.XmlConverterJob; import eu.dnetlib.dhp.oa.provision.model.SerializableSolrInputDocument; import eu.dnetlib.dhp.oa.provision.model.TupleWrapper; @@ -48,7 +47,7 @@ public class IrishOaiExporterJob { final ArgumentApplicationParser parser = new ArgumentApplicationParser( IOUtils .toString( - XmlConverterJob.class + IrishOaiExporterJob.class .getResourceAsStream("/eu/dnetlib/dhp/oa/oaipmh/input_params_irish_oai_exporter.json"))); parser.parseArgument(args); From ce2364743a445e42c434c910852dc26a1b75a7a8 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Thu, 6 Jun 2024 10:43:43 +0200 Subject: [PATCH 79/97] applying changes from PR#442: Fix for missing collectedfrom after dedup --- .../oaf/utils/MergeEntitiesComparator.java | 104 ++++++++++++++++++ .../dhp/schema/oaf/utils/MergeUtils.java | 53 +++++---- .../dhp/oa/dedup/DatasetMergerTest.java | 103 +++++++++++++++++ .../dhp/oa/dedup/EntityMergerTest.java | 10 +- .../dnetlib/dhp/dedup/json/dataset_merge.json | 2 + 5 files changed, 239 insertions(+), 33 deletions(-) create mode 100644 dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeEntitiesComparator.java create mode 100644 dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/DatasetMergerTest.java create mode 100644 dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/json/dataset_merge.json diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeEntitiesComparator.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeEntitiesComparator.java new file mode 100644 index 000000000..5792fc10f --- /dev/null +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeEntitiesComparator.java @@ -0,0 +1,104 @@ + +package eu.dnetlib.dhp.schema.oaf.utils; + +import static eu.dnetlib.dhp.schema.common.ModelConstants.CROSSREF_ID; + +import java.util.*; +import java.util.stream.Collectors; + +import eu.dnetlib.dhp.schema.common.ModelConstants; +import eu.dnetlib.dhp.schema.oaf.KeyValue; +import eu.dnetlib.dhp.schema.oaf.Oaf; +import eu.dnetlib.dhp.schema.oaf.OafEntity; +import eu.dnetlib.dhp.schema.oaf.Result; + +public class MergeEntitiesComparator implements Comparator { + static final List PID_AUTHORITIES = Arrays + .asList( + ModelConstants.ARXIV_ID, + ModelConstants.PUBMED_CENTRAL_ID, + ModelConstants.EUROPE_PUBMED_CENTRAL_ID, + ModelConstants.DATACITE_ID, + ModelConstants.CROSSREF_ID); + + static final List RESULT_TYPES = Arrays + .asList( + ModelConstants.ORP_RESULTTYPE_CLASSID, + ModelConstants.SOFTWARE_RESULTTYPE_CLASSID, + ModelConstants.DATASET_RESULTTYPE_CLASSID, + ModelConstants.PUBLICATION_RESULTTYPE_CLASSID); + + public static final Comparator INSTANCE = new MergeEntitiesComparator(); + + @Override + public int compare(Oaf left, Oaf right) { + if (left == null && right == null) + return 0; + if (left == null) + return -1; + if (right == null) + return 1; + + int res = 0; + + // pid authority + int cfp1 = left + .getCollectedfrom() + .stream() + .map(kv -> PID_AUTHORITIES.indexOf(kv.getKey())) + .max(Integer::compare) + .orElse(-1); + int cfp2 = right + .getCollectedfrom() + .stream() + .map(kv -> PID_AUTHORITIES.indexOf(kv.getKey())) + .max(Integer::compare) + .orElse(-1); + + if (cfp1 >= 0 && cfp1 > cfp2) { + return 1; + } else if (cfp2 >= 0 && cfp2 > cfp1) { + return -1; + } + + // trust + if (left.getDataInfo() != null && right.getDataInfo() != null) { + res = left.getDataInfo().getTrust().compareTo(right.getDataInfo().getTrust()); + } + + // result type + if (res == 0) { + if (left instanceof Result && right instanceof Result) { + Result r1 = (Result) left; + Result r2 = (Result) right; + + if (r1.getResulttype() == null || r1.getResulttype().getClassid() == null) { + if (r2.getResulttype() != null && r2.getResulttype().getClassid() != null) { + return -1; + } + } else if (r2.getResulttype() == null || r2.getResulttype().getClassid() == null) { + return 1; + } + + int rt1 = RESULT_TYPES.indexOf(r1.getResulttype().getClassid()); + int rt2 = RESULT_TYPES.indexOf(r2.getResulttype().getClassid()); + + if (rt1 >= 0 && rt1 > rt2) { + return 1; + } else if (rt2 >= 0 && rt2 > rt1) { + return -1; + } + } + } + + // id + if (res == 0) { + if (left instanceof OafEntity && right instanceof OafEntity) { + res = ((OafEntity) left).getId().compareTo(((OafEntity) right).getId()); + } + } + + return res; + } + +} diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils.java index 28db94766..f1221add3 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeUtils.java @@ -40,27 +40,12 @@ public class MergeUtils { public static T mergeGroup(String s, Iterator oafEntityIterator, boolean checkDelegateAuthority) { - TreeSet sortedEntities = new TreeSet<>((o1, o2) -> { - int res = 0; - if (o1.getDataInfo() != null && o2.getDataInfo() != null) { - res = o1.getDataInfo().getTrust().compareTo(o2.getDataInfo().getTrust()); - } + ArrayList sortedEntities = new ArrayList<>(); + oafEntityIterator.forEachRemaining(sortedEntities::add); + sortedEntities.sort(MergeEntitiesComparator.INSTANCE.reversed()); - if (res == 0) { - if (o1 instanceof Result && o2 instanceof Result) { - return ResultTypeComparator.INSTANCE.compare((Result) o1, (Result) o2); - } - } - - return res; - }); - - while (oafEntityIterator.hasNext()) { - sortedEntities.add(oafEntityIterator.next()); - } - - Iterator it = sortedEntities.descendingIterator(); + Iterator it = sortedEntities.iterator(); T merged = it.next(); while (it.hasNext()) { @@ -143,7 +128,7 @@ public class MergeUtils { * https://graph.openaire.eu/docs/data-model/pids-and-identifiers#delegated-authorities and in that case it prefers * such version. *

- * Otherwise, it considers a resulttype priority order implemented in {@link ResultTypeComparator} + * Otherwise, it considers a resulttype priority order implemented in {@link MergeEntitiesComparator} * and proceeds with the canonical property merging. * * @param left @@ -161,8 +146,9 @@ public class MergeUtils { if (!leftFromDelegatedAuthority && rightFromDelegatedAuthority) { return right; } + // TODO: raise trust to have preferred fields from one or the other?? - if (new ResultTypeComparator().compare(left, right) < 0) { + if (MergeEntitiesComparator.INSTANCE.compare(left, right) > 0) { return mergeResultFields(left, right); } else { return mergeResultFields(right, left); @@ -225,9 +211,9 @@ public class MergeUtils { private static List mergeLists(final List left, final List right, int trust, Function keyExtractor, BinaryOperator merger) { - if (left == null) { - return right; - } else if (right == null) { + if (left == null || left.isEmpty()) { + return right != null ? right : new ArrayList<>(); + } else if (right == null || right.isEmpty()) { return left; } @@ -405,7 +391,7 @@ public class MergeUtils { } // should be an instance attribute, get the first non-null value - merge.setLanguage(coalesce(merge.getLanguage(), enrich.getLanguage())); + merge.setLanguage(coalesceQualifier(merge.getLanguage(), enrich.getLanguage())); // distinct countries, do not manage datainfo merge.setCountry(mergeQualifiers(merge.getCountry(), enrich.getCountry(), trust)); @@ -575,6 +561,13 @@ public class MergeUtils { return m != null ? m : e; } + private static Qualifier coalesceQualifier(Qualifier m, Qualifier e) { + if (m == null || m.getClassid() == null || StringUtils.isBlank(m.getClassid())) { + return e; + } + return m; + } + private static List mergeAuthors(List author, List author1, int trust) { List> authors = new ArrayList<>(); if (author != null) { @@ -587,6 +580,10 @@ public class MergeUtils { } private static String instanceKeyExtractor(Instance i) { + // three levels of concatenating: + // 1. :: + // 2. @@ + // 3. || return String .join( "::", @@ -594,10 +591,10 @@ public class MergeUtils { kvKeyExtractor(i.getCollectedfrom()), qualifierKeyExtractor(i.getAccessright()), qualifierKeyExtractor(i.getInstancetype()), - Optional.ofNullable(i.getUrl()).map(u -> String.join("::", u)).orElse(null), + Optional.ofNullable(i.getUrl()).map(u -> String.join("@@", u)).orElse(null), Optional .ofNullable(i.getPid()) - .map(pp -> pp.stream().map(MergeUtils::spKeyExtractor).collect(Collectors.joining("::"))) + .map(pp -> pp.stream().map(MergeUtils::spKeyExtractor).collect(Collectors.joining("@@"))) .orElse(null)); } @@ -706,7 +703,7 @@ public class MergeUtils { private static String spKeyExtractor(StructuredProperty sp) { return Optional .ofNullable(sp) - .map(s -> Joiner.on("::").join(s, qualifierKeyExtractor(s.getQualifier()))) + .map(s -> Joiner.on("||").join(qualifierKeyExtractor(s.getQualifier()), s.getValue())) .orElse(null); } diff --git a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/DatasetMergerTest.java b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/DatasetMergerTest.java new file mode 100644 index 000000000..726814c43 --- /dev/null +++ b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/DatasetMergerTest.java @@ -0,0 +1,103 @@ + +package eu.dnetlib.dhp.oa.dedup; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.io.Serializable; +import java.lang.reflect.InvocationTargetException; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import org.codehaus.jackson.map.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import eu.dnetlib.dhp.schema.oaf.DataInfo; +import eu.dnetlib.dhp.schema.oaf.Dataset; +import eu.dnetlib.dhp.schema.oaf.utils.MergeUtils; +import eu.dnetlib.pace.util.MapDocumentUtil; +import scala.Tuple2; + +class DatasetMergerTest implements Serializable { + + private List> datasets; + + private String testEntityBasePath; + private DataInfo dataInfo; + private final String dedupId = "50|doi_________::3d18564ef27ebe9ef3bd8b4dec67e148"; + private Dataset dataset_top; + + @BeforeEach + public void setUp() throws Exception { + testEntityBasePath = Paths + .get(SparkDedupTest.class.getResource("/eu/dnetlib/dhp/dedup/json").toURI()) + .toFile() + .getAbsolutePath(); + + datasets = readSample(testEntityBasePath + "/dataset_merge.json", Dataset.class); + + dataset_top = getTopPub(datasets); + + dataInfo = setDI(); + } + + @Test + void datasetMergerTest() throws InstantiationException, IllegalAccessException, InvocationTargetException { + Dataset pub_merged = MergeUtils.mergeGroup(dedupId, datasets.stream().map(Tuple2::_2).iterator()); + + // verify id + assertEquals(dedupId, pub_merged.getId()); + assertEquals(2, pub_merged.getInstance().size()); + } + + public DataInfo setDI() { + DataInfo dataInfo = new DataInfo(); + dataInfo.setTrust("0.9"); + dataInfo.setDeletedbyinference(false); + dataInfo.setInferenceprovenance("testing"); + dataInfo.setInferred(true); + return dataInfo; + } + + public Dataset getTopPub(List> publications) { + + Double maxTrust = 0.0; + Dataset maxPub = new Dataset(); + for (Tuple2 publication : publications) { + Double pubTrust = Double.parseDouble(publication._2().getDataInfo().getTrust()); + if (pubTrust > maxTrust) { + maxTrust = pubTrust; + maxPub = publication._2(); + } + } + return maxPub; + } + + public List> readSample(String path, Class clazz) { + List> res = new ArrayList<>(); + BufferedReader reader; + try { + reader = new BufferedReader(new FileReader(path)); + String line = reader.readLine(); + while (line != null) { + res + .add( + new Tuple2<>( + MapDocumentUtil.getJPathString("$.id", line), + new ObjectMapper().readValue(line, clazz))); + // read next line + line = reader.readLine(); + } + reader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + + return res; + } + +} diff --git a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/EntityMergerTest.java b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/EntityMergerTest.java index 4a5a3bd1b..995407edb 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/EntityMergerTest.java +++ b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/EntityMergerTest.java @@ -93,14 +93,14 @@ class EntityMergerTest implements Serializable { assertEquals(pub_top.getJournal().getConferencedate(), pub_merged.getJournal().getConferencedate()); assertEquals(pub_top.getJournal().getConferenceplace(), pub_merged.getJournal().getConferenceplace()); assertEquals("OPEN", pub_merged.getBestaccessright().getClassid()); - assertEquals(pub_top.getResulttype(), pub_merged.getResulttype()); - assertEquals(pub_top.getLanguage(), pub_merged.getLanguage()); - assertEquals(pub_top.getPublisher(), pub_merged.getPublisher()); - assertEquals(pub_top.getEmbargoenddate(), pub_merged.getEmbargoenddate()); + assertEquals(pub_top.getResulttype().getClassid(), pub_merged.getResulttype().getClassid()); + assertEquals(pub_top.getLanguage().getClassid(), pub_merged.getLanguage().getClassid()); + assertEquals("Elsevier BV", pub_merged.getPublisher().getValue()); + assertEquals(pub_top.getEmbargoenddate().getValue(), pub_merged.getEmbargoenddate().getValue()); assertEquals(pub_top.getResourcetype().getClassid(), ""); assertEquals(pub_top.getDateoftransformation(), pub_merged.getDateoftransformation()); assertEquals(pub_top.getOaiprovenance(), pub_merged.getOaiprovenance()); - assertEquals(pub_top.getDateofcollection(), pub_merged.getDateofcollection()); + // assertEquals(pub_top.getDateofcollection(), pub_merged.getDateofcollection()); assertEquals(3, pub_merged.getInstance().size()); assertEquals(2, pub_merged.getCountry().size()); assertEquals(0, pub_merged.getSubject().size()); diff --git a/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/json/dataset_merge.json b/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/json/dataset_merge.json new file mode 100644 index 000000000..86d1a8133 --- /dev/null +++ b/dhp-workflows/dhp-dedup-openaire/src/test/resources/eu/dnetlib/dhp/dedup/json/dataset_merge.json @@ -0,0 +1,2 @@ +{"publisher": {"value": "DANS Data Station Archaeology"}, "dateofcollection": "2024-04-30T12:49:55+0000", "dataInfo": {"invisible": false, "provenanceaction": {"classid": "sysimport:actionset", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "trust": "0.9", "inferred": false, "deletedbyinference": true}, "bestaccessright": {"classid": "UNKNOWN", "classname": "not available", "schemeid": "dnet:access_modes", "schemename": "dnet:access_modes"}, "language": {"classid": "und", "classname": "Undetermined", "schemeid": "dnet:languages", "schemename": "dnet:languages"}, "author": [{"affiliation": [{"dataInfo": {"invisible": false, "provenanceaction": {"classid": "sysimport:actionset", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "trust": "0.9", "inferred": false, "deletedbyinference": false}, "value": "(Geonius)"}], "fullname": "S.A.H. Augustin", "pid": [], "rank": 1}, {"affiliation": [{"dataInfo": {"invisible": false, "provenanceaction": {"classid": "sysimport:actionset", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "trust": "0.9", "inferred": false, "deletedbyinference": false}, "value": "(Geonius)"}], "fullname": "J.J.G. Geraeds", "pid": [], "rank": 2}], "resourcetype": {"classid": "UNKNOWN", "classname": "Unknown", "schemeid": "dnet:dataCite_resource", "schemename": "dnet:dataCite_resource"}, "originalId": ["50|datacite____::3d18564ef27ebe9ef3bd8b4dec67e148", "10.17026/ar/vt9ya1"], "description": [{"value": "In opdracht van Gemeente Beesel heeft Geonius Archeologie in maart 2023 een Inventariserend Veldonderzoek door middel van Proefsleuven (IVO-P) uitgevoerd voor het plangebied Heijackerstraat te Beesel in de gemeente Beesel. Aanleiding voor het uitvoeren van het archeologisch onderzoek vormt de aanvraag van een omgevingsvergunning voor bouw van 20 nieuwe woningen. Uit het vooronderzoek is gebleken dat het plangebied in een dalvakteterras ligt rondom opgestoven landduinen langsheen de Maas. De bodem bestaat volgens de bodemkaart uit vorstvaaggronden. Het plangebied is in het verleden voor zover kon worden vastgesteld in gebruik geweest als bouwland en is niet bebouwd geweest. Het IVO-O heeft uitgewezen dat de bodemopbouw deels intact is, al lijken in sommige boringen sporen van vergravingen of verstoringen te bevatten. Op grond van de resultaten van het vooronderzoek is een hoge verwachting opgesteld op het voorkomen van archeologische waarden uit het paleolithicum tot aan de vroege middeleeuwen. Voor de periode late middeleeuwen en nieuwe tijd is een lage verwachting opgesteld. Op grond van de resultaten van het vooronderzoek is een IVO-P uitgevoerd. Hierbij is een vindplaats aangetroffen bestaande uit drie subrecente greppels en een tweetal recente verstoringen. De vindplaats is als niet behoudenswaardig gewaardeerd. Aanbevolen is het plangebied vrij te geven voor de geplande ontwikkeling."}], "title": [{"qualifier": {"classid": "main title", "classname": "main title", "schemeid": "dnet:dataCite_title", "schemename": "dnet:dataCite_title"}, "value": "Archeologisch onderzoek IVO-P plangebied Heijackerstraat te Beesel"}], "pid": [{"dataInfo": {"invisible": false, "provenanceaction": {"classid": "sysimport:actionset", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "trust": "0.9", "inferred": false, "deletedbyinference": false}, "qualifier": {"classid": "doi", "classname": "Digital Object Identifier", "schemeid": "dnet:pid_types", "schemename": "dnet:pid_types"}, "value": "10.17026/ar/vt9ya1"}], "id": "50|doi_________::3d18564ef27ebe9ef3bd8b4dec67e148", "instance": [{"refereed": {"classid": "0002", "classname": "nonPeerReviewed", "schemeid": "dnet:review_levels", "schemename": "dnet:review_levels"}, "hostedby": {"key": "10|re3data_____::84e123776089ce3c7a33db98d9cd15a8", "value": "EASY"}, "url": ["https://dx.doi.org/10.17026/ar/vt9ya1"], "pid": [{"dataInfo": {"invisible": false, "provenanceaction": {"classid": "sysimport:actionset", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "trust": "0.9", "inferred": false, "deletedbyinference": false}, "qualifier": {"classid": "doi", "classname": "Digital Object Identifier", "schemeid": "dnet:pid_types", "schemename": "dnet:pid_types"}, "value": "10.17026/ar/vt9ya1"}], "instanceTypeMapping": [{"originalType": "Dataset", "typeLabel": "dataset", "vocabularyName": "openaire::coar_resource_types_3_1", "typeCode": "http://purl.org/coar/resource_type/c_ddb1"}], "dateofacceptance": {"value": "2024-01-01"}, "collectedfrom": {"key": "10|openaire____::9e3be59865b2c1c335d32dae2fe7b254", "value": "Datacite"}, "accessright": {"classid": "UNKNOWN", "classname": "not available", "schemeid": "dnet:access_modes", "schemename": "dnet:access_modes"}, "instancetype": {"classid": "0021", "classname": "Dataset", "schemeid": "dnet:publication_resource", "schemename": "dnet:publication_resource"}}], "relevantdate": [{"qualifier": {"classid": "issued", "classname": "issued", "schemeid": "dnet:dataCite_date", "schemename": "dnet:dataCite_date"}, "value": "2024-01-01"}], "resulttype": {"classid": "dataset", "classname": "dataset", "schemeid": "dnet:result_typologies", "schemename": "dnet:result_typologies"}, "context": [], "collectedfrom": [{"key": "10|openaire____::9e3be59865b2c1c335d32dae2fe7b254", "value": "Datacite"}], "dateoftransformation": "2024-04-30T12:49:55+0000", "subject": [], "dateofacceptance": {"value": "2024-01-01"}, "metaResourceType": {"classid": "Research Data", "classname": "Research Data", "schemeid": "openaire::meta_resource_types", "schemename": "openaire::meta_resource_types"}} +{"geolocation": [{"box": "", "place": "", "point": ""}], "dataInfo": {"invisible": false, "provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "trust": "0.9", "inferred": false, "deletedbyinference": true}, "resourcetype": {"classid": "dataset", "classname": "dataset", "schemeid": "dnet:dataCite_resource", "schemename": "dnet:dataCite_resource"}, "pid": [], "contributor": [{"dataInfo": {"invisible": false, "provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "trust": "0.9", "inferred": false, "deletedbyinference": false}, "value": "Geonius"}], "oaiprovenance": {"originDescription": {"metadataNamespace": "", "harvestDate": "2024-05-05T04:33:31Z", "baseURL": "https://easy.dans.knaw.nl/oai", "datestamp": "", "altered": true, "identifier": "oai:easy.dans.knaw.nl:easy-dataset:341200"}}, "bestaccessright": {"classid": "OPEN", "classname": "Open Access", "schemeid": "dnet:access_modes", "schemename": "dnet:access_modes"}, "relevantdate": [], "collectedfrom": [{"key": "10|re3data_____::730f562f9efe8a3b3742d2da510d4335", "value": "B2FIND"}], "id": "50|r3730f562f9e::ace629fb505b6b4343faca03edde1841", "subject": [{"dataInfo": {"invisible": false, "provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "trust": "0.9", "inferred": false, "deletedbyinference": false}, "qualifier": {"classid": "keyword", "classname": "keyword", "schemeid": "dnet:subject_classification_typologies", "schemename": "dnet:subject_classification_typologies"}, "value": "Ancient Cultures"}, {"dataInfo": {"invisible": false, "provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "trust": "0.9", "inferred": false, "deletedbyinference": false}, "qualifier": {"classid": "keyword", "classname": "keyword", "schemeid": "dnet:subject_classification_typologies", "schemename": "dnet:subject_classification_typologies"}, "value": "Humanities"}, {"dataInfo": {"invisible": false, "provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "trust": "0.9", "inferred": false, "deletedbyinference": false}, "qualifier": {"classid": "keyword", "classname": "keyword", "schemeid": "dnet:subject_classification_typologies", "schemename": "dnet:subject_classification_typologies"}, "value": "Archaeology"}], "lastupdatetimestamp": 1716803651625, "author": [{"surname": "Augustin", "name": "S. A. H.", "pid": [], "rank": 1, "affiliation": [], "fullname": "S.A.H. Augustin"}, {"surname": "Geraeds", "name": "J. J. G.", "pid": [], "rank": 2, "affiliation": [], "fullname": "J.J.G. Geraeds"}], "instance": [{"refereed": {"classid": "0002", "classname": "nonPeerReviewed", "schemeid": "dnet:review_levels", "schemename": "dnet:review_levels"}, "hostedby": {"key": "10|re3data_____::84e123776089ce3c7a33db98d9cd15a8", "value": "DANS-EASY"}, "url": ["http://dx.doi.org/https://doi.org/10.17026/AR/VT9YA1"], "pid": [], "instanceTypeMapping": [{"originalType": "Dataset", "typeLabel": "dataset", "vocabularyName": "openaire::coar_resource_types_3_1", "typeCode": "http://purl.org/coar/resource_type/c_ddb1"}], "alternateIdentifier": [{"dataInfo": {"invisible": false, "provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "trust": "0.9", "inferred": false, "deletedbyinference": false}, "qualifier": {"classid": "doi", "classname": "Digital Object Identifier", "schemeid": "dnet:pid_types", "schemename": "dnet:pid_types"}, "value": "10.17026/ar/vt9ya1"}, {"dataInfo": {"invisible": false, "provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "trust": "0.9", "inferred": false, "deletedbyinference": false}, "qualifier": {"classid": "doi", "classname": "Digital Object Identifier", "schemeid": "dnet:pid_types", "schemename": "dnet:pid_types"}, "value": "10.17026/ar/vt9ya1"}, {"dataInfo": {"invisible": false, "provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "trust": "0.9", "inferred": false, "deletedbyinference": false}, "qualifier": {"classid": "doi", "classname": "Digital Object Identifier", "schemeid": "dnet:pid_types", "schemename": "dnet:pid_types"}, "value": "10.17026/ar/vt9ya1"}], "dateofacceptance": {"dataInfo": {"invisible": false, "provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "trust": "0.9", "inferred": false, "deletedbyinference": false}, "value": "2024-01-01"}, "collectedfrom": {"key": "10|re3data_____::730f562f9efe8a3b3742d2da510d4335", "value": "B2FIND"}, "accessright": {"classid": "OPEN", "classname": "Open Access", "schemeid": "dnet:access_modes", "schemename": "dnet:access_modes"}, "instancetype": {"classid": "0021", "classname": "Dataset", "schemeid": "dnet:publication_resource", "schemename": "dnet:publication_resource"}}], "dateofcollection": "2024-05-10T00:02:09+0000", "metaResourceType": {"classid": "Research Data", "classname": "Research Data", "schemeid": "openaire::meta_resource_types", "schemename": "openaire::meta_resource_types"}, "dateoftransformation": "2024-05-26T00:23:54.028Z", "description": [{"dataInfo": {"invisible": false, "provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "trust": "0.9", "inferred": false, "deletedbyinference": false}, "value": "In opdracht van Gemeente Beesel heeft Geonius Archeologie in maart 2023 een Inventariserend Veldonderzoek door middel van Proefsleuven (IVO-P) uitgevoerd voor het plangebied Heijackerstraat te Beesel in de gemeente Beesel. Aanleiding voor het uitvoeren van het archeologisch onderzoek vormt de aanvraag van een omgevingsvergunning voor bouw van 20 nieuwe woningen. Uit het vooronderzoek is gebleken dat het plangebied in een dalvakteterras ligt rondom opgestoven landduinen langsheen de Maas. De bodem bestaat volgens de bodemkaart uit vorstvaaggronden. Het plangebied is in het verleden voor zover kon worden vastgesteld in gebruik geweest als bouwland en is niet bebouwd geweest. Het IVO-O heeft uitgewezen dat de bodemopbouw deels intact is, al lijken in sommige boringen sporen van vergravingen of verstoringen te bevatten. Op grond van de resultaten van het vooronderzoek is een hoge verwachting opgesteld op het voorkomen van archeologische waarden uit het paleolithicum tot aan de vroege middeleeuwen. Voor de periode late middeleeuwen en nieuwe tijd is een lage verwachting opgesteld. Op grond van de resultaten van het vooronderzoek is een IVO-P uitgevoerd. Hierbij is een vindplaats aangetroffen bestaande uit drie subrecente greppels en een tweetal recente verstoringen. De vindplaats is als niet behoudenswaardig gewaardeerd. Aanbevolen is het plangebied vrij te geven voor de geplande ontwikkeling."}], "format": [], "coverage": [], "externalReference": [], "publisher": {"dataInfo": {"invisible": false, "provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "trust": "0.9", "inferred": false, "deletedbyinference": false}, "value": "Data Archiving and Networked Services (DANS)"}, "context": [], "eoscifguidelines": [], "language": {"classid": "und", "classname": "Undetermined", "schemeid": "dnet:languages", "schemename": "dnet:languages"}, "resulttype": {"classid": "dataset", "classname": "dataset", "schemeid": "dnet:result_typologies", "schemename": "dnet:result_typologies"}, "country": [], "extraInfo": [], "originalId": ["oai:easy.dans.knaw.nl:easy-dataset:341200", "50|r3730f562f9e::ace629fb505b6b4343faca03edde1841"], "source": [], "dateofacceptance": {"dataInfo": {"invisible": false, "provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "trust": "0.9", "inferred": false, "deletedbyinference": false}, "value": "2024-01-01"}, "title": [{"dataInfo": {"invisible": false, "provenanceaction": {"classid": "sysimport:crosswalk:repository", "classname": "Harvested", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "trust": "0.9", "inferred": false, "deletedbyinference": false}, "qualifier": {"classid": "main title", "classname": "main title", "schemeid": "dnet:dataCite_title", "schemename": "dnet:dataCite_title"}, "value": "Archeologisch onderzoek IVO-P plangebied Heijackerstraat te Beesel"}]} \ No newline at end of file From 92c3abd5a4e969cc47313b851f2f93f5c7dcfc87 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Wed, 29 May 2024 14:36:49 +0200 Subject: [PATCH 80/97] [graph cleaning] use sparkExecutorMemory to define also the memoryOverhead --- .../eu/dnetlib/dhp/oa/graph/clean/oozie_app/workflow.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/oozie_app/workflow.xml index f6bf053cd..4188cb018 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/clean/oozie_app/workflow.xml @@ -156,6 +156,7 @@ --executor-cores=${sparkExecutorCores} --executor-memory=${sparkExecutorMemory} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -190,6 +191,7 @@ --executor-cores=${sparkExecutorCores} --executor-memory=${sparkExecutorMemory} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -224,6 +226,7 @@ --executor-cores=${sparkExecutorCores} --executor-memory=${sparkExecutorMemory} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -258,6 +261,7 @@ --executor-cores=${sparkExecutorCores} --executor-memory=${sparkExecutorMemory} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -292,6 +296,7 @@ --executor-cores=${sparkExecutorCores} --executor-memory=${sparkExecutorMemory} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -326,6 +331,7 @@ --executor-cores=${sparkExecutorCores} --executor-memory=${sparkExecutorMemory} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -360,6 +366,7 @@ --executor-cores=${sparkExecutorCores} --executor-memory=${sparkExecutorMemory} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -394,6 +401,7 @@ --executor-cores=${sparkExecutorCores} --executor-memory=${sparkExecutorMemory} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} From 1477406ecc06eafcc704c55e2a317a74e13fcf62 Mon Sep 17 00:00:00 2001 From: Miriam Baglioni Date: Wed, 5 Jun 2024 16:20:40 +0200 Subject: [PATCH 81/97] [bulkTag] fixed issue that made project disappear in graph_10_enriched --- .../src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java index 9e1acc7b2..354741690 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java @@ -172,7 +172,7 @@ public class SparkBulkTagJob { .option("compression", "gzip") .json(outputPath + "project"); - readPath(spark, outputPath + "project", Datasource.class) + readPath(spark, outputPath + "project", Project.class) .write() .mode(SaveMode.Overwrite) .option("compression", "gzip") From ec79405cc9e4cbdadeb983be3b01408c259bc751 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 7 Jun 2024 11:30:31 +0200 Subject: [PATCH 82/97] [graph raw] set organization type from openorgs --- .../dhp/oa/graph/raw/MigrateDbEntitiesApplication.java | 1 + .../dnetlib/dhp/oa/graph/sql/queryOpenOrgsForProvision.sql | 7 +++++-- pom.xml | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java index da6885db3..c9a32cde6 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/MigrateDbEntitiesApplication.java @@ -398,6 +398,7 @@ public class MigrateDbEntitiesApplication extends AbstractMigrationApplication i o.setEcsmevalidated(field(Boolean.toString(rs.getBoolean("ecsmevalidated")), info)); o.setEcnutscode(field(Boolean.toString(rs.getBoolean("ecnutscode")), info)); o.setCountry(prepareQualifierSplitting(rs.getString("country"))); + o.setOrganizationType(Organization.OrganizationType.valueOf(rs.getString("typology"))); o.setDataInfo(info); o.setLastupdatetimestamp(lastUpdateTimestamp); diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryOpenOrgsForProvision.sql b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryOpenOrgsForProvision.sql index 0ec303939..16ad9e265 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryOpenOrgsForProvision.sql +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/sql/queryOpenOrgsForProvision.sql @@ -28,7 +28,8 @@ SELECT (array_remove(array_cat(ARRAY[o.ec_internationalorganization], array_agg(od.ec_internationalorganization)), NULL))[1] AS ecinternationalorganization, (array_remove(array_cat(ARRAY[o.ec_enterprise], array_agg(od.ec_enterprise)), NULL))[1] AS ecenterprise, (array_remove(array_cat(ARRAY[o.ec_smevalidated], array_agg(od.ec_smevalidated)), NULL))[1] AS ecsmevalidated, - (array_remove(array_cat(ARRAY[o.ec_nutscode], array_agg(od.ec_nutscode)), NULL))[1] AS ecnutscode + (array_remove(array_cat(ARRAY[o.ec_nutscode], array_agg(od.ec_nutscode)), NULL))[1] AS ecnutscode, + org_types.name AS typology FROM organizations o LEFT OUTER JOIN acronyms a ON (a.id = o.id) LEFT OUTER JOIN urls u ON (u.id = o.id) @@ -37,6 +38,7 @@ FROM organizations o LEFT OUTER JOIN oa_duplicates d ON (o.id = d.local_id AND d.reltype != 'is_different') LEFT OUTER JOIN organizations od ON (d.oa_original_id = od.id) LEFT OUTER JOIN other_ids idup ON (od.id = idup.id) + LEFT OUTER JOIN org_types ON (org_types.val = o.type) WHERE o.status = 'approved' OR o.status = 'suggested' GROUP BY @@ -44,4 +46,5 @@ GROUP BY o.name, o.creation_date, o.modification_date, - o.country; \ No newline at end of file + o.country, + org_types.name; \ No newline at end of file diff --git a/pom.xml b/pom.xml index cc8d509f7..9e554204d 100644 --- a/pom.xml +++ b/pom.xml @@ -960,7 +960,7 @@ 1.1.3 1.7 1.0.7 - [6.1.2] + [6.1.3-SNAPSHOT] cdh5.9.2 3.5 11.0.2 From c7265724182b7747540e69eaec2732799dca2d0f Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Fri, 7 Jun 2024 12:03:26 +0200 Subject: [PATCH 83/97] changed some parameters in OSF test --- .../dhp/collection/plugin/rest/OsfPreprintCollectorTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/OsfPreprintCollectorTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/OsfPreprintCollectorTest.java index 0e64f8bab..a1b723e33 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/OsfPreprintCollectorTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/plugin/rest/OsfPreprintCollectorTest.java @@ -36,8 +36,8 @@ public class OsfPreprintCollectorTest { private final String resultTotalXpath = "/*/*[local-name()='links']/*[local-name()='meta']/*[local-name()='total']"; private final String resumptionParam = "page"; - private final String resumptionType = "page"; - private final String resumptionXpath = "/*/*[local-name()='links']/*[local-name()='next']"; + private final String resumptionType = "scan"; + private final String resumptionXpath = "substring-before(substring-after(/*/*[local-name()='links']/*[local-name()='next'], 'page='), '&')"; private final String resultSizeParam = "page[size]"; private final String resultSizeValue = "100"; From 3776327a8cc3e725af8af49255598681eac83d1f Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Mon, 10 Jun 2024 15:22:33 +0200 Subject: [PATCH 84/97] hostedby patching to work with the updated Crossref contents, resolved conflict --- .../SparkApplyHostedByMapToResult.scala | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/oa/graph/hostedbymap/SparkApplyHostedByMapToResult.scala b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/oa/graph/hostedbymap/SparkApplyHostedByMapToResult.scala index a900fc241..db7edf53e 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/oa/graph/hostedbymap/SparkApplyHostedByMapToResult.scala +++ b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/oa/graph/hostedbymap/SparkApplyHostedByMapToResult.scala @@ -25,27 +25,38 @@ object SparkApplyHostedByMapToResult { val i = p.getInstance().asScala if (i.size == 1) { val inst: Instance = i.head - inst.getHostedby.setKey(ei.getHostedById) - inst.getHostedby.setValue(ei.getName) - if (ei.getOpenAccess) { - inst.setAccessright( - OafMapperUtils.accessRight( - ModelConstants.ACCESS_RIGHT_OPEN, - "Open Access", - ModelConstants.DNET_ACCESS_MODES, - ModelConstants.DNET_ACCESS_MODES - ) - ) - inst.getAccessright.setOpenAccessRoute(OpenAccessRoute.gold) - p.setBestaccessright(OafMapperUtils.createBestAccessRights(p.getInstance())); - } + patchInstance(p, ei, inst) + } else { + val cf = i.map(ii => ii.getCollectedfrom.getValue) + if (cf.contains("Crossref")) { + i.foreach(ii => { + patchInstance(p, ei, ii) + }) + } } } p })(Encoders.bean(classOf[Publication])) } + private def patchInstance(p: Publication, ei: EntityInfo, inst: Instance): Unit = { + inst.getHostedby.setKey(ei.getHostedById) + inst.getHostedby.setValue(ei.getName) + if (ei.getOpenAccess) { + inst.setAccessright( + OafMapperUtils.accessRight( + ModelConstants.ACCESS_RIGHT_OPEN, + "Open Access", + ModelConstants.DNET_ACCESS_MODES, + ModelConstants.DNET_ACCESS_MODES + ) + ) + inst.getAccessright.setOpenAccessRoute(OpenAccessRoute.gold) + p.setBestaccessright(OafMapperUtils.createBestAccessRights(p.getInstance())); + } + } + def main(args: Array[String]): Unit = { val logger: Logger = LoggerFactory.getLogger(getClass) From b0eba210c068219580cfa78c17aa23f2e1e170f8 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Mon, 10 Jun 2024 16:15:07 +0200 Subject: [PATCH 85/97] [actionset promotion] use sparkExecutorMemory to define also the memoryOverhead --- dhp-shade-package/dependency-reduced-pom.xml | 113 ++++++++++++++++++ .../wf/dataset/oozie_app/workflow.xml | 2 + .../wf/datasource/oozie_app/workflow.xml | 1 + .../wf/main/oozie_app/workflow.xml | 1 + .../wf/organization/oozie_app/workflow.xml | 1 + .../oozie_app/workflow.xml | 4 +- .../wf/project/oozie_app/workflow.xml | 1 + .../wf/publication/oozie_app/workflow.xml | 6 +- .../wf/relation/oozie_app/workflow.xml | 3 +- .../wf/software/oozie_app/workflow.xml | 4 +- 10 files changed, 131 insertions(+), 5 deletions(-) create mode 100644 dhp-shade-package/dependency-reduced-pom.xml diff --git a/dhp-shade-package/dependency-reduced-pom.xml b/dhp-shade-package/dependency-reduced-pom.xml new file mode 100644 index 000000000..04843072f --- /dev/null +++ b/dhp-shade-package/dependency-reduced-pom.xml @@ -0,0 +1,113 @@ + + + + dhp + eu.dnetlib.dhp + 1.2.5-SNAPSHOT + + 4.0.0 + dhp-shade-package + This module create a jar of all module dependencies + + + + maven-shade-plugin + + + package + + shade + + + + + eu.dnetlib.dhp.oa.dedup.SparkCreateSimRels + + + + META-INF/cxf/bus-extensions.txt + + + + + *:* + + META-INF/maven/** + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + com + repackaged.com.google.common + + com.google.common.** + + + + + + + + + + + + org.projectlombok + lombok + 1.18.28 + provided + + + org.junit.jupiter + junit-jupiter + 5.6.1 + test + + + junit-jupiter-api + org.junit.jupiter + + + junit-jupiter-params + org.junit.jupiter + + + junit-jupiter-engine + org.junit.jupiter + + + + + org.mockito + mockito-core + 3.3.3 + test + + + byte-buddy + net.bytebuddy + + + byte-buddy-agent + net.bytebuddy + + + + + org.mockito + mockito-junit-jupiter + 3.3.3 + test + + + + + DHPSite + ${dhp.site.stage.path}/dhp-common + + + diff --git a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/dataset/oozie_app/workflow.xml b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/dataset/oozie_app/workflow.xml index 5401b45ca..b1bc1d6e1 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/dataset/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/dataset/oozie_app/workflow.xml @@ -103,6 +103,7 @@ --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -156,6 +157,7 @@ --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} diff --git a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/datasource/oozie_app/workflow.xml b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/datasource/oozie_app/workflow.xml index f9bd66ae3..9a84f4708 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/datasource/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/datasource/oozie_app/workflow.xml @@ -95,6 +95,7 @@ --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} diff --git a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/main/oozie_app/workflow.xml b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/main/oozie_app/workflow.xml index 393f04e89..65ddd402b 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/main/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/main/oozie_app/workflow.xml @@ -125,6 +125,7 @@ --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} diff --git a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/organization/oozie_app/workflow.xml b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/organization/oozie_app/workflow.xml index ebfdeee31..0d73b498d 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/organization/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/organization/oozie_app/workflow.xml @@ -95,6 +95,7 @@ --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} diff --git a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/otherresearchproduct/oozie_app/workflow.xml b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/otherresearchproduct/oozie_app/workflow.xml index 02399ed9b..ca8362c9b 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/otherresearchproduct/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/otherresearchproduct/oozie_app/workflow.xml @@ -103,6 +103,7 @@ --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -155,11 +156,12 @@ --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.sql.shuffle.partitions=2560 + --conf spark.sql.shuffle.partitions=8000 --inputGraphTablePath${workingDir}/otherresearchproduct --graphTableClassNameeu.dnetlib.dhp.schema.oaf.OtherResearchProduct diff --git a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/project/oozie_app/workflow.xml b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/project/oozie_app/workflow.xml index 57c2357b4..37310da79 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/project/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/project/oozie_app/workflow.xml @@ -95,6 +95,7 @@ --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} diff --git a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/publication/oozie_app/workflow.xml b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/publication/oozie_app/workflow.xml index 92b114776..a4b0b237c 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/publication/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/publication/oozie_app/workflow.xml @@ -103,11 +103,12 @@ --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.sql.shuffle.partitions=7000 + --conf spark.sql.shuffle.partitions=15000 --inputGraphTablePath${inputGraphRootPath}/publication --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Publication @@ -156,11 +157,12 @@ --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.sql.shuffle.partitions=7000 + --conf spark.sql.shuffle.partitions=15000 --inputGraphTablePath${workingDir}/publication --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Publication diff --git a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/relation/oozie_app/workflow.xml b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/relation/oozie_app/workflow.xml index e9e5f0b45..f72847ba8 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/relation/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/relation/oozie_app/workflow.xml @@ -95,11 +95,12 @@ --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.sql.shuffle.partitions=10000 + --conf spark.sql.shuffle.partitions=15000 --inputGraphTablePath${inputGraphRootPath}/relation --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Relation diff --git a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/software/oozie_app/workflow.xml b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/software/oozie_app/workflow.xml index 1d36ddf94..dbe8a63c1 100644 --- a/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/software/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-actionmanager/src/main/resources/eu/dnetlib/dhp/actionmanager/wf/software/oozie_app/workflow.xml @@ -103,6 +103,7 @@ --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} @@ -155,11 +156,12 @@ --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.sql.shuffle.partitions=2560 + --conf spark.sql.shuffle.partitions=4000 --inputGraphTablePath${workingDir}/software --graphTableClassNameeu.dnetlib.dhp.schema.oaf.Software From 85c1eae7e0da86e25aa2da74d90ac82ecf150e27 Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Mon, 10 Jun 2024 19:03:30 +0200 Subject: [PATCH 86/97] Fixes for pagination strategy looping at end of download --- .../collection/plugin/rest/RestIterator.java | 58 +++++++++++++------ 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java index 2518fd92f..9037a454e 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java @@ -12,6 +12,8 @@ import java.util.Iterator; import java.util.Map; import java.util.Queue; import java.util.concurrent.PriorityBlockingQueue; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import javax.xml.transform.OutputKeys; import javax.xml.transform.Transformer; @@ -19,16 +21,10 @@ import javax.xml.transform.TransformerConfigurationException; import javax.xml.transform.TransformerFactory; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpression; -import javax.xml.xpath.XPathExpressionException; -import javax.xml.xpath.XPathFactory; +import javax.xml.xpath.*; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpHeaders; -import org.apache.http.entity.ContentType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Node; @@ -51,7 +47,6 @@ import eu.dnetlib.dhp.common.collection.HttpClientParams; * */ public class RestIterator implements Iterator { - private static final Logger log = LoggerFactory.getLogger(RestIterator.class); public static final String UTF_8 = "UTF-8"; private static final int MAX_ATTEMPTS = 5; @@ -60,11 +55,15 @@ public class RestIterator implements Iterator { private final String AUTHBASIC = "basic"; + private static final String XML_HEADER = ""; + private static final String EMPTY_XML = XML_HEADER + "<" + JsonUtils.XML_WRAP_TAG + ">"; + private final String baseUrl; private final String resumptionType; private final String resumptionParam; private final String resultFormatValue; - private String queryParams; + private String queryParams = ""; private final int resultSizeValue; private int resumptionInt = 0; // integer resumption token (first record to harvest) private int resultTotal = -1; @@ -158,7 +157,12 @@ public class RestIterator implements Iterator { } private void initQueue() { - this.query = this.baseUrl + "?" + this.queryParams + this.querySize + this.queryFormat; + if (queryParams.equals("") && querySize.equals("") && queryFormat.equals("")) { + query = baseUrl; + } else { + query = baseUrl + "?" + queryParams + querySize + queryFormat; + } + log.info("REST calls starting with {}", this.query); } @@ -219,9 +223,8 @@ public class RestIterator implements Iterator { try { String resultJson; - String resultXml = ""; + String resultXml = XML_HEADER; String nextQuery = ""; - final String emptyXml = resultXml + "<" + JsonUtils.XML_WRAP_TAG + ">"; Node resultNode = null; NodeList nodeList = null; String qUrlArgument = ""; @@ -236,6 +239,21 @@ public class RestIterator implements Iterator { } } + // find pagination page start number in queryParam and remove before start the first query + if ((resumptionType.toLowerCase().equals("pagination") || resumptionType.toLowerCase().equals("page")) + && (query.contains("paginationStart="))) { + + final Matcher m = Pattern.compile("paginationStart=([0-9]+)").matcher(query); + m.find(); // guaranteed to be true for this regex + + String[] pageVal = m.group(0).split("="); + pagination = Integer.parseInt(pageVal[1]); + + // remove page start number from queryParams + query = query.replaceFirst("&?paginationStart=[0-9]+", ""); + + } + try { log.info("requesting URL [{}]", query); @@ -261,7 +279,7 @@ public class RestIterator implements Iterator { this.resultStream = IOUtils.toInputStream(resultXml, UTF_8); } - if (!(emptyXml).equalsIgnoreCase(resultXml)) { + if (!isEmptyXml(resultXml)) { resultNode = (Node) this.xpath .evaluate("/", new InputSource(this.resultStream), XPathConstants.NODE); nodeList = (NodeList) this.xprEntity.evaluate(resultNode, XPathConstants.NODESET); @@ -270,8 +288,7 @@ public class RestIterator implements Iterator { final StringWriter sw = new StringWriter(); this.transformer.transform(new DOMSource(nodeList.item(i)), new StreamResult(sw)); final String toEnqueue = sw.toString(); - if ((toEnqueue == null) || StringUtils.isBlank(toEnqueue) - || emptyXml.equalsIgnoreCase(toEnqueue)) { + if ((toEnqueue == null) || StringUtils.isBlank(toEnqueue) || isEmptyXml(toEnqueue)) { log .warn( "The following record resulted in empty item for the feeding queue: {}", resultXml); @@ -299,6 +316,7 @@ public class RestIterator implements Iterator { throw new CollectorException("Mode: discover, Param 'resultSizeValue' is less than 2"); } qUrlArgument = qUrl.getQuery(); + final String[] arrayQUrlArgument = qUrlArgument.split("&"); for (final String arrayUrlArgStr : arrayQUrlArgument) { if (arrayUrlArgStr.startsWith(this.resumptionParam)) { @@ -312,7 +330,7 @@ public class RestIterator implements Iterator { } } - if (((emptyXml).equalsIgnoreCase(resultXml)) + if (isEmptyXml(resultXml) || ((nodeList != null) && (nodeList.getLength() < this.resultSizeValue))) { // resumptionStr = ""; if (nodeList != null) { @@ -331,13 +349,13 @@ public class RestIterator implements Iterator { case "pagination": case "page": // pagination, iterate over page numbers - this.pagination += 1; - if (nodeList != null) { + if (nodeList != null && nodeList.getLength() > 0) { this.discoverResultSize += nodeList.getLength(); } else { this.resultTotal = this.discoverResultSize; this.pagination = this.discoverResultSize; } + this.pagination += 1; this.resumptionInt = this.pagination; this.resumptionStr = Integer.toString(this.resumptionInt); break; @@ -415,6 +433,10 @@ public class RestIterator implements Iterator { } + private boolean isEmptyXml(String s) { + return EMPTY_XML.equalsIgnoreCase(s); + } + private boolean isInteger(final String s) { boolean isValidInteger = false; try { From 9da006e98ceb53a120be36b5529f90ec765639b6 Mon Sep 17 00:00:00 2001 From: Miriam Baglioni Date: Tue, 11 Jun 2024 10:28:32 +0200 Subject: [PATCH 87/97] [SDGFoSActionSet]remove datainfo for the result. It is not needed (qualifier.classid = UPDATE) useless since subject do not go at the level of the instance --- .../PrepareFOSSparkJob.java | 14 +------------- .../PrepareSDGSparkJob.java | 14 +------------- 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareFOSSparkJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareFOSSparkJob.java index c248423d4..ec957a208 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareFOSSparkJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareFOSSparkJob.java @@ -115,19 +115,7 @@ public class PrepareFOSSparkJob implements Serializable { .forEach( l -> add(sbjs, getSubject(l, FOS_CLASS_ID, FOS_CLASS_NAME, UPDATE_SUBJECT_FOS_CLASS_ID, true))); r.setSubject(sbjs); - r - .setDataInfo( - OafMapperUtils - .dataInfo( - false, null, true, - false, - OafMapperUtils - .qualifier( - ModelConstants.PROVENANCE_ENRICH, - null, - ModelConstants.DNET_PROVENANCE_ACTIONS, - ModelConstants.DNET_PROVENANCE_ACTIONS), - null)); + return r; } diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareSDGSparkJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareSDGSparkJob.java index bfdf14234..a88607986 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareSDGSparkJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareSDGSparkJob.java @@ -81,19 +81,7 @@ public class PrepareSDGSparkJob implements Serializable { s -> sbjs .add(getSubject(s.getSbj(), SDG_CLASS_ID, SDG_CLASS_NAME, UPDATE_SUBJECT_SDG_CLASS_ID))); r.setSubject(sbjs); - r - .setDataInfo( - OafMapperUtils - .dataInfo( - false, null, true, - false, - OafMapperUtils - .qualifier( - ModelConstants.PROVENANCE_ENRICH, - null, - ModelConstants.DNET_PROVENANCE_ACTIONS, - ModelConstants.DNET_PROVENANCE_ACTIONS), - null)); + return r; }, Encoders.bean(Result.class)) .write() From a8d68c9d294518734d9e6ac724ab40d9e4ff5d3d Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 11 Jun 2024 12:40:50 +0200 Subject: [PATCH 88/97] avoid NPEs --- .../oaf/utils/MergeEntitiesComparator.java | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeEntitiesComparator.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeEntitiesComparator.java index 5792fc10f..ff6c2689a 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeEntitiesComparator.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/MergeEntitiesComparator.java @@ -1,13 +1,9 @@ package eu.dnetlib.dhp.schema.oaf.utils; -import static eu.dnetlib.dhp.schema.common.ModelConstants.CROSSREF_ID; - import java.util.*; -import java.util.stream.Collectors; import eu.dnetlib.dhp.schema.common.ModelConstants; -import eu.dnetlib.dhp.schema.oaf.KeyValue; import eu.dnetlib.dhp.schema.oaf.Oaf; import eu.dnetlib.dhp.schema.oaf.OafEntity; import eu.dnetlib.dhp.schema.oaf.Result; @@ -42,17 +38,23 @@ public class MergeEntitiesComparator implements Comparator { int res = 0; // pid authority - int cfp1 = left - .getCollectedfrom() - .stream() - .map(kv -> PID_AUTHORITIES.indexOf(kv.getKey())) - .max(Integer::compare) + int cfp1 = Optional + .ofNullable(left.getCollectedfrom()) + .map( + cf -> cf + .stream() + .map(kv -> PID_AUTHORITIES.indexOf(kv.getKey())) + .max(Integer::compare) + .orElse(-1)) .orElse(-1); - int cfp2 = right - .getCollectedfrom() - .stream() - .map(kv -> PID_AUTHORITIES.indexOf(kv.getKey())) - .max(Integer::compare) + int cfp2 = Optional + .ofNullable(right.getCollectedfrom()) + .map( + cf -> cf + .stream() + .map(kv -> PID_AUTHORITIES.indexOf(kv.getKey())) + .max(Integer::compare) + .orElse(-1)) .orElse(-1); if (cfp1 >= 0 && cfp1 > cfp2) { From 11fe3a4fe0af870ee519de5dd546d2d28483c4a4 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 11 Jun 2024 14:21:01 +0200 Subject: [PATCH 89/97] [graph resolution] use sparkExecutorMemory to define also the memoryOverhead --- .../eu/dnetlib/dhp/oa/graph/resolution/oozie_app/workflow.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/resolution/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/resolution/oozie_app/workflow.xml index 74e792f07..916a9f2b1 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/resolution/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/resolution/oozie_app/workflow.xml @@ -45,6 +45,7 @@ --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.shuffle.partitions=15000 --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} @@ -79,6 +80,7 @@ --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=${sparkExecutorMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.shuffle.partitions=10000 --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} From d90cb099b8deababf4428626fd9e7f164fbe2ae1 Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Tue, 11 Jun 2024 20:23:44 +0200 Subject: [PATCH 90/97] Fix for paginationStart parameter management --- .../eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java index 9037a454e..caef266d7 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java @@ -249,7 +249,8 @@ public class RestIterator implements Iterator { String[] pageVal = m.group(0).split("="); pagination = Integer.parseInt(pageVal[1]); - // remove page start number from queryParams + // remove page start number from query and queryParams + queryParams = queryParams.replaceFirst("&?paginationStart=[0-9]+", ""); query = query.replaceFirst("&?paginationStart=[0-9]+", ""); } From 9bf2bda1c6916ea45675a71e9c115c83182371bd Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Wed, 12 Jun 2024 13:28:51 +0200 Subject: [PATCH 91/97] Fix: next returned a null value at end of stream --- .../collection/plugin/rest/RestIterator.java | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java index caef266d7..0895d5f43 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/plugin/rest/RestIterator.java @@ -176,19 +176,6 @@ public class RestIterator implements Iterator { */ @Override public boolean hasNext() { - if (this.recordQueue.isEmpty() && this.query.isEmpty()) { - disconnect(); - return false; - } - return true; - } - - /* - * (non-Javadoc) - * @see java.util.Iterator#next() - */ - @Override - public String next() { synchronized (this.recordQueue) { while (this.recordQueue.isEmpty() && !this.query.isEmpty()) { try { @@ -198,6 +185,23 @@ public class RestIterator implements Iterator { throw new RuntimeException(e); } } + + if (!this.recordQueue.isEmpty()) { + return true; + } + + disconnect(); + return false; + } + } + + /* + * (non-Javadoc) + * @see java.util.Iterator#next() + */ + @Override + public String next() { + synchronized (this.recordQueue) { return this.recordQueue.poll(); } } From d942a1101bfb60b536f56e39ab9a7a00925ac6f3 Mon Sep 17 00:00:00 2001 From: LSmyrnaios Date: Fri, 14 Jun 2024 12:14:38 +0300 Subject: [PATCH 92/97] Miscellaneous updates to the copying operation to Impala Cluster: - Show some counts and the elapsed time for various sub-tasks. - Code polishing. --- .../oozie_app/copyDataToImpalaCluster.sh | 35 +++++++++++-------- .../oozie_app/copyDataToImpalaCluster.sh | 35 +++++++++++-------- .../oozie_app/copyDataToImpalaCluster.sh | 35 +++++++++++-------- .../oozie_app/copyDataToImpalaCluster.sh | 35 +++++++++++-------- 4 files changed, 80 insertions(+), 60 deletions(-) diff --git a/dhp-workflows/dhp-stats-hist-snaps/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-hist-snaps/oozie_app/copyDataToImpalaCluster.sh b/dhp-workflows/dhp-stats-hist-snaps/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-hist-snaps/oozie_app/copyDataToImpalaCluster.sh index ca0f7a643..55deada40 100644 --- a/dhp-workflows/dhp-stats-hist-snaps/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-hist-snaps/oozie_app/copyDataToImpalaCluster.sh +++ b/dhp-workflows/dhp-stats-hist-snaps/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-hist-snaps/oozie_app/copyDataToImpalaCluster.sh @@ -55,20 +55,20 @@ function print_elapsed_time() hours=$((elapsed_time / 3600)) minutes=$(((elapsed_time % 3600) / 60)) seconds=$((elapsed_time % 60)) - printf "\nElapsed time: %02d:%02d:%02d\n\n" $hours $minutes $seconds + printf "%02d:%02d:%02d" $hours $minutes $seconds } function copydb() { db=$1 - start_db_time=$(date +%s) echo -e "\nStart processing db: '${db}'..\n" + start_db_time=$(date +%s) # Delete the old DB from Impala cluster (if exists). impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "drop database if exists ${db} cascade" |& tee error.log # impala-shell prints all logs in stderr, so wee need to capture them and put them in a file, in order to perform "grep" on them later log_errors=`cat error.log | grep -E "WARN|ERROR|FAILED"` if [ -n "$log_errors" ]; then - echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN DROPPING THE OLD DATABASE! EXITING...\n\n" + echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN DROPPING THE OLD DATABASE FROM IMPALA CLUSTER! EXITING...\n\n" rm -f error.log if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then exit 2 @@ -78,6 +78,7 @@ function copydb() { fi echo -e "\n\nCopying files of '${db}', from Ocean to Impala cluster..\n" + start_file_transfer_time=$(date +%s) # Using max-bandwidth of: 70 * 150 Mb/s = 10.5 Gb/s # Using max memory of: 70 * 6144 = 430 Gb # Using 1MB as a buffer-size. @@ -93,7 +94,7 @@ function copydb() { ${OCEAN_HDFS_NODE}/user/hive/warehouse/${db}.db ${IMPALA_HDFS_DB_BASE_PATH} if [ $? -eq 0 ]; then # Check the exit status of the "hadoop distcp" command. - echo -e "\nSuccessfully copied the files of '${db}' from Ocean to Impala cluster.\n" + echo -e "\nSuccessfully copied the files of '${db}' from Ocean to Impala cluster, after: $(print_elapsed_time start_file_transfer_time)\n" else echo -e "\n\nERROR: FAILED TO TRANSFER THE FILES OF '${db}', WITH 'hadoop distcp'. GOT EXIT STATUS: $?\n\n" rm -f error.log @@ -118,6 +119,7 @@ function copydb() { fi echo -e "\nCreating schema for db: '${db}'\n" + start_create_schema_time=$(date +%s) # create the new database (with the same name) impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "create database ${db}" @@ -128,7 +130,8 @@ function copydb() { all_create_view_statements=() num_tables=0 - entities_on_ocean=`hive -e "show tables in ${db};" | sed 's/WARN:.*//g'` # Get the tables and views without any potential the "WARN" logs. + entities_on_ocean=(`hive -e "show tables in ${db};" | sed 's/WARN:.*//g'`) # Get the tables and views without any potential the "WARN" logs. + echo -e "\nGoing to create ${#entities_on_ocean[@]} entities for db '${db}'..\n" for i in ${entities_on_ocean[@]}; do # Use un-quoted values, as the elements are single-words. # Check if this is a view by showing the create-statement where it should print "create view" for a view, not the "create table". Unfortunately, there is no "show views" command. create_entity_statement=`hive --database ${db} -e "show create table ${i};"` # We need to use the "--database", instead of including it inside the query, in order to return the statements with the '`' chars being in the right place to be used by impala-shell. However, we need to add the db-name in the "CREATE VIEW view_name" statement. @@ -152,8 +155,9 @@ function copydb() { if [ -n "$log_errors" ]; then echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN CREATING TABLE '${i}'!\n\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then + rm -f error.log exit 6 - fi + fi # This error is not FATAL, do we do not return from this function, in normal circumstances. fi fi fi @@ -208,8 +212,11 @@ function copydb() { previous_num_of_views_to_retry=$new_num_of_views_to_retry done + entities_on_impala=(`impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} --delimited -q "show tables in ${db}"`) + echo -e "\nThe schema of db '${db}', along with ${#entities_on_impala[@]} entities have been created, on Impala cluster, after: $(print_elapsed_time start_create_schema_time)\n" + + start_compute_stats_time=$(date +%s) echo -e "\nComputing stats for tables..\n" - entities_on_impala=`impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} --delimited -q "show tables in ${db}"` for i in ${entities_on_impala[@]}; do # Use un-quoted values, as the elemetns are single-words. # Taking the create table statement from the Ocean cluster, just to check if its a view, as the output is easier than using impala-shell from Impala cluster. create_view_statement=`hive -e "show create table ${db}.${i};" | grep "CREATE VIEW"` # This grep works here, as we do not want to match multiple-lines. @@ -221,20 +228,18 @@ function copydb() { fi done + echo -e "\nFinished computing stats for tables, after: $(print_elapsed_time start_compute_stats_time)\n" + rm -f error.log # Cleanup the temp log-file. + # Check if the entities in both clusters are the same, down to the exact names, not just the counts. (they are sorted in the same way both in hive and impala) - if [ "${entities_on_impala[@]}" == "${entities_on_ocean[@]}" ]; then - echo -e "\nAll entities have been copied to Impala cluster.\n" + if [[ "${entities_on_impala[@]}" == "${entities_on_ocean[@]}" ]]; then + echo -e "\nAll entities have been copied to Impala cluster.\n\nFinished processing db: '${db}', after: $(print_elapsed_time start_db_time)\n" else - echo -e "\n\nERROR: 1 OR MORE ENTITIES OF DB '${db}' FAILED TO BE COPIED TO IMPALA CLUSTER!\n\n" - rm -f error.log + echo -e "\n\nERROR: $((${#entities_on_ocean[@]} - ${#entities_on_impala[@]})) ENTITIES OF DB '${db}' FAILED TO BE COPIED TO IMPALA CLUSTER!\n\n\nFinished processing db: '${db}', after: $(print_elapsed_time start_db_time)\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then exit 8 fi fi - - rm -f error.log - echo -e "\n\nFinished processing db: ${db}\n" - print_elapsed_time start_db_time } diff --git a/dhp-workflows/dhp-stats-monitor-irish/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor-irish/oozie_app/copyDataToImpalaCluster.sh b/dhp-workflows/dhp-stats-monitor-irish/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor-irish/oozie_app/copyDataToImpalaCluster.sh index ca0f7a643..55deada40 100644 --- a/dhp-workflows/dhp-stats-monitor-irish/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor-irish/oozie_app/copyDataToImpalaCluster.sh +++ b/dhp-workflows/dhp-stats-monitor-irish/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor-irish/oozie_app/copyDataToImpalaCluster.sh @@ -55,20 +55,20 @@ function print_elapsed_time() hours=$((elapsed_time / 3600)) minutes=$(((elapsed_time % 3600) / 60)) seconds=$((elapsed_time % 60)) - printf "\nElapsed time: %02d:%02d:%02d\n\n" $hours $minutes $seconds + printf "%02d:%02d:%02d" $hours $minutes $seconds } function copydb() { db=$1 - start_db_time=$(date +%s) echo -e "\nStart processing db: '${db}'..\n" + start_db_time=$(date +%s) # Delete the old DB from Impala cluster (if exists). impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "drop database if exists ${db} cascade" |& tee error.log # impala-shell prints all logs in stderr, so wee need to capture them and put them in a file, in order to perform "grep" on them later log_errors=`cat error.log | grep -E "WARN|ERROR|FAILED"` if [ -n "$log_errors" ]; then - echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN DROPPING THE OLD DATABASE! EXITING...\n\n" + echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN DROPPING THE OLD DATABASE FROM IMPALA CLUSTER! EXITING...\n\n" rm -f error.log if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then exit 2 @@ -78,6 +78,7 @@ function copydb() { fi echo -e "\n\nCopying files of '${db}', from Ocean to Impala cluster..\n" + start_file_transfer_time=$(date +%s) # Using max-bandwidth of: 70 * 150 Mb/s = 10.5 Gb/s # Using max memory of: 70 * 6144 = 430 Gb # Using 1MB as a buffer-size. @@ -93,7 +94,7 @@ function copydb() { ${OCEAN_HDFS_NODE}/user/hive/warehouse/${db}.db ${IMPALA_HDFS_DB_BASE_PATH} if [ $? -eq 0 ]; then # Check the exit status of the "hadoop distcp" command. - echo -e "\nSuccessfully copied the files of '${db}' from Ocean to Impala cluster.\n" + echo -e "\nSuccessfully copied the files of '${db}' from Ocean to Impala cluster, after: $(print_elapsed_time start_file_transfer_time)\n" else echo -e "\n\nERROR: FAILED TO TRANSFER THE FILES OF '${db}', WITH 'hadoop distcp'. GOT EXIT STATUS: $?\n\n" rm -f error.log @@ -118,6 +119,7 @@ function copydb() { fi echo -e "\nCreating schema for db: '${db}'\n" + start_create_schema_time=$(date +%s) # create the new database (with the same name) impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "create database ${db}" @@ -128,7 +130,8 @@ function copydb() { all_create_view_statements=() num_tables=0 - entities_on_ocean=`hive -e "show tables in ${db};" | sed 's/WARN:.*//g'` # Get the tables and views without any potential the "WARN" logs. + entities_on_ocean=(`hive -e "show tables in ${db};" | sed 's/WARN:.*//g'`) # Get the tables and views without any potential the "WARN" logs. + echo -e "\nGoing to create ${#entities_on_ocean[@]} entities for db '${db}'..\n" for i in ${entities_on_ocean[@]}; do # Use un-quoted values, as the elements are single-words. # Check if this is a view by showing the create-statement where it should print "create view" for a view, not the "create table". Unfortunately, there is no "show views" command. create_entity_statement=`hive --database ${db} -e "show create table ${i};"` # We need to use the "--database", instead of including it inside the query, in order to return the statements with the '`' chars being in the right place to be used by impala-shell. However, we need to add the db-name in the "CREATE VIEW view_name" statement. @@ -152,8 +155,9 @@ function copydb() { if [ -n "$log_errors" ]; then echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN CREATING TABLE '${i}'!\n\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then + rm -f error.log exit 6 - fi + fi # This error is not FATAL, do we do not return from this function, in normal circumstances. fi fi fi @@ -208,8 +212,11 @@ function copydb() { previous_num_of_views_to_retry=$new_num_of_views_to_retry done + entities_on_impala=(`impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} --delimited -q "show tables in ${db}"`) + echo -e "\nThe schema of db '${db}', along with ${#entities_on_impala[@]} entities have been created, on Impala cluster, after: $(print_elapsed_time start_create_schema_time)\n" + + start_compute_stats_time=$(date +%s) echo -e "\nComputing stats for tables..\n" - entities_on_impala=`impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} --delimited -q "show tables in ${db}"` for i in ${entities_on_impala[@]}; do # Use un-quoted values, as the elemetns are single-words. # Taking the create table statement from the Ocean cluster, just to check if its a view, as the output is easier than using impala-shell from Impala cluster. create_view_statement=`hive -e "show create table ${db}.${i};" | grep "CREATE VIEW"` # This grep works here, as we do not want to match multiple-lines. @@ -221,20 +228,18 @@ function copydb() { fi done + echo -e "\nFinished computing stats for tables, after: $(print_elapsed_time start_compute_stats_time)\n" + rm -f error.log # Cleanup the temp log-file. + # Check if the entities in both clusters are the same, down to the exact names, not just the counts. (they are sorted in the same way both in hive and impala) - if [ "${entities_on_impala[@]}" == "${entities_on_ocean[@]}" ]; then - echo -e "\nAll entities have been copied to Impala cluster.\n" + if [[ "${entities_on_impala[@]}" == "${entities_on_ocean[@]}" ]]; then + echo -e "\nAll entities have been copied to Impala cluster.\n\nFinished processing db: '${db}', after: $(print_elapsed_time start_db_time)\n" else - echo -e "\n\nERROR: 1 OR MORE ENTITIES OF DB '${db}' FAILED TO BE COPIED TO IMPALA CLUSTER!\n\n" - rm -f error.log + echo -e "\n\nERROR: $((${#entities_on_ocean[@]} - ${#entities_on_impala[@]})) ENTITIES OF DB '${db}' FAILED TO BE COPIED TO IMPALA CLUSTER!\n\n\nFinished processing db: '${db}', after: $(print_elapsed_time start_db_time)\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then exit 8 fi fi - - rm -f error.log - echo -e "\n\nFinished processing db: ${db}\n" - print_elapsed_time start_db_time } diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh index dd2203eef..43498abd2 100644 --- a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh +++ b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh @@ -55,20 +55,20 @@ function print_elapsed_time() hours=$((elapsed_time / 3600)) minutes=$(((elapsed_time % 3600) / 60)) seconds=$((elapsed_time % 60)) - printf "\nElapsed time: %02d:%02d:%02d\n\n" $hours $minutes $seconds + printf "%02d:%02d:%02d" $hours $minutes $seconds } function copydb() { db=$1 - start_db_time=$(date +%s) echo -e "\nStart processing db: '${db}'..\n" + start_db_time=$(date +%s) # Delete the old DB from Impala cluster (if exists). impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "drop database if exists ${db} cascade" |& tee error.log # impala-shell prints all logs in stderr, so wee need to capture them and put them in a file, in order to perform "grep" on them later log_errors=`cat error.log | grep -E "WARN|ERROR|FAILED"` if [ -n "$log_errors" ]; then - echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN DROPPING THE OLD DATABASE! EXITING...\n\n" + echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN DROPPING THE OLD DATABASE FROM IMPALA CLUSTER! EXITING...\n\n" rm -f error.log if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then exit 2 @@ -78,6 +78,7 @@ function copydb() { fi echo -e "\n\nCopying files of '${db}', from Ocean to Impala cluster..\n" + start_file_transfer_time=$(date +%s) # Using max-bandwidth of: 70 * 150 Mb/s = 10.5 Gb/s # Using max memory of: 70 * 6144 = 430 Gb # Using 1MB as a buffer-size. @@ -93,7 +94,7 @@ function copydb() { ${OCEAN_HDFS_NODE}/user/hive/warehouse/${db}.db ${IMPALA_HDFS_DB_BASE_PATH} if [ $? -eq 0 ]; then # Check the exit status of the "hadoop distcp" command. - echo -e "\nSuccessfully copied the files of '${db}' from Ocean to Impala cluster.\n" + echo -e "\nSuccessfully copied the files of '${db}' from Ocean to Impala cluster, after: $(print_elapsed_time start_file_transfer_time)\n" else echo -e "\n\nERROR: FAILED TO TRANSFER THE FILES OF '${db}', WITH 'hadoop distcp'. GOT EXIT STATUS: $?\n\n" rm -f error.log @@ -118,6 +119,7 @@ function copydb() { fi echo -e "\nCreating schema for db: '${db}'\n" + start_create_schema_time=$(date +%s) # create the new database (with the same name) impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "create database ${db}" @@ -128,7 +130,8 @@ function copydb() { all_create_view_statements=() num_tables=0 - entities_on_ocean=`hive -e "show tables in ${db};" | sed 's/WARN:.*//g'` # Get the tables and views without any potential the "WARN" logs. + entities_on_ocean=(`hive -e "show tables in ${db};" | sed 's/WARN:.*//g'`) # Get the tables and views without any potential the "WARN" logs. + echo -e "\nGoing to create ${#entities_on_ocean[@]} entities for db '${db}'..\n" for i in ${entities_on_ocean[@]}; do # Use un-quoted values, as the elements are single-words. # Check if this is a view by showing the create-statement where it should print "create view" for a view, not the "create table". Unfortunately, there is no "show views" command. create_entity_statement=`hive --database ${db} -e "show create table ${i};"` # We need to use the "--database", instead of including it inside the query, in order to return the statements with the '`' chars being in the right place to be used by impala-shell. However, we need to add the db-name in the "CREATE VIEW view_name" statement. @@ -152,8 +155,9 @@ function copydb() { if [ -n "$log_errors" ]; then echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN CREATING TABLE '${i}'!\n\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then + rm -f error.log exit 6 - fi + fi # This error is not FATAL, do we do not return from this function, in normal circumstances. fi fi fi @@ -208,8 +212,11 @@ function copydb() { previous_num_of_views_to_retry=$new_num_of_views_to_retry done + entities_on_impala=(`impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} --delimited -q "show tables in ${db}"`) + echo -e "\nThe schema of db '${db}', along with ${#entities_on_impala[@]} entities have been created, on Impala cluster, after: $(print_elapsed_time start_create_schema_time)\n" + + start_compute_stats_time=$(date +%s) echo -e "\nComputing stats for tables..\n" - entities_on_impala=`impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} --delimited -q "show tables in ${db}"` for i in ${entities_on_impala[@]}; do # Use un-quoted values, as the elemetns are single-words. # Taking the create table statement from the Ocean cluster, just to check if its a view, as the output is easier than using impala-shell from Impala cluster. create_view_statement=`hive -e "show create table ${db}.${i};" | grep "CREATE VIEW"` # This grep works here, as we do not want to match multiple-lines. @@ -221,20 +228,18 @@ function copydb() { fi done + echo -e "\nFinished computing stats for tables, after: $(print_elapsed_time start_compute_stats_time)\n" + rm -f error.log # Cleanup the temp log-file. + # Check if the entities in both clusters are the same, down to the exact names, not just the counts. (they are sorted in the same way both in hive and impala) - if [ "${entities_on_impala[@]}" == "${entities_on_ocean[@]}" ]; then - echo -e "\nAll entities have been copied to Impala cluster.\n" + if [[ "${entities_on_impala[@]}" == "${entities_on_ocean[@]}" ]]; then + echo -e "\nAll entities have been copied to Impala cluster.\n\nFinished processing db: '${db}', after: $(print_elapsed_time start_db_time)\n" else - echo -e "\n\nERROR: 1 OR MORE ENTITIES OF DB '${db}' FAILED TO BE COPIED TO IMPALA CLUSTER!\n\n" - rm -f error.log + echo -e "\n\nERROR: $((${#entities_on_ocean[@]} - ${#entities_on_impala[@]})) ENTITIES OF DB '${db}' FAILED TO BE COPIED TO IMPALA CLUSTER!\n\n\nFinished processing db: '${db}', after: $(print_elapsed_time start_db_time)\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then exit 8 fi fi - - rm -f error.log - echo -e "\n\nFinished processing db: ${db}\n" - print_elapsed_time start_db_time } diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh index 918775f49..1d5842d06 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh @@ -57,20 +57,20 @@ function print_elapsed_time() hours=$((elapsed_time / 3600)) minutes=$(((elapsed_time % 3600) / 60)) seconds=$((elapsed_time % 60)) - printf "\nElapsed time: %02d:%02d:%02d\n\n" $hours $minutes $seconds + printf "%02d:%02d:%02d" $hours $minutes $seconds } function copydb() { db=$1 - start_db_time=$(date +%s) echo -e "\nStart processing db: '${db}'..\n" + start_db_time=$(date +%s) # Delete the old DB from Impala cluster (if exists). impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "drop database if exists ${db} cascade" |& tee error.log # impala-shell prints all logs in stderr, so wee need to capture them and put them in a file, in order to perform "grep" on them later log_errors=`cat error.log | grep -E "WARN|ERROR|FAILED"` if [ -n "$log_errors" ]; then - echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN DROPPING THE OLD DATABASE! EXITING...\n\n" + echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN DROPPING THE OLD DATABASE FROM IMPALA CLUSTER! EXITING...\n\n" rm -f error.log if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then exit 2 @@ -80,6 +80,7 @@ function copydb() { fi echo -e "\n\nCopying files of '${db}', from Ocean to Impala cluster..\n" + start_file_transfer_time=$(date +%s) # Using max-bandwidth of: 70 * 150 Mb/s = 10.5 Gb/s # Using max memory of: 70 * 6144 = 430 Gb # Using 1MB as a buffer-size. @@ -95,7 +96,7 @@ function copydb() { ${OCEAN_HDFS_NODE}/user/hive/warehouse/${db}.db ${IMPALA_HDFS_DB_BASE_PATH} if [ $? -eq 0 ]; then # Check the exit status of the "hadoop distcp" command. - echo -e "\nSuccessfully copied the files of '${db}' from Ocean to Impala cluster.\n" + echo -e "\nSuccessfully copied the files of '${db}' from Ocean to Impala cluster, after: $(print_elapsed_time start_file_transfer_time)\n" else echo -e "\n\nERROR: FAILED TO TRANSFER THE FILES OF '${db}', WITH 'hadoop distcp'. GOT EXIT STATUS: $?\n\n" rm -f error.log @@ -120,6 +121,7 @@ function copydb() { fi echo -e "\nCreating schema for db: '${db}'\n" + start_create_schema_time=$(date +%s) # create the new database (with the same name) impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "create database ${db}" @@ -130,7 +132,8 @@ function copydb() { all_create_view_statements=() num_tables=0 - entities_on_ocean=`hive -e "show tables in ${db};" | sed 's/WARN:.*//g'` # Get the tables and views without any potential the "WARN" logs. + entities_on_ocean=(`hive -e "show tables in ${db};" | sed 's/WARN:.*//g'`) # Get the tables and views without any potential the "WARN" logs. + echo -e "\nGoing to create ${#entities_on_ocean[@]} entities for db '${db}'..\n" for i in ${entities_on_ocean[@]}; do # Use un-quoted values, as the elements are single-words. # Check if this is a view by showing the create-statement where it should print "create view" for a view, not the "create table". Unfortunately, there is no "show views" command. create_entity_statement=`hive --database ${db} -e "show create table ${i};"` # We need to use the "--database", instead of including it inside the query, in order to return the statements with the '`' chars being in the right place to be used by impala-shell. However, we need to add the db-name in the "CREATE VIEW view_name" statement. @@ -154,8 +157,9 @@ function copydb() { if [ -n "$log_errors" ]; then echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN CREATING TABLE '${i}'!\n\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then + rm -f error.log exit 6 - fi + fi # This error is not FATAL, do we do not return from this function, in normal circumstances. fi fi fi @@ -210,8 +214,11 @@ function copydb() { previous_num_of_views_to_retry=$new_num_of_views_to_retry done + entities_on_impala=(`impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} --delimited -q "show tables in ${db}"`) + echo -e "\nThe schema of db '${db}', along with ${#entities_on_impala[@]} entities have been created, on Impala cluster, after: $(print_elapsed_time start_create_schema_time)\n" + + start_compute_stats_time=$(date +%s) echo -e "\nComputing stats for tables..\n" - entities_on_impala=`impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} --delimited -q "show tables in ${db}"` for i in ${entities_on_impala[@]}; do # Use un-quoted values, as the elemetns are single-words. # Taking the create table statement from the Ocean cluster, just to check if its a view, as the output is easier than using impala-shell from Impala cluster. create_view_statement=`hive -e "show create table ${db}.${i};" | grep "CREATE VIEW"` # This grep works here, as we do not want to match multiple-lines. @@ -223,20 +230,18 @@ function copydb() { fi done + echo -e "\nFinished computing stats for tables, after: $(print_elapsed_time start_compute_stats_time)\n" + rm -f error.log # Cleanup the temp log-file. + # Check if the entities in both clusters are the same, down to the exact names, not just the counts. (they are sorted in the same way both in hive and impala) - if [ "${entities_on_impala[@]}" == "${entities_on_ocean[@]}" ]; then - echo -e "\nAll entities have been copied to Impala cluster.\n" + if [[ "${entities_on_impala[@]}" == "${entities_on_ocean[@]}" ]]; then + echo -e "\nAll entities have been copied to Impala cluster.\n\nFinished processing db: '${db}', after: $(print_elapsed_time start_db_time)\n" else - echo -e "\n\nERROR: 1 OR MORE ENTITIES OF DB '${db}' FAILED TO BE COPIED TO IMPALA CLUSTER!\n\n" - rm -f error.log + echo -e "\n\nERROR: $((${#entities_on_ocean[@]} - ${#entities_on_impala[@]})) ENTITIES OF DB '${db}' FAILED TO BE COPIED TO IMPALA CLUSTER!\n\n\nFinished processing db: '${db}', after: $(print_elapsed_time start_db_time)\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then exit 8 fi fi - - rm -f error.log - echo -e "\n\nFinished processing db: ${db}\n" - print_elapsed_time start_db_time } STATS_DB=$1 From 38636942c7af3f400f01618a6667f07fa29268d0 Mon Sep 17 00:00:00 2001 From: Antonis Lempesis Date: Fri, 14 Jun 2024 15:11:19 +0300 Subject: [PATCH 93/97] filtering out deletedbyinference and invinsible results from accessroute --- .../eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step14.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step14.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step14.sql index 7bad34e86..dafec9a6f 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step14.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step14.sql @@ -65,4 +65,5 @@ DROP TABLE IF EXISTS ${stats_db_name}.result_accessroute purge; CREATE TABLE IF NOT EXISTS ${stats_db_name}.result_accessroute STORED AS PARQUET as select distinct substr(id,4) as id, accessroute from ${openaire_db_name}.result -lateral view explode (instance.accessright.openaccessroute) openaccessroute as accessroute; + lateral view explode (instance.accessright.openaccessroute) openaccessroute as accessroute +WHERE datainfo.deletedbyinference=false and datainfo.invisible = FALSE; From 3095047e5e6cf1cba744264296f5e01f96bfb4b4 Mon Sep 17 00:00:00 2001 From: LSmyrnaios Date: Tue, 18 Jun 2024 14:40:41 +0300 Subject: [PATCH 94/97] Miscellaneous updates to the copying operation to Impala Cluster: - Fix not breaking out of the VIEWS-infinite-loop when the "SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR" is set to "false". - Exit the script when no HDFS-active-node was found, independently of the "SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR". - Fix view_name-recognition in a log-message, by using the more advanced "Perl-Compatible Regular Expressions" in "grep". - Add error-handling for "compute stats" errors. --- .../oozie_app/copyDataToImpalaCluster.sh | 24 ++++++++++++------- .../oozie_app/copyDataToImpalaCluster.sh | 24 ++++++++++++------- .../oozie_app/copyDataToImpalaCluster.sh | 24 ++++++++++++------- .../oozie_app/copyDataToImpalaCluster.sh | 24 ++++++++++++------- 4 files changed, 64 insertions(+), 32 deletions(-) diff --git a/dhp-workflows/dhp-stats-hist-snaps/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-hist-snaps/oozie_app/copyDataToImpalaCluster.sh b/dhp-workflows/dhp-stats-hist-snaps/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-hist-snaps/oozie_app/copyDataToImpalaCluster.sh index 55deada40..978cf4a9a 100644 --- a/dhp-workflows/dhp-stats-hist-snaps/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-hist-snaps/oozie_app/copyDataToImpalaCluster.sh +++ b/dhp-workflows/dhp-stats-hist-snaps/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-hist-snaps/oozie_app/copyDataToImpalaCluster.sh @@ -32,9 +32,7 @@ while [ $COUNTER -lt 3 ]; do done if [ -z "$IMPALA_HDFS_NODE" ]; then echo -e "\n\nERROR: PROBLEM WHEN SETTING THE HDFS-NODE FOR IMPALA CLUSTER! | AFTER ${COUNTER} RETRIES.\n\n" - if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then - exit 1 - fi + exit 1 # This is fatal and we have to exit independently of the "SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR" config, as none of the DBs will be able to get transferred. fi echo -e "Active IMPALA HDFS Node: ${IMPALA_HDFS_NODE} , after ${COUNTER} retries.\n\n" @@ -148,7 +146,7 @@ function copydb() { echo -e "\nERROR: THE TABLE \"${i}\" HAD NO FILES TO GET THE SCHEMA FROM! IT'S EMPTY!\n\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then exit 5 - fi + fi # This error is not FATAL, do we do not return from this function, in normal circumstances. else impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "create table ${db}.${i} like parquet '${CURRENT_PRQ_FILE}' stored as parquet;" |& tee error.log log_errors=`cat error.log | grep -E "WARN|ERROR|FAILED"` @@ -188,7 +186,7 @@ function copydb() { specific_errors=`cat error.log | grep -E "FAILED: ParseException line 1:13 missing TABLE at 'view'|ERROR: AnalysisException: Could not resolve table reference:"` if [ -n "$specific_errors" ]; then echo -e "\nspecific_errors: ${specific_errors}\n" - echo -e "\nView '$(cat error.log | grep -Eo "Query: CREATE VIEW ([^\s]+)" | sed 's/Query: CREATE VIEW //g')' failed to be created, possibly because it depends on another view.\n" + echo -e "\nView '$(cat error.log | grep -Po "Query: CREATE VIEW ([^\s]+)" | sed 's/Query: CREATE VIEW //g')' failed to be created, possibly because it depends on another view.\n" ((new_num_of_views_to_retry++)) # Increment it here, instead of acquiring the array's size in the end, as that doesn't work for some reason. else all_create_view_statements=("${all_create_view_statements[@]/$create_view_statement}") # Remove the current successful statement from the list. @@ -200,9 +198,11 @@ function copydb() { # Although the above command reduces the "active" elements to just the few to-be-retried, it does not manage to make the array return the its true size through the "${#all_create_view_statements[@]}" statement. So we use counters. if [[ $new_num_of_views_to_retry -eq $previous_num_of_views_to_retry ]]; then - echo -e "\n\nERROR: THE NUMBER OF VIEWS TO RETRY HAS NOT BEEN REDUCED! THE SCRIPT IS LIKELY GOING TO AN INFINITE-LOOP! EXITING..\n\n" + echo -e "\n\nERROR: THE NUMBER OF VIEWS TO RETRY HAS NOT BEEN REDUCED! THE SCRIPT IS LIKELY GOING TO AN INFINITE-LOOP! BREAKING-OUT..\n\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then exit 7 + else + break # Break form the inf-loop of views and continue by computing stats for the tables. fi elif [[ $new_num_of_views_to_retry -gt 0 ]]; then echo -e "\nTo be retried \"create_view_statements\" (${new_num_of_views_to_retry}):\n\n${all_create_view_statements[@]}\n" @@ -224,7 +224,15 @@ function copydb() { # Invalidate metadata of this DB's tables, in order for Impala to be aware of all parquet files put inside the tables' directories, previously, by "hadoop distcp". impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "INVALIDATE METADATA ${db}.${i}" sleep 1 - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "compute stats ${db}.${i}"; + impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "compute stats ${db}.${i}" |& tee error.log + log_errors=`cat error.log | grep -E "WARN|ERROR|FAILED"` + if [ -n "$log_errors" ]; then + echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN COMPUTING STATS FOR TABLE '${i}'!\n\n" + if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then + rm -f error.log + exit 8 + fi # This error is not FATAL, do we do not return from this function, in normal circumstances. + fi fi done @@ -237,7 +245,7 @@ function copydb() { else echo -e "\n\nERROR: $((${#entities_on_ocean[@]} - ${#entities_on_impala[@]})) ENTITIES OF DB '${db}' FAILED TO BE COPIED TO IMPALA CLUSTER!\n\n\nFinished processing db: '${db}', after: $(print_elapsed_time start_db_time)\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then - exit 8 + exit 9 fi fi } diff --git a/dhp-workflows/dhp-stats-monitor-irish/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor-irish/oozie_app/copyDataToImpalaCluster.sh b/dhp-workflows/dhp-stats-monitor-irish/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor-irish/oozie_app/copyDataToImpalaCluster.sh index 55deada40..978cf4a9a 100644 --- a/dhp-workflows/dhp-stats-monitor-irish/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor-irish/oozie_app/copyDataToImpalaCluster.sh +++ b/dhp-workflows/dhp-stats-monitor-irish/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor-irish/oozie_app/copyDataToImpalaCluster.sh @@ -32,9 +32,7 @@ while [ $COUNTER -lt 3 ]; do done if [ -z "$IMPALA_HDFS_NODE" ]; then echo -e "\n\nERROR: PROBLEM WHEN SETTING THE HDFS-NODE FOR IMPALA CLUSTER! | AFTER ${COUNTER} RETRIES.\n\n" - if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then - exit 1 - fi + exit 1 # This is fatal and we have to exit independently of the "SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR" config, as none of the DBs will be able to get transferred. fi echo -e "Active IMPALA HDFS Node: ${IMPALA_HDFS_NODE} , after ${COUNTER} retries.\n\n" @@ -148,7 +146,7 @@ function copydb() { echo -e "\nERROR: THE TABLE \"${i}\" HAD NO FILES TO GET THE SCHEMA FROM! IT'S EMPTY!\n\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then exit 5 - fi + fi # This error is not FATAL, do we do not return from this function, in normal circumstances. else impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "create table ${db}.${i} like parquet '${CURRENT_PRQ_FILE}' stored as parquet;" |& tee error.log log_errors=`cat error.log | grep -E "WARN|ERROR|FAILED"` @@ -188,7 +186,7 @@ function copydb() { specific_errors=`cat error.log | grep -E "FAILED: ParseException line 1:13 missing TABLE at 'view'|ERROR: AnalysisException: Could not resolve table reference:"` if [ -n "$specific_errors" ]; then echo -e "\nspecific_errors: ${specific_errors}\n" - echo -e "\nView '$(cat error.log | grep -Eo "Query: CREATE VIEW ([^\s]+)" | sed 's/Query: CREATE VIEW //g')' failed to be created, possibly because it depends on another view.\n" + echo -e "\nView '$(cat error.log | grep -Po "Query: CREATE VIEW ([^\s]+)" | sed 's/Query: CREATE VIEW //g')' failed to be created, possibly because it depends on another view.\n" ((new_num_of_views_to_retry++)) # Increment it here, instead of acquiring the array's size in the end, as that doesn't work for some reason. else all_create_view_statements=("${all_create_view_statements[@]/$create_view_statement}") # Remove the current successful statement from the list. @@ -200,9 +198,11 @@ function copydb() { # Although the above command reduces the "active" elements to just the few to-be-retried, it does not manage to make the array return the its true size through the "${#all_create_view_statements[@]}" statement. So we use counters. if [[ $new_num_of_views_to_retry -eq $previous_num_of_views_to_retry ]]; then - echo -e "\n\nERROR: THE NUMBER OF VIEWS TO RETRY HAS NOT BEEN REDUCED! THE SCRIPT IS LIKELY GOING TO AN INFINITE-LOOP! EXITING..\n\n" + echo -e "\n\nERROR: THE NUMBER OF VIEWS TO RETRY HAS NOT BEEN REDUCED! THE SCRIPT IS LIKELY GOING TO AN INFINITE-LOOP! BREAKING-OUT..\n\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then exit 7 + else + break # Break form the inf-loop of views and continue by computing stats for the tables. fi elif [[ $new_num_of_views_to_retry -gt 0 ]]; then echo -e "\nTo be retried \"create_view_statements\" (${new_num_of_views_to_retry}):\n\n${all_create_view_statements[@]}\n" @@ -224,7 +224,15 @@ function copydb() { # Invalidate metadata of this DB's tables, in order for Impala to be aware of all parquet files put inside the tables' directories, previously, by "hadoop distcp". impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "INVALIDATE METADATA ${db}.${i}" sleep 1 - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "compute stats ${db}.${i}"; + impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "compute stats ${db}.${i}" |& tee error.log + log_errors=`cat error.log | grep -E "WARN|ERROR|FAILED"` + if [ -n "$log_errors" ]; then + echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN COMPUTING STATS FOR TABLE '${i}'!\n\n" + if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then + rm -f error.log + exit 8 + fi # This error is not FATAL, do we do not return from this function, in normal circumstances. + fi fi done @@ -237,7 +245,7 @@ function copydb() { else echo -e "\n\nERROR: $((${#entities_on_ocean[@]} - ${#entities_on_impala[@]})) ENTITIES OF DB '${db}' FAILED TO BE COPIED TO IMPALA CLUSTER!\n\n\nFinished processing db: '${db}', after: $(print_elapsed_time start_db_time)\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then - exit 8 + exit 9 fi fi } diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh index 43498abd2..55ae3114e 100644 --- a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh +++ b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh @@ -32,9 +32,7 @@ while [ $COUNTER -lt 3 ]; do done if [ -z "$IMPALA_HDFS_NODE" ]; then echo -e "\n\nERROR: PROBLEM WHEN SETTING THE HDFS-NODE FOR IMPALA CLUSTER! | AFTER ${COUNTER} RETRIES.\n\n" - if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then - exit 1 - fi + exit 1 # This is fatal and we have to exit independently of the "SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR" config, as none of the DBs will be able to get transferred. fi echo -e "Active IMPALA HDFS Node: ${IMPALA_HDFS_NODE} , after ${COUNTER} retries.\n\n" @@ -148,7 +146,7 @@ function copydb() { echo -e "\nERROR: THE TABLE \"${i}\" HAD NO FILES TO GET THE SCHEMA FROM! IT'S EMPTY!\n\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then exit 5 - fi + fi # This error is not FATAL, do we do not return from this function, in normal circumstances. else impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "create table ${db}.${i} like parquet '${CURRENT_PRQ_FILE}' stored as parquet;" |& tee error.log log_errors=`cat error.log | grep -E "WARN|ERROR|FAILED"` @@ -188,7 +186,7 @@ function copydb() { specific_errors=`cat error.log | grep -E "FAILED: ParseException line 1:13 missing TABLE at 'view'|ERROR: AnalysisException: Could not resolve table reference:"` if [ -n "$specific_errors" ]; then echo -e "\nspecific_errors: ${specific_errors}\n" - echo -e "\nView '$(cat error.log | grep -Eo "Query: CREATE VIEW ([^\s]+)" | sed 's/Query: CREATE VIEW //g')' failed to be created, possibly because it depends on another view.\n" + echo -e "\nView '$(cat error.log | grep -Po "Query: CREATE VIEW ([^\s]+)" | sed 's/Query: CREATE VIEW //g')' failed to be created, possibly because it depends on another view.\n" ((new_num_of_views_to_retry++)) # Increment it here, instead of acquiring the array's size in the end, as that doesn't work for some reason. else all_create_view_statements=("${all_create_view_statements[@]/$create_view_statement}") # Remove the current successful statement from the list. @@ -200,9 +198,11 @@ function copydb() { # Although the above command reduces the "active" elements to just the few to-be-retried, it does not manage to make the array return the its true size through the "${#all_create_view_statements[@]}" statement. So we use counters. if [[ $new_num_of_views_to_retry -eq $previous_num_of_views_to_retry ]]; then - echo -e "\n\nERROR: THE NUMBER OF VIEWS TO RETRY HAS NOT BEEN REDUCED! THE SCRIPT IS LIKELY GOING TO AN INFINITE-LOOP! EXITING..\n\n" + echo -e "\n\nERROR: THE NUMBER OF VIEWS TO RETRY HAS NOT BEEN REDUCED! THE SCRIPT IS LIKELY GOING TO AN INFINITE-LOOP! BREAKING-OUT..\n\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then exit 7 + else + break # Break form the inf-loop of views and continue by computing stats for the tables. fi elif [[ $new_num_of_views_to_retry -gt 0 ]]; then echo -e "\nTo be retried \"create_view_statements\" (${new_num_of_views_to_retry}):\n\n${all_create_view_statements[@]}\n" @@ -224,7 +224,15 @@ function copydb() { # Invalidate metadata of this DB's tables, in order for Impala to be aware of all parquet files put inside the tables' directories, previously, by "hadoop distcp". impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "INVALIDATE METADATA ${db}.${i}" sleep 1 - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "compute stats ${db}.${i}"; + impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "compute stats ${db}.${i}" |& tee error.log + log_errors=`cat error.log | grep -E "WARN|ERROR|FAILED"` + if [ -n "$log_errors" ]; then + echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN COMPUTING STATS FOR TABLE '${i}'!\n\n" + if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then + rm -f error.log + exit 8 + fi # This error is not FATAL, do we do not return from this function, in normal circumstances. + fi fi done @@ -237,7 +245,7 @@ function copydb() { else echo -e "\n\nERROR: $((${#entities_on_ocean[@]} - ${#entities_on_impala[@]})) ENTITIES OF DB '${db}' FAILED TO BE COPIED TO IMPALA CLUSTER!\n\n\nFinished processing db: '${db}', after: $(print_elapsed_time start_db_time)\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then - exit 8 + exit 9 fi fi } diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh index 1d5842d06..07a8a4534 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh @@ -30,9 +30,7 @@ while [ $COUNTER -lt 3 ]; do done if [ -z "$IMPALA_HDFS_NODE" ]; then echo -e "\n\nERROR: PROBLEM WHEN SETTING THE HDFS-NODE FOR IMPALA CLUSTER! | AFTER ${COUNTER} RETRIES.\n\n" - if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then - exit 1 - fi + exit 1 # This is fatal and we have to exit independently of the "SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR" config, as none of the DBs will be able to get transferred. fi echo -e "Active IMPALA HDFS Node: ${IMPALA_HDFS_NODE} , after ${COUNTER} retries.\n\n" @@ -150,7 +148,7 @@ function copydb() { echo -e "\nERROR: THE TABLE \"${i}\" HAD NO FILES TO GET THE SCHEMA FROM! IT'S EMPTY!\n\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then exit 5 - fi + fi # This error is not FATAL, do we do not return from this function, in normal circumstances. else impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "create table ${db}.${i} like parquet '${CURRENT_PRQ_FILE}' stored as parquet;" |& tee error.log log_errors=`cat error.log | grep -E "WARN|ERROR|FAILED"` @@ -190,7 +188,7 @@ function copydb() { specific_errors=`cat error.log | grep -E "FAILED: ParseException line 1:13 missing TABLE at 'view'|ERROR: AnalysisException: Could not resolve table reference:"` if [ -n "$specific_errors" ]; then echo -e "\nspecific_errors: ${specific_errors}\n" - echo -e "\nView '$(cat error.log | grep -Eo "Query: CREATE VIEW ([^\s]+)" | sed 's/Query: CREATE VIEW //g')' failed to be created, possibly because it depends on another view.\n" + echo -e "\nView '$(cat error.log | grep -Po "Query: CREATE VIEW ([^\s]+)" | sed 's/Query: CREATE VIEW //g')' failed to be created, possibly because it depends on another view.\n" ((new_num_of_views_to_retry++)) # Increment it here, instead of acquiring the array's size in the end, as that doesn't work for some reason. else all_create_view_statements=("${all_create_view_statements[@]/$create_view_statement}") # Remove the current successful statement from the list. @@ -202,9 +200,11 @@ function copydb() { # Although the above command reduces the "active" elements to just the few to-be-retried, it does not manage to make the array return the its true size through the "${#all_create_view_statements[@]}" statement. So we use counters. if [[ $new_num_of_views_to_retry -eq $previous_num_of_views_to_retry ]]; then - echo -e "\n\nERROR: THE NUMBER OF VIEWS TO RETRY HAS NOT BEEN REDUCED! THE SCRIPT IS LIKELY GOING TO AN INFINITE-LOOP! EXITING..\n\n" + echo -e "\n\nERROR: THE NUMBER OF VIEWS TO RETRY HAS NOT BEEN REDUCED! THE SCRIPT IS LIKELY GOING TO AN INFINITE-LOOP! BREAKING-OUT..\n\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then exit 7 + else + break # Break form the inf-loop of views and continue by computing stats for the tables. fi elif [[ $new_num_of_views_to_retry -gt 0 ]]; then echo -e "\nTo be retried \"create_view_statements\" (${new_num_of_views_to_retry}):\n\n${all_create_view_statements[@]}\n" @@ -226,7 +226,15 @@ function copydb() { # Invalidate metadata of this DB's tables, in order for Impala to be aware of all parquet files put inside the tables' directories, previously, by "hadoop distcp". impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "INVALIDATE METADATA ${db}.${i}" sleep 1 - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "compute stats ${db}.${i}"; + impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "compute stats ${db}.${i}" |& tee error.log + log_errors=`cat error.log | grep -E "WARN|ERROR|FAILED"` + if [ -n "$log_errors" ]; then + echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN COMPUTING STATS FOR TABLE '${i}'!\n\n" + if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then + rm -f error.log + exit 8 + fi # This error is not FATAL, do we do not return from this function, in normal circumstances. + fi fi done @@ -239,7 +247,7 @@ function copydb() { else echo -e "\n\nERROR: $((${#entities_on_ocean[@]} - ${#entities_on_impala[@]})) ENTITIES OF DB '${db}' FAILED TO BE COPIED TO IMPALA CLUSTER!\n\n\nFinished processing db: '${db}', after: $(print_elapsed_time start_db_time)\n" if [[ SHOULD_EXIT_WHOLE_SCRIPT_UPON_ERROR -eq 1 ]]; then - exit 8 + exit 9 fi fi } From ac270f795b998e6774da8ff4a6d4d729816f2c10 Mon Sep 17 00:00:00 2001 From: Miriam Baglioni Date: Wed, 19 Jun 2024 11:11:52 +0200 Subject: [PATCH 95/97] [IrishFunderList]make changed according to 9635 comment 14, 15 and 16 --- .../dhp/collection/crossref/irish_funder.json | 25 +++---------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/crossref/irish_funder.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/crossref/irish_funder.json index e4f491e5c..9482904c5 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/crossref/irish_funder.json +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/crossref/irish_funder.json @@ -58,7 +58,7 @@ "uri": "http://dx.doi.org/10.13039/100010414", "name": "Health Research Board", "synonym": [ - "501100001590" + "501100001590", "501100023273" ] }, { @@ -85,12 +85,6 @@ "name": "Irish College of General Practitioners", "synonym": [] }, - { - "id": "100012734", - "uri": "http://dx.doi.org/10.13039/100012734", - "name": "Department for Culture, Heritage and the Gaeltacht, Ireland", - "synonym": [] - }, { "id": "100012754", "uri": "http://dx.doi.org/10.13039/100012754", @@ -233,7 +227,7 @@ "id": "100018064", "uri": "http://dx.doi.org/10.13039/100018064", "name": "Department of Tourism, Culture, Arts, Gaeltacht, Sport and Media", - "synonym": [] + "synonym": ["100012734"] }, { "id": "100018172", @@ -319,12 +313,7 @@ "name": "Centre for Ageing Research and Development in Ireland", "synonym": [] }, - { - "id": "501100001583", - "uri": "http://dx.doi.org/10.13039/501100001583", - "name": "Cystinosis Foundation Ireland", - "synonym": [] - }, + { "id": "501100001584", "uri": "http://dx.doi.org/10.13039/501100001584", @@ -605,7 +594,7 @@ "id": "501100009315", "uri": "http://dx.doi.org/10.13039/501100009315", "name": "Cystinosis Ireland", - "synonym": [] + "synonym": ["501100001583"] }, { "id": "501100010808", @@ -763,12 +752,6 @@ "name": "Institute of Technology, Tralee", "synonym": [] }, - { - "id": "501100023273", - "uri": "http://dx.doi.org/10.13039/501100023273", - "name": "HRB Clinical Research Facility Galway", - "synonym": [] - }, { "id": "501100023378", "uri": "http://dx.doi.org/10.13039/501100023378", From d35edac212181c32037f20dc2a3e59e7458f9eb0 Mon Sep 17 00:00:00 2001 From: Miriam Baglioni Date: Thu, 20 Jun 2024 12:28:28 +0200 Subject: [PATCH 96/97] [IrishFunderList]make changed according to 9635 comment 20, 21, 22 and 23 --- .../dhp/collection/crossref/irish_funder.json | 30 ++----------------- 1 file changed, 3 insertions(+), 27 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/crossref/irish_funder.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/crossref/irish_funder.json index 9482904c5..e50dc2dee 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/crossref/irish_funder.json +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/crossref/irish_funder.json @@ -85,18 +85,6 @@ "name": "Irish College of General Practitioners", "synonym": [] }, - { - "id": "100012754", - "uri": "http://dx.doi.org/10.13039/100012754", - "name": "Horizon Pharma", - "synonym": [] - }, - { - "id": "100012891", - "uri": "http://dx.doi.org/10.13039/100012891", - "name": "Medical Research Charities Group", - "synonym": [] - }, { "id": "100012919", "uri": "http://dx.doi.org/10.13039/100012919", @@ -275,13 +263,13 @@ "id": "100019637", "uri": "http://dx.doi.org/10.13039/100019637", "name": "Horizon Therapeutics", - "synonym": [] + "synonym": ["100012754"] }, { "id": "100020174", "uri": "http://dx.doi.org/10.13039/100020174", "name": "Health Research Charities Ireland", - "synonym": [] + "synonym": ["100012891"] }, { "id": "100020202", @@ -510,7 +498,7 @@ "id": "501100003037", "uri": "http://dx.doi.org/10.13039/501100003037", "name": "Elan", - "synonym": [] + "synonym": ["501100021694"] }, { "id": "501100003496", @@ -584,12 +572,6 @@ "name": "Technological University Dublin", "synonym": [] }, - { - "id": "501100009269", - "uri": "http://dx.doi.org/10.13039/501100009269", - "name": "Programme of Competitive Forestry Research for Development", - "synonym": [] - }, { "id": "501100009315", "uri": "http://dx.doi.org/10.13039/501100009315", @@ -716,12 +698,6 @@ "name": "Insight SFI Research Centre for Data Analytics", "synonym": [] }, - { - "id": "501100021694", - "uri": "http://dx.doi.org/10.13039/501100021694", - "name": "Elan Pharma International", - "synonym": [] - }, { "id": "501100021838", "uri": "http://dx.doi.org/10.13039/501100021838", From 66cd28f70a94536e38ff6368b65c88b9b2d3bbfc Mon Sep 17 00:00:00 2001 From: LSmyrnaios Date: Thu, 20 Jun 2024 14:33:46 +0300 Subject: [PATCH 97/97] - Fix not using the "export HADOOP_USER_NAME" statement in "createPDFsAggregated.sh", which caused permission-issues when creating tables with Impala. - Remove unused "--user" parameter in "impala-shell" calls. - Code polishing. --- .../oozie_app/copyDataToImpalaCluster.sh | 14 +++++----- .../oozie_app/copyDataToImpalaCluster.sh | 14 +++++----- .../oozie_app/copyDataToImpalaCluster.sh | 14 +++++----- .../oozie_app/copyDataToImpalaCluster.sh | 19 ++++++------- .../stats/oozie_app/createPDFsAggregated.sh | 28 +++++++++++-------- 5 files changed, 46 insertions(+), 43 deletions(-) diff --git a/dhp-workflows/dhp-stats-hist-snaps/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-hist-snaps/oozie_app/copyDataToImpalaCluster.sh b/dhp-workflows/dhp-stats-hist-snaps/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-hist-snaps/oozie_app/copyDataToImpalaCluster.sh index 978cf4a9a..09ea1b393 100644 --- a/dhp-workflows/dhp-stats-hist-snaps/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-hist-snaps/oozie_app/copyDataToImpalaCluster.sh +++ b/dhp-workflows/dhp-stats-hist-snaps/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-hist-snaps/oozie_app/copyDataToImpalaCluster.sh @@ -63,7 +63,7 @@ function copydb() { start_db_time=$(date +%s) # Delete the old DB from Impala cluster (if exists). - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "drop database if exists ${db} cascade" |& tee error.log # impala-shell prints all logs in stderr, so wee need to capture them and put them in a file, in order to perform "grep" on them later + impala-shell -i ${IMPALA_HOSTNAME} -q "drop database if exists ${db} cascade" |& tee error.log # impala-shell prints all logs in stderr, so wee need to capture them and put them in a file, in order to perform "grep" on them later log_errors=`cat error.log | grep -E "WARN|ERROR|FAILED"` if [ -n "$log_errors" ]; then echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN DROPPING THE OLD DATABASE FROM IMPALA CLUSTER! EXITING...\n\n" @@ -120,7 +120,7 @@ function copydb() { start_create_schema_time=$(date +%s) # create the new database (with the same name) - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "create database ${db}" + impala-shell -i ${IMPALA_HOSTNAME} -q "create database ${db}" # Because "Hive" and "Impala" do not have compatible schemas, we cannot use the "show create table " output from hive to create the exact same table in impala. # So, we have to find at least one parquet file (check if it's there) from the table in the ocean cluster for impala to use it to extract the table-schema itself from that file. @@ -148,7 +148,7 @@ function copydb() { exit 5 fi # This error is not FATAL, do we do not return from this function, in normal circumstances. else - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "create table ${db}.${i} like parquet '${CURRENT_PRQ_FILE}' stored as parquet;" |& tee error.log + impala-shell -i ${IMPALA_HOSTNAME} -q "create table ${db}.${i} like parquet '${CURRENT_PRQ_FILE}' stored as parquet;" |& tee error.log log_errors=`cat error.log | grep -E "WARN|ERROR|FAILED"` if [ -n "$log_errors" ]; then echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN CREATING TABLE '${i}'!\n\n" @@ -182,7 +182,7 @@ function copydb() { new_num_of_views_to_retry=0 for create_view_statement in "${all_create_view_statements[@]}"; do # Here we use double quotes, as the elements are phrases, instead of single-words. - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "${create_view_statement}" |& tee error.log # impala-shell prints all logs in stderr, so wee need to capture them and put them in a file, in order to perform "grep" on them later + impala-shell -i ${IMPALA_HOSTNAME} -q "${create_view_statement}" |& tee error.log # impala-shell prints all logs in stderr, so wee need to capture them and put them in a file, in order to perform "grep" on them later specific_errors=`cat error.log | grep -E "FAILED: ParseException line 1:13 missing TABLE at 'view'|ERROR: AnalysisException: Could not resolve table reference:"` if [ -n "$specific_errors" ]; then echo -e "\nspecific_errors: ${specific_errors}\n" @@ -212,7 +212,7 @@ function copydb() { previous_num_of_views_to_retry=$new_num_of_views_to_retry done - entities_on_impala=(`impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} --delimited -q "show tables in ${db}"`) + entities_on_impala=(`impala-shell -i ${IMPALA_HOSTNAME} --delimited -q "show tables in ${db}"`) echo -e "\nThe schema of db '${db}', along with ${#entities_on_impala[@]} entities have been created, on Impala cluster, after: $(print_elapsed_time start_create_schema_time)\n" start_compute_stats_time=$(date +%s) @@ -222,9 +222,9 @@ function copydb() { create_view_statement=`hive -e "show create table ${db}.${i};" | grep "CREATE VIEW"` # This grep works here, as we do not want to match multiple-lines. if [ -z "$create_view_statement" ]; then # If it's a table, then go load the data to it. # Invalidate metadata of this DB's tables, in order for Impala to be aware of all parquet files put inside the tables' directories, previously, by "hadoop distcp". - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "INVALIDATE METADATA ${db}.${i}" + impala-shell -i ${IMPALA_HOSTNAME} -q "INVALIDATE METADATA ${db}.${i}" sleep 1 - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "compute stats ${db}.${i}" |& tee error.log + impala-shell -i ${IMPALA_HOSTNAME} -q "compute stats ${db}.${i}" |& tee error.log log_errors=`cat error.log | grep -E "WARN|ERROR|FAILED"` if [ -n "$log_errors" ]; then echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN COMPUTING STATS FOR TABLE '${i}'!\n\n" diff --git a/dhp-workflows/dhp-stats-monitor-irish/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor-irish/oozie_app/copyDataToImpalaCluster.sh b/dhp-workflows/dhp-stats-monitor-irish/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor-irish/oozie_app/copyDataToImpalaCluster.sh index 978cf4a9a..09ea1b393 100644 --- a/dhp-workflows/dhp-stats-monitor-irish/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor-irish/oozie_app/copyDataToImpalaCluster.sh +++ b/dhp-workflows/dhp-stats-monitor-irish/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor-irish/oozie_app/copyDataToImpalaCluster.sh @@ -63,7 +63,7 @@ function copydb() { start_db_time=$(date +%s) # Delete the old DB from Impala cluster (if exists). - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "drop database if exists ${db} cascade" |& tee error.log # impala-shell prints all logs in stderr, so wee need to capture them and put them in a file, in order to perform "grep" on them later + impala-shell -i ${IMPALA_HOSTNAME} -q "drop database if exists ${db} cascade" |& tee error.log # impala-shell prints all logs in stderr, so wee need to capture them and put them in a file, in order to perform "grep" on them later log_errors=`cat error.log | grep -E "WARN|ERROR|FAILED"` if [ -n "$log_errors" ]; then echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN DROPPING THE OLD DATABASE FROM IMPALA CLUSTER! EXITING...\n\n" @@ -120,7 +120,7 @@ function copydb() { start_create_schema_time=$(date +%s) # create the new database (with the same name) - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "create database ${db}" + impala-shell -i ${IMPALA_HOSTNAME} -q "create database ${db}" # Because "Hive" and "Impala" do not have compatible schemas, we cannot use the "show create table " output from hive to create the exact same table in impala. # So, we have to find at least one parquet file (check if it's there) from the table in the ocean cluster for impala to use it to extract the table-schema itself from that file. @@ -148,7 +148,7 @@ function copydb() { exit 5 fi # This error is not FATAL, do we do not return from this function, in normal circumstances. else - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "create table ${db}.${i} like parquet '${CURRENT_PRQ_FILE}' stored as parquet;" |& tee error.log + impala-shell -i ${IMPALA_HOSTNAME} -q "create table ${db}.${i} like parquet '${CURRENT_PRQ_FILE}' stored as parquet;" |& tee error.log log_errors=`cat error.log | grep -E "WARN|ERROR|FAILED"` if [ -n "$log_errors" ]; then echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN CREATING TABLE '${i}'!\n\n" @@ -182,7 +182,7 @@ function copydb() { new_num_of_views_to_retry=0 for create_view_statement in "${all_create_view_statements[@]}"; do # Here we use double quotes, as the elements are phrases, instead of single-words. - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "${create_view_statement}" |& tee error.log # impala-shell prints all logs in stderr, so wee need to capture them and put them in a file, in order to perform "grep" on them later + impala-shell -i ${IMPALA_HOSTNAME} -q "${create_view_statement}" |& tee error.log # impala-shell prints all logs in stderr, so wee need to capture them and put them in a file, in order to perform "grep" on them later specific_errors=`cat error.log | grep -E "FAILED: ParseException line 1:13 missing TABLE at 'view'|ERROR: AnalysisException: Could not resolve table reference:"` if [ -n "$specific_errors" ]; then echo -e "\nspecific_errors: ${specific_errors}\n" @@ -212,7 +212,7 @@ function copydb() { previous_num_of_views_to_retry=$new_num_of_views_to_retry done - entities_on_impala=(`impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} --delimited -q "show tables in ${db}"`) + entities_on_impala=(`impala-shell -i ${IMPALA_HOSTNAME} --delimited -q "show tables in ${db}"`) echo -e "\nThe schema of db '${db}', along with ${#entities_on_impala[@]} entities have been created, on Impala cluster, after: $(print_elapsed_time start_create_schema_time)\n" start_compute_stats_time=$(date +%s) @@ -222,9 +222,9 @@ function copydb() { create_view_statement=`hive -e "show create table ${db}.${i};" | grep "CREATE VIEW"` # This grep works here, as we do not want to match multiple-lines. if [ -z "$create_view_statement" ]; then # If it's a table, then go load the data to it. # Invalidate metadata of this DB's tables, in order for Impala to be aware of all parquet files put inside the tables' directories, previously, by "hadoop distcp". - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "INVALIDATE METADATA ${db}.${i}" + impala-shell -i ${IMPALA_HOSTNAME} -q "INVALIDATE METADATA ${db}.${i}" sleep 1 - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "compute stats ${db}.${i}" |& tee error.log + impala-shell -i ${IMPALA_HOSTNAME} -q "compute stats ${db}.${i}" |& tee error.log log_errors=`cat error.log | grep -E "WARN|ERROR|FAILED"` if [ -n "$log_errors" ]; then echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN COMPUTING STATS FOR TABLE '${i}'!\n\n" diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh index 55ae3114e..d75412df8 100644 --- a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh +++ b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh @@ -63,7 +63,7 @@ function copydb() { start_db_time=$(date +%s) # Delete the old DB from Impala cluster (if exists). - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "drop database if exists ${db} cascade" |& tee error.log # impala-shell prints all logs in stderr, so wee need to capture them and put them in a file, in order to perform "grep" on them later + impala-shell -i ${IMPALA_HOSTNAME} -q "drop database if exists ${db} cascade" |& tee error.log # impala-shell prints all logs in stderr, so wee need to capture them and put them in a file, in order to perform "grep" on them later log_errors=`cat error.log | grep -E "WARN|ERROR|FAILED"` if [ -n "$log_errors" ]; then echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN DROPPING THE OLD DATABASE FROM IMPALA CLUSTER! EXITING...\n\n" @@ -120,7 +120,7 @@ function copydb() { start_create_schema_time=$(date +%s) # create the new database (with the same name) - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "create database ${db}" + impala-shell -i ${IMPALA_HOSTNAME} -q "create database ${db}" # Because "Hive" and "Impala" do not have compatible schemas, we cannot use the "show create table " output from hive to create the exact same table in impala. # So, we have to find at least one parquet file (check if it's there) from the table in the ocean cluster for impala to use it to extract the table-schema itself from that file. @@ -148,7 +148,7 @@ function copydb() { exit 5 fi # This error is not FATAL, do we do not return from this function, in normal circumstances. else - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "create table ${db}.${i} like parquet '${CURRENT_PRQ_FILE}' stored as parquet;" |& tee error.log + impala-shell -i ${IMPALA_HOSTNAME} -q "create table ${db}.${i} like parquet '${CURRENT_PRQ_FILE}' stored as parquet;" |& tee error.log log_errors=`cat error.log | grep -E "WARN|ERROR|FAILED"` if [ -n "$log_errors" ]; then echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN CREATING TABLE '${i}'!\n\n" @@ -182,7 +182,7 @@ function copydb() { new_num_of_views_to_retry=0 for create_view_statement in "${all_create_view_statements[@]}"; do # Here we use double quotes, as the elements are phrases, instead of single-words. - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "${create_view_statement}" |& tee error.log # impala-shell prints all logs in stderr, so wee need to capture them and put them in a file, in order to perform "grep" on them later + impala-shell -i ${IMPALA_HOSTNAME} -q "${create_view_statement}" |& tee error.log # impala-shell prints all logs in stderr, so wee need to capture them and put them in a file, in order to perform "grep" on them later specific_errors=`cat error.log | grep -E "FAILED: ParseException line 1:13 missing TABLE at 'view'|ERROR: AnalysisException: Could not resolve table reference:"` if [ -n "$specific_errors" ]; then echo -e "\nspecific_errors: ${specific_errors}\n" @@ -212,7 +212,7 @@ function copydb() { previous_num_of_views_to_retry=$new_num_of_views_to_retry done - entities_on_impala=(`impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} --delimited -q "show tables in ${db}"`) + entities_on_impala=(`impala-shell -i ${IMPALA_HOSTNAME} --delimited -q "show tables in ${db}"`) echo -e "\nThe schema of db '${db}', along with ${#entities_on_impala[@]} entities have been created, on Impala cluster, after: $(print_elapsed_time start_create_schema_time)\n" start_compute_stats_time=$(date +%s) @@ -222,9 +222,9 @@ function copydb() { create_view_statement=`hive -e "show create table ${db}.${i};" | grep "CREATE VIEW"` # This grep works here, as we do not want to match multiple-lines. if [ -z "$create_view_statement" ]; then # If it's a table, then go load the data to it. # Invalidate metadata of this DB's tables, in order for Impala to be aware of all parquet files put inside the tables' directories, previously, by "hadoop distcp". - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "INVALIDATE METADATA ${db}.${i}" + impala-shell -i ${IMPALA_HOSTNAME} -q "INVALIDATE METADATA ${db}.${i}" sleep 1 - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "compute stats ${db}.${i}" |& tee error.log + impala-shell -i ${IMPALA_HOSTNAME} -q "compute stats ${db}.${i}" |& tee error.log log_errors=`cat error.log | grep -E "WARN|ERROR|FAILED"` if [ -n "$log_errors" ]; then echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN COMPUTING STATS FOR TABLE '${i}'!\n\n" diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh index 07a8a4534..96c61d91a 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh @@ -65,7 +65,7 @@ function copydb() { start_db_time=$(date +%s) # Delete the old DB from Impala cluster (if exists). - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "drop database if exists ${db} cascade" |& tee error.log # impala-shell prints all logs in stderr, so wee need to capture them and put them in a file, in order to perform "grep" on them later + impala-shell -i ${IMPALA_HOSTNAME} -q "drop database if exists ${db} cascade" |& tee error.log # impala-shell prints all logs in stderr, so wee need to capture them and put them in a file, in order to perform "grep" on them later log_errors=`cat error.log | grep -E "WARN|ERROR|FAILED"` if [ -n "$log_errors" ]; then echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN DROPPING THE OLD DATABASE FROM IMPALA CLUSTER! EXITING...\n\n" @@ -122,7 +122,7 @@ function copydb() { start_create_schema_time=$(date +%s) # create the new database (with the same name) - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "create database ${db}" + impala-shell -i ${IMPALA_HOSTNAME} -q "create database ${db}" # Because "Hive" and "Impala" do not have compatible schemas, we cannot use the "show create table " output from hive to create the exact same table in impala. # So, we have to find at least one parquet file (check if it's there) from the table in the ocean cluster for impala to use it to extract the table-schema itself from that file. @@ -150,7 +150,7 @@ function copydb() { exit 5 fi # This error is not FATAL, do we do not return from this function, in normal circumstances. else - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "create table ${db}.${i} like parquet '${CURRENT_PRQ_FILE}' stored as parquet;" |& tee error.log + impala-shell -i ${IMPALA_HOSTNAME} -q "create table ${db}.${i} like parquet '${CURRENT_PRQ_FILE}' stored as parquet;" |& tee error.log log_errors=`cat error.log | grep -E "WARN|ERROR|FAILED"` if [ -n "$log_errors" ]; then echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN CREATING TABLE '${i}'!\n\n" @@ -184,7 +184,7 @@ function copydb() { new_num_of_views_to_retry=0 for create_view_statement in "${all_create_view_statements[@]}"; do # Here we use double quotes, as the elements are phrases, instead of single-words. - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "${create_view_statement}" |& tee error.log # impala-shell prints all logs in stderr, so wee need to capture them and put them in a file, in order to perform "grep" on them later + impala-shell -i ${IMPALA_HOSTNAME} -q "${create_view_statement}" |& tee error.log # impala-shell prints all logs in stderr, so wee need to capture them and put them in a file, in order to perform "grep" on them later specific_errors=`cat error.log | grep -E "FAILED: ParseException line 1:13 missing TABLE at 'view'|ERROR: AnalysisException: Could not resolve table reference:"` if [ -n "$specific_errors" ]; then echo -e "\nspecific_errors: ${specific_errors}\n" @@ -214,7 +214,7 @@ function copydb() { previous_num_of_views_to_retry=$new_num_of_views_to_retry done - entities_on_impala=(`impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} --delimited -q "show tables in ${db}"`) + entities_on_impala=(`impala-shell -i ${IMPALA_HOSTNAME} --delimited -q "show tables in ${db}"`) echo -e "\nThe schema of db '${db}', along with ${#entities_on_impala[@]} entities have been created, on Impala cluster, after: $(print_elapsed_time start_create_schema_time)\n" start_compute_stats_time=$(date +%s) @@ -224,9 +224,9 @@ function copydb() { create_view_statement=`hive -e "show create table ${db}.${i};" | grep "CREATE VIEW"` # This grep works here, as we do not want to match multiple-lines. if [ -z "$create_view_statement" ]; then # If it's a table, then go load the data to it. # Invalidate metadata of this DB's tables, in order for Impala to be aware of all parquet files put inside the tables' directories, previously, by "hadoop distcp". - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "INVALIDATE METADATA ${db}.${i}" + impala-shell -i ${IMPALA_HOSTNAME} -q "INVALIDATE METADATA ${db}.${i}" sleep 1 - impala-shell --user ${HADOOP_USER_NAME} -i ${IMPALA_HOSTNAME} -q "compute stats ${db}.${i}" |& tee error.log + impala-shell -i ${IMPALA_HOSTNAME} -q "compute stats ${db}.${i}" |& tee error.log log_errors=`cat error.log | grep -E "WARN|ERROR|FAILED"` if [ -n "$log_errors" ]; then echo -e "\n\nERROR: THERE WAS A PROBLEM WHEN COMPUTING STATS FOR TABLE '${i}'!\n\n" @@ -271,8 +271,7 @@ copydb $MONITOR_DB'_institutions' copydb $MONITOR_DB'_ris_tail' contexts="knowmad::other dh-ch::other enermaps::other gotriple::other neanias-atmospheric::other rural-digital-europe::other covid-19::other aurora::other neanias-space::other north-america-studies::other north-american-studies::other eutopia::other" -for i in ${contexts} -do - tmp=`echo "$i" | sed 's/'-'/'_'/g' | sed 's/'::'/'_'/g'` +for i in ${contexts}; do + tmp=`echo "$i" | sed 's/'-'/'_'/g' | sed 's/'::'/'_'/g'` copydb ${MONITOR_DB}'_'${tmp} done \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/createPDFsAggregated.sh b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/createPDFsAggregated.sh index 46631a0c2..9eec0bb20 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/createPDFsAggregated.sh +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/createPDFsAggregated.sh @@ -6,21 +6,26 @@ then ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder} fi +export HADOOP_USER_NAME=$3 + +IMPALA_HOSTNAME='impala-cluster-dn1.openaire.eu' + function createPDFsAggregated() { db=$1 -impala-shell --user $HADOOP_USER_NAME -i impala-cluster-dn1.openaire.eu -d ${db} -q "drop table if exists indi_is_result_accessible"; + impala-shell -i ${IMPALA_HOSTNAME} -d ${db} -q "drop table if exists indi_is_result_accessible"; -impala-shell --user $HADOOP_USER_NAME -i impala-cluster-dn1.openaire.eu -d ${db} -q "create table indi_is_result_accessible stored as parquet as + impala-shell -i ${IMPALA_HOSTNAME} -d ${db} -q "create table indi_is_result_accessible stored as parquet as select distinct p.id, coalesce(is_result_accessible, 0) as is_result_accessible from result p left outer join - (select id, 1 as is_result_accessible from (select pl.* from result r - join pdfaggregation_i.publication p on r.id=p.id - join pdfaggregation_i.payload pl on pl.id=p.id - union all - select pl.* from result r - join pdfaggregation_i.publication p on r.id=p.dedupid - join pdfaggregation_i.payload pl on pl.id=p.id) foo) tmp on p.id=tmp.id"; + (select id, 1 as is_result_accessible from (select pl.* from result r + join pdfaggregation_i.publication p on r.id=p.id + join pdfaggregation_i.payload pl on pl.id=p.id + union all + select pl.* from result r + join pdfaggregation_i.publication p on r.id=p.dedupid + join pdfaggregation_i.payload pl on pl.id=p.id) foo) + tmp on p.id=tmp.id"; } STATS_DB=$1 @@ -35,8 +40,7 @@ createPDFsAggregated $MONITOR_DB'_institutions' createPDFsAggregated $MONITOR_DB'_ris_tail' contexts="knowmad::other dh-ch::other enermaps::other gotriple::other neanias-atmospheric::other rural-digital-europe::other covid-19::other aurora::other neanias-space::other north-america-studies::other north-american-studies::other eutopia::other" -for i in ${contexts} -do - tmp=`echo "$i" | sed 's/'-'/'_'/g' | sed 's/'::'/'_'/g'` +for i in ${contexts}; do + tmp=`echo "$i" | sed 's/'-'/'_'/g' | sed 's/'::'/'_'/g'` createPDFsAggregated ${MONITOR_DB}'_'${tmp} done \ No newline at end of file