From 9fc8ebe98b6e95faf27cd162ebf836f21447b168 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 19 Apr 2023 09:32:13 +0200 Subject: [PATCH 001/148] refactoring --- .../dhp/common/api/ZenodoAPIClient.java | 38 +++++++------------ 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java index 2aeccfcf2..544da78f5 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java @@ -9,13 +9,13 @@ import java.util.concurrent.TimeUnit; import org.apache.http.HttpHeaders; import org.apache.http.entity.ContentType; +import org.jetbrains.annotations.NotNull; import com.google.gson.Gson; import eu.dnetlib.dhp.common.api.zenodo.ZenodoModel; import eu.dnetlib.dhp.common.api.zenodo.ZenodoModelList; import okhttp3.*; -import org.jetbrains.annotations.NotNull; public class ZenodoAPIClient implements Serializable { @@ -80,7 +80,7 @@ public class ZenodoAPIClient implements Serializable { int responseCode = conn.getResponseCode(); conn.disconnect(); - if(!checkOKStatus(responseCode)) + if (!checkOKStatus(responseCode)) throw new IOException("Unexpected code " + responseCode + body); ZenodoModel newSubmission = new Gson().fromJson(body, ZenodoModel.class); @@ -115,7 +115,7 @@ public class ZenodoAPIClient implements Serializable { } int responseCode = conn.getResponseCode(); - if(! checkOKStatus(responseCode)){ + if (!checkOKStatus(responseCode)) { throw new IOException("Unexpected code " + responseCode + getBody(conn)); } @@ -126,7 +126,7 @@ public class ZenodoAPIClient implements Serializable { private String getBody(HttpURLConnection conn) throws IOException { String body = "{}"; try (BufferedReader br = new BufferedReader( - new InputStreamReader(conn.getInputStream(), "utf-8"))) { + new InputStreamReader(conn.getInputStream(), "utf-8"))) { StringBuilder response = new StringBuilder(); String responseLine = null; while ((responseLine = br.readLine()) != null) { @@ -155,7 +155,6 @@ public class ZenodoAPIClient implements Serializable { conn.setDoOutput(true); conn.setRequestMethod("PUT"); - try (OutputStream os = conn.getOutputStream()) { byte[] input = metadata.getBytes("utf-8"); os.write(input, 0, input.length); @@ -164,19 +163,18 @@ public class ZenodoAPIClient implements Serializable { final int responseCode = conn.getResponseCode(); conn.disconnect(); - if(!checkOKStatus(responseCode)) + if (!checkOKStatus(responseCode)) throw new IOException("Unexpected code " + responseCode + getBody(conn)); return responseCode; - } - private boolean checkOKStatus(int responseCode) { + private boolean checkOKStatus(int responseCode) { - if(HttpURLConnection.HTTP_OK != responseCode || - HttpURLConnection.HTTP_CREATED != responseCode) - return true ; + if (HttpURLConnection.HTTP_OK != responseCode || + HttpURLConnection.HTTP_CREATED != responseCode) + return true; return false; } @@ -233,7 +231,6 @@ public class ZenodoAPIClient implements Serializable { conn.setDoOutput(true); conn.setRequestMethod("POST"); - try (OutputStream os = conn.getOutputStream()) { byte[] input = json.getBytes("utf-8"); os.write(input, 0, input.length); @@ -245,7 +242,7 @@ public class ZenodoAPIClient implements Serializable { int responseCode = conn.getResponseCode(); conn.disconnect(); - if(!checkOKStatus(responseCode)) + if (!checkOKStatus(responseCode)) throw new IOException("Unexpected code " + responseCode + body); ZenodoModel zenodoModel = new Gson().fromJson(body, ZenodoModel.class); @@ -290,13 +287,12 @@ public class ZenodoAPIClient implements Serializable { int responseCode = conn.getResponseCode(); conn.disconnect(); - if(!checkOKStatus(responseCode)) + if (!checkOKStatus(responseCode)) throw new IOException("Unexpected code " + responseCode + body); ZenodoModel zenodoModel = new Gson().fromJson(body, ZenodoModel.class); bucket = zenodoModel.getLinks().getBucket(); - return responseCode; } @@ -331,22 +327,16 @@ public class ZenodoAPIClient implements Serializable { conn.setDoOutput(true); conn.setRequestMethod("GET"); - - String body = getBody(conn); int responseCode = conn.getResponseCode(); conn.disconnect(); - if(!checkOKStatus(responseCode)) + if (!checkOKStatus(responseCode)) throw new IOException("Unexpected code " + responseCode + body); - - return body; - - } private String getBucket(String inputUurl) throws IOException { @@ -363,15 +353,13 @@ public class ZenodoAPIClient implements Serializable { int responseCode = conn.getResponseCode(); conn.disconnect(); - if(!checkOKStatus(responseCode)) + if (!checkOKStatus(responseCode)) throw new IOException("Unexpected code " + responseCode + body); ZenodoModel zenodoModel = new Gson().fromJson(body, ZenodoModel.class); return zenodoModel.getLinks().getBucket(); - - } } From d9506035e47b951331d26ddf5ab484b9179d825d Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 9 Jun 2023 12:05:02 +0200 Subject: [PATCH 002/148] [ZenodoApi] gone back to okhttp3 to send the payload. --- .../dhp/common/api/ZenodoAPIClient.java | 76 +++++++++++++------ .../dhp/common/api/ZenodoAPIClientTest.java | 33 ++++++-- 2 files changed, 80 insertions(+), 29 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java index 544da78f5..a69624e3b 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java @@ -90,36 +90,68 @@ public class ZenodoAPIClient implements Serializable { return responseCode; } +// /** +// * Upload files in Zenodo. +// * +// * @param is the inputStream for the file to upload +// * @param file_name the name of the file as it will appear on Zenodo +// * @return the response code +// */ +// public int uploadIS(InputStream is, String file_name) throws IOException { +// +// URL url = new URL(bucket + "/" + file_name); +// HttpURLConnection conn = (HttpURLConnection) url.openConnection(); +// conn.setChunkedStreamingMode(8192); +// conn.setRequestProperty(HttpHeaders.CONTENT_TYPE, "application/zip"); +// conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token); +// conn.setDoOutput(true); +// conn.setRequestMethod("PUT"); +// conn.setRequestProperty("connection", "close"); +// +// byte[] buf = new byte[8192]; +// int length; +// try (OutputStream os = conn.getOutputStream()) { +// +// while ((length = is.read(buf)) != -1) { +// os.write(buf, 0, length); +// } +// +// } +// int responseCode = conn.getResponseCode(); +// if (!checkOKStatus(responseCode)) { +// throw new IOException("Unexpected code " + responseCode + getBody(conn)); +// } +// +// return responseCode; +// } + /** * Upload files in Zenodo. * * @param is the inputStream for the file to upload * @param file_name the name of the file as it will appear on Zenodo + * @param len the size of the file * @return the response code */ - public int uploadIS(InputStream is, String file_name) throws IOException { + public int uploadIS(InputStream is, String file_name, long len) throws IOException { + OkHttpClient httpClient = new OkHttpClient.Builder() + .writeTimeout(600, TimeUnit.SECONDS) + .readTimeout(600, TimeUnit.SECONDS) + .connectTimeout(600, TimeUnit.SECONDS) + .build(); - URL url = new URL(bucket + "/" + file_name); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - conn.setRequestProperty(HttpHeaders.CONTENT_TYPE, "application/zip"); - conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token); - conn.setDoOutput(true); - conn.setRequestMethod("PUT"); - - byte[] buf = new byte[8192]; - int length; - try (OutputStream os = conn.getOutputStream()) { - while ((length = is.read(buf)) != -1) { - os.write(buf, 0, length); - } + Request request = new Request.Builder() + .url(bucket + "/" + file_name) + .addHeader(HttpHeaders.CONTENT_TYPE, "application/zip") // add request headers + .addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token) + .put(InputStreamRequestBody.create(MEDIA_TYPE_ZIP, is, len)) + .build(); + try (Response response = httpClient.newCall(request).execute()) { + if (!response.isSuccessful()) + throw new IOException("Unexpected code " + response + response.body().string()); + return response.code(); } - int responseCode = conn.getResponseCode(); - if (!checkOKStatus(responseCode)) { - throw new IOException("Unexpected code " + responseCode + getBody(conn)); - } - - return responseCode; } @NotNull @@ -172,8 +204,8 @@ public class ZenodoAPIClient implements Serializable { private boolean checkOKStatus(int responseCode) { - if (HttpURLConnection.HTTP_OK != responseCode || - HttpURLConnection.HTTP_CREATED != responseCode) + if (HttpURLConnection.HTTP_OK == responseCode || + HttpURLConnection.HTTP_CREATED == responseCode) return true; return false; } diff --git a/dhp-common/src/test/java/eu/dnetlib/dhp/common/api/ZenodoAPIClientTest.java b/dhp-common/src/test/java/eu/dnetlib/dhp/common/api/ZenodoAPIClientTest.java index 92c1dcda3..87b68617b 100644 --- a/dhp-common/src/test/java/eu/dnetlib/dhp/common/api/ZenodoAPIClientTest.java +++ b/dhp-common/src/test/java/eu/dnetlib/dhp/common/api/ZenodoAPIClientTest.java @@ -15,7 +15,7 @@ import org.junit.jupiter.api.Test; class ZenodoAPIClientTest { private final String URL_STRING = "https://sandbox.zenodo.org/api/deposit/depositions"; - private final String ACCESS_TOKEN = ""; + private final String ACCESS_TOKEN = "OzzOsyucEIHxCEfhlpsMo3myEiwpCza3trCRL7ddfGTAK9xXkIP2MbXd6Vg4"; private final String CONCEPT_REC_ID = "657113"; @@ -33,7 +33,7 @@ class ZenodoAPIClientTest { InputStream is = new FileInputStream(file); - Assertions.assertEquals(200, client.uploadIS(is, "COVID-19.json.gz")); + Assertions.assertEquals(200, client.uploadIS(is, "COVID-19.json.gz", file.length())); String metadata = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/common/api/metadata.json")); @@ -51,18 +51,18 @@ class ZenodoAPIClientTest { Assertions.assertEquals(201, client.newDeposition()); File file = new File(getClass() - .getResource("/eu/dnetlib/dhp/common/api/COVID-19.json.gz") + .getResource("/eu/dnetlib/dhp/common/api/newVersion") .getPath()); InputStream is = new FileInputStream(file); - Assertions.assertEquals(200, client.uploadIS(is, "COVID-19.json.gz")); + Assertions.assertEquals(200, client.uploadIS(is, "COVID-19.json.gz", file.length())); String metadata = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/common/api/metadata.json")); Assertions.assertEquals(200, client.sendMretadata(metadata)); - Assertions.assertEquals(202, client.publish()); + // Assertions.assertEquals(202, client.publish()); } @@ -80,7 +80,7 @@ class ZenodoAPIClientTest { InputStream is = new FileInputStream(file); - Assertions.assertEquals(200, client.uploadIS(is, "newVersion_deposition")); + Assertions.assertEquals(200, client.uploadIS(is, "newVersion_deposition", file.length())); Assertions.assertEquals(202, client.publish()); @@ -100,10 +100,29 @@ class ZenodoAPIClientTest { InputStream is = new FileInputStream(file); - Assertions.assertEquals(200, client.uploadIS(is, "newVersion_deposition")); + Assertions.assertEquals(200, client.uploadIS(is, "newVersion_deposition", file.length())); Assertions.assertEquals(202, client.publish()); } + @Test + void depositBigFile() throws MissingConceptDoiException, IOException { + ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING, + ACCESS_TOKEN); + + Assertions.assertEquals(201, client.newDeposition()); + + File file = new File("/Users/miriam.baglioni/Desktop/EOSC_DUMP/publication.tar"); +// File file = new File(getClass() +// .getResource("/eu/dnetlib/dhp/common/api/newVersion2") +// .getPath()); + + InputStream is = new FileInputStream(file); + + Assertions.assertEquals(200, client.uploadIS(is, "newVersion_deposition", file.length())); + + //Assertions.assertEquals(202, client.publish()); + } + } From e4b27182d0c55ef2dd566e7d0f12da2b6e32a6b1 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 21 Jun 2023 11:15:53 +0200 Subject: [PATCH 003/148] [master] refactoring --- .../dhp/common/api/ZenodoAPIClient.java | 18 +++++++++--------- .../dhp/common/api/ZenodoAPIClientTest.java | 6 +++--- .../dnetlib/dhp/oa/graph/raw/MappersTest.java | 3 +-- .../provision/IndexRecordTransformerTest.java | 4 +++- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java index a69624e3b..0164bd249 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java @@ -135,17 +135,17 @@ public class ZenodoAPIClient implements Serializable { */ public int uploadIS(InputStream is, String file_name, long len) throws IOException { OkHttpClient httpClient = new OkHttpClient.Builder() - .writeTimeout(600, TimeUnit.SECONDS) - .readTimeout(600, TimeUnit.SECONDS) - .connectTimeout(600, TimeUnit.SECONDS) - .build(); + .writeTimeout(600, TimeUnit.SECONDS) + .readTimeout(600, TimeUnit.SECONDS) + .connectTimeout(600, TimeUnit.SECONDS) + .build(); Request request = new Request.Builder() - .url(bucket + "/" + file_name) - .addHeader(HttpHeaders.CONTENT_TYPE, "application/zip") // add request headers - .addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token) - .put(InputStreamRequestBody.create(MEDIA_TYPE_ZIP, is, len)) - .build(); + .url(bucket + "/" + file_name) + .addHeader(HttpHeaders.CONTENT_TYPE, "application/zip") // add request headers + .addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token) + .put(InputStreamRequestBody.create(MEDIA_TYPE_ZIP, is, len)) + .build(); try (Response response = httpClient.newCall(request).execute()) { if (!response.isSuccessful()) diff --git a/dhp-common/src/test/java/eu/dnetlib/dhp/common/api/ZenodoAPIClientTest.java b/dhp-common/src/test/java/eu/dnetlib/dhp/common/api/ZenodoAPIClientTest.java index 87b68617b..15ca81d17 100644 --- a/dhp-common/src/test/java/eu/dnetlib/dhp/common/api/ZenodoAPIClientTest.java +++ b/dhp-common/src/test/java/eu/dnetlib/dhp/common/api/ZenodoAPIClientTest.java @@ -62,7 +62,7 @@ class ZenodoAPIClientTest { Assertions.assertEquals(200, client.sendMretadata(metadata)); - // Assertions.assertEquals(202, client.publish()); + // Assertions.assertEquals(202, client.publish()); } @@ -109,7 +109,7 @@ class ZenodoAPIClientTest { @Test void depositBigFile() throws MissingConceptDoiException, IOException { ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING, - ACCESS_TOKEN); + ACCESS_TOKEN); Assertions.assertEquals(201, client.newDeposition()); @@ -122,7 +122,7 @@ class ZenodoAPIClientTest { Assertions.assertEquals(200, client.uploadIS(is, "newVersion_deposition", file.length())); - //Assertions.assertEquals(202, client.publish()); + // Assertions.assertEquals(202, client.publish()); } } diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java index bfd6d461d..55b49ee4f 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java @@ -1010,7 +1010,7 @@ class MappersTest { @Test void testD4Science() throws IOException { final String xml = IOUtils - .toString(Objects.requireNonNull(getClass().getResourceAsStream("d4science.xml"))); + .toString(Objects.requireNonNull(getClass().getResourceAsStream("d4science.xml"))); final List actual = new OdfToOafMapper(vocs, false, true).processMdRecord(xml); assertNotNull(actual); assertFalse(actual.isEmpty()); @@ -1023,7 +1023,6 @@ class MappersTest { } - private void assertValidId(final String id) { // System.out.println(id); diff --git a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/IndexRecordTransformerTest.java b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/IndexRecordTransformerTest.java index 74f203cbf..b5a5b5f0d 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/IndexRecordTransformerTest.java +++ b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/IndexRecordTransformerTest.java @@ -136,9 +136,11 @@ public class IndexRecordTransformerTest { @Test public void testForEdithDemoCovid() throws IOException, TransformerException { - final String record = IOUtils.toString(getClass().getResourceAsStream("edith-demo/10.3390-pr9111967-covid.xml")); + final String record = IOUtils + .toString(getClass().getResourceAsStream("edith-demo/10.3390-pr9111967-covid.xml")); testRecordTransformation(record); } + @Test public void testForEdithDemoEthics() throws IOException, TransformerException { final String record = IOUtils.toString(getClass().getResourceAsStream("edith-demo/10.2196-33081-ethics.xml")); From 8621377917082245383934e41b96ccb63e8e49ea Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 30 Jun 2023 19:02:44 +0200 Subject: [PATCH 004/148] [UsageCount] fixed typo in attribute name for datasource table --- .../dhp/actionmanager/usagestats/SparkAtomicActionUsageJob.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/usagestats/SparkAtomicActionUsageJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/usagestats/SparkAtomicActionUsageJob.java index 9b444c6fa..e62b80f6a 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/usagestats/SparkAtomicActionUsageJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/usagestats/SparkAtomicActionUsageJob.java @@ -75,7 +75,7 @@ public class SparkAtomicActionUsageJob implements Serializable { removeOutputDir(spark, outputPath); prepareData(dbname, spark, workingPath + "/usageDb", "usage_stats", "result_id"); prepareData(dbname, spark, workingPath + "/projectDb", "project_stats", "id"); - prepareData(dbname, spark, workingPath + "/datasourceDb", "datasource_stats", "repositor_id"); + prepareData(dbname, spark, workingPath + "/datasourceDb", "datasource_stats", "repository_id"); writeActionSet(spark, workingPath, outputPath); }); } From 163b2ee2a8f27755a36de97c2f6115d27b367165 Mon Sep 17 00:00:00 2001 From: dimitrispie Date: Thu, 13 Jul 2023 15:25:00 +0300 Subject: [PATCH 005/148] Changes 1. Monitor updates 2. Bug fixes during copy to impala cluster --- .../oozie_app/config-default.xml | 30 + .../oozie_app/copyDataToImpalaCluster.sh | 75 ++ .../oozie_app/finalizeImpalaCluster.sh | 29 + .../graph/stats-monitor/oozie_app/monitor.sh | 54 ++ .../oozie_app/scripts/updateMonitorDB.sql | 138 ++++ .../oozie_app/scripts/updateMonitorDBAll.sql | 150 ++++ .../scripts/updateMonitorDB_institutions.sql | 12 + .../stats-monitor/oozie_app/workflow.xml | 110 +++ .../oozie_app/copyDataToImpalaCluster.sh | 8 +- .../stats/oozie_app/finalizeImpalaCluster.sh | 10 +- .../dhp/oa/graph/stats/oozie_app/monitor.sh | 22 +- .../graph/stats/oozie_app/scripts/step15.sql | 11 +- .../scripts/step16-createIndicatorsTables.sql | 718 +++++++++--------- .../scripts/step20-createMonitorDB.sql | 106 +-- .../scripts/step20-createMonitorDBAll.sql | 276 +++++++ .../scripts/step20-createMonitorDB_RIs.sql | 2 +- .../step20-createMonitorDB_RIs_tail.sql | 2 +- .../scripts/step20-createMonitorDB_funded.sql | 2 +- .../step20-createMonitorDB_institutions.sql | 9 +- .../scripts/step21-createObservatoryDB.sql | 38 +- .../dhp/oa/graph/stats/oozie_app/workflow.xml | 18 +- 21 files changed, 1347 insertions(+), 473 deletions(-) create mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/config-default.xml create mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh create mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/finalizeImpalaCluster.sh create mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/monitor.sh create mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB.sql create mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDBAll.sql create mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB_institutions.sql create mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/workflow.xml create mode 100644 dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/config-default.xml b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/config-default.xml new file mode 100644 index 000000000..b2a1322e6 --- /dev/null +++ b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/config-default.xml @@ -0,0 +1,30 @@ + + + jobTracker + ${jobTracker} + + + nameNode + ${nameNode} + + + oozie.use.system.libpath + true + + + oozie.action.sharelib.for.spark + spark2 + + + hive_metastore_uris + thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083 + + + hive_jdbc_url + jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000/;UseNativeQuery=1;?spark.executor.memory=22166291558;spark.yarn.executor.memoryOverhead=3225;spark.driver.memory=15596411699;spark.yarn.driver.memoryOverhead=1228 + + + oozie.wf.workflow.notification.url + {serviceUrl}/v1/oozieNotification/jobUpdate?jobId=$jobId%26status=$status + + \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh new file mode 100644 index 000000000..1587f7152 --- /dev/null +++ b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh @@ -0,0 +1,75 @@ +export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs +export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami) +if ! [ -L $link_folder ] +then + rm -Rf "$link_folder" + ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder} +fi + +#export HADOOP_USER_NAME=$2 + +function copydb() { + + export HADOOP_USER="dimitris.pierrakos" + export HADOOP_USER_NAME='dimitris.pierrakos' + + db=$1 + FILE=("hive_wf_tmp_"$RANDOM) + hdfs dfs -mkdir hdfs://impala-cluster-mn1.openaire.eu:8020/tmp/$FILE/ + + # change ownership to impala +# hdfs dfs -conf /etc/impala_cluster/hdfs-site.xml -chmod -R 777 /tmp/$FILE/${db}.db + hdfs dfs -conf /etc/impala_cluster/hdfs-site.xml -chmod -R 777 /tmp/$FILE/ + + + # copy the databases from ocean to impala + echo "copying $db" + hadoop distcp -Dmapreduce.map.memory.mb=6144 -pb hdfs://nameservice1/user/hive/warehouse/${db}.db hdfs://impala-cluster-mn1.openaire.eu:8020/tmp/$FILE/ + + hdfs dfs -conf /etc/impala_cluster/hdfs-site.xml -chmod -R 777 /tmp/$FILE/${db}.db + + # drop tables from db + for i in `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} --delimited -q "show tables"`; + do + `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "drop table $i;"`; + done + + # drop views from db + for i in `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} --delimited -q "show tables"`; + do + `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "drop view $i;"`; + done + + # delete the database + impala-shell -i impala-cluster-dn1.openaire.eu -q "drop database if exists ${db} cascade"; + + # create the databases + impala-shell -i impala-cluster-dn1.openaire.eu -q "create database ${db}"; + + impala-shell -q "INVALIDATE METADATA" + echo "creating schema for ${db}" + for (( k = 0; k < 5; k ++ )); do + for i in `impala-shell -d ${db} --delimited -q "show tables"`; + do + impala-shell -d ${db} --delimited -q "show create table $i"; + done | sed 's/"$/;/' | sed 's/^"//' | sed 's/[[:space:]]\date[[:space:]]/`date`/g' | impala-shell --user $HADOOP_USER_NAME -i impala-cluster-dn1.openaire.eu -c -f - + done + + # load the data from /tmp in the respective tables + echo "copying data in tables and computing stats" + for i in `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} --delimited -q "show tables"`; + do + impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "load data inpath '/tmp/$FILE/${db}.db/$i' into table $i"; + impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "compute stats $i"; + done + + # deleting the remaining directory from hdfs +hdfs dfs -conf /etc/impala_cluster/hdfs-site.xml -rm -R /tmp/$FILE/${db}.db +} + +MONITOR_DB=$1 +#HADOOP_USER_NAME=$2 + +copydb $MONITOR_DB'_institutions' +copydb $MONITOR_DB + diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/finalizeImpalaCluster.sh b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/finalizeImpalaCluster.sh new file mode 100644 index 000000000..a7227e0c8 --- /dev/null +++ b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/finalizeImpalaCluster.sh @@ -0,0 +1,29 @@ +export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs +export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami) +if ! [ -L $link_folder ] +then + rm -Rf "$link_folder" + ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder} +fi + +function createShadowDB() { + SOURCE=$1 + SHADOW=$2 + + # drop views from db + for i in `impala-shell -i impala-cluster-dn1.openaire.eu -d ${SHADOW} --delimited -q "show tables"`; + do + `impala-shell -i impala-cluster-dn1.openaire.eu -d ${SHADOW} -q "drop view $i;"`; + done + + impala-shell -i impala-cluster-dn1.openaire.eu -q "drop database ${SHADOW} CASCADE"; + impala-shell -i impala-cluster-dn1.openaire.eu -q "create database if not exists ${SHADOW}"; +# impala-shell -i impala-cluster-dn1.openaire.eu -d ${SHADOW} -q "show tables" | sed "s/^/drop view if exists ${SHADOW}./" | sed "s/$/;/" | impala-shell -i impala-cluster-dn1.openaire.eu -f - + impala-shell -i impala-cluster-dn1.openaire.eu -d ${SOURCE} -q "show tables" --delimited | sed "s/\(.*\)/create view ${SHADOW}.\1 as select * from ${SOURCE}.\1;/" | impala-shell -i impala-cluster-dn1.openaire.eu -f - +} + +MONITOR_DB=$1 +MONITOR_DB_SHADOW=$2 + +createShadowDB $MONITOR_DB'_institutions' $MONITOR_DB'_institutions_shadow' +createShadowDB $MONITOR_DB $MONITOR_DB'_shadow' diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/monitor.sh b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/monitor.sh new file mode 100644 index 000000000..4f1889c9e --- /dev/null +++ b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/monitor.sh @@ -0,0 +1,54 @@ +export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs +export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami) +if ! [ -L $link_folder ] +then + rm -Rf "$link_folder" + ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder} +fi + +export SOURCE=$1 +export TARGET=$2 +export SHADOW=$3 +export SCRIPT_PATH=$4 +export SCRIPT_PATH2=$5 +export SCRIPT_PATH2=$6 + +export HIVE_OPTS="-hiveconf mapred.job.queue.name=analytics -hiveconf hive.spark.client.connect.timeout=120000ms -hiveconf hive.spark.client.server.connect.timeout=300000ms -hiveconf spark.executor.memory=19166291558 -hiveconf spark.yarn.executor.memoryOverhead=3225 -hiveconf spark.driver.memory=11596411699 -hiveconf spark.yarn.driver.memoryOverhead=1228" +export HADOOP_USER_NAME="oozie" + +echo "Getting file from " $4 +hdfs dfs -copyToLocal $4 + +echo "Getting file from " $5 +hdfs dfs -copyToLocal $5 + +echo "Getting file from " $6 +hdfs dfs -copyToLocal $6 + +#update Institutions DB +cat updateMonitorDB_institutions.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2_institutions/g1" > foo +hive $HIVE_OPTS -f foo +cat updateMonitorDB.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2_institutions/g1" > foo +hive $HIVE_OPTS -f foo + +echo "Hive shell finished" + +echo "Updating shadow monitor insitutions database" +hive -e "drop database if exists ${SHADOW}_institutions cascade" +hive -e "create database if not exists ${SHADOW}_institutions" +hive $HIVE_OPTS --database ${2}_institutions -e "show tables" | grep -v WARN | sed "s/\(.*\)/create view ${SHADOW}_institutions.\1 as select * from ${2}_institutions.\1;/" > foo +hive -f foo +echo "Shadow db monitor insitutions ready!" + +#update Monitor DB +cat updateMonitorDBAll.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2/g1" > foo +hive $HIVE_OPTS -f foo + +echo "Hive shell finished" + +echo "Updating shadow monitor database" +hive -e "drop database if exists ${SHADOW} cascade" +hive -e "create database if not exists ${SHADOW}" +hive $HIVE_OPTS --database ${2} -e "show tables" | grep -v WARN | sed "s/\(.*\)/create view ${SHADOW}.\1 as select * from ${2}.\1;/" > foo +hive -f foo +echo "Shadow db monitor insitutions ready!" diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB.sql b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB.sql new file mode 100644 index 000000000..248b7e564 --- /dev/null +++ b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB.sql @@ -0,0 +1,138 @@ +INSERT INTO TARGET.result select * from TARGET.result_new; +ANALYZE TABLE TARGET.result COMPUTE STATISTICS; + +INSERT INTO TARGET.result_citations select * from SOURCE.result_citations orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_citations COMPUTE STATISTICS; + +INSERT INTO TARGET.result_references_oc select * from SOURCE.result_references_oc orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_references_oc COMPUTE STATISTICS; + +INSERT INTO TARGET.result_classifications select * from SOURCE.result_classifications orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_classifications COMPUTE STATISTICS; + +INSERT INTO TARGET.result_apc select * from SOURCE.result_apc orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_apc COMPUTE STATISTICS; + +INSERT INTO TARGET.result_concepts select * from SOURCE.result_concepts orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_concepts COMPUTE STATISTICS; + +INSERT INTO TARGET.result_datasources select * from SOURCE.result_datasources orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_datasources COMPUTE STATISTICS; + +INSERT INTO TARGET.result_fundercount select * from SOURCE.result_fundercount orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_fundercount COMPUTE STATISTICS; + +INSERT INTO TARGET.result_gold select * from SOURCE.result_gold orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_gold COMPUTE STATISTICS; + +INSERT INTO TARGET.result_greenoa select * from SOURCE.result_greenoa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_greenoa COMPUTE STATISTICS; + +INSERT INTO TARGET.result_languages select * from SOURCE.result_languages orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_languages COMPUTE STATISTICS; + +INSERT INTO TARGET.result_licenses select * from SOURCE.result_licenses orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_licenses COMPUTE STATISTICS; + +INSERT INTO TARGET.result_oids select * from SOURCE.result_oids orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_oids COMPUTE STATISTICS; + +INSERT INTO TARGET.result_organization select * from SOURCE.result_organization orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_organization COMPUTE STATISTICS; + +INSERT INTO TARGET.result_peerreviewed select * from SOURCE.result_peerreviewed orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_peerreviewed COMPUTE STATISTICS; + +INSERT INTO TARGET.result_pids select * from SOURCE.result_pids orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_pids COMPUTE STATISTICS; + +INSERT INTO TARGET.result_projectcount select * from SOURCE.result_projectcount orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_projectcount COMPUTE STATISTICS; + +INSERT INTO TARGET.result_projects select * from SOURCE.result_projects orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_projects COMPUTE STATISTICS; + +INSERT INTO TARGET.result_refereed select * from SOURCE.result_refereed orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_refereed COMPUTE STATISTICS; + +INSERT INTO TARGET.result_sources select * from SOURCE.result_sources orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_sources COMPUTE STATISTICS; + +INSERT INTO TARGET.result_topics select * from SOURCE.result_topics orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_topics COMPUTE STATISTICS; + +INSERT INTO TARGET.result_fos select * from SOURCE.result_fos orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_fos COMPUTE STATISTICS; + +INSERT INTO TARGET.result_accessroute select * from SOURCE.result_accessroute orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_accessroute COMPUTE STATISTICS; + +create or replace view TARGET.foo1 as select * from SOURCE.result_result rr where rr.source in (select id from TARGET.result_new); +create or replace view TARGET.foo2 as select * from SOURCE.result_result rr where rr.target in (select id from TARGET.result_new); +insert into TARGET.result_result select distinct * from (select * from TARGET.foo1 union all select * from TARGET.foo2) foufou; +drop view TARGET.foo1; +drop view TARGET.foo2; +ANALYZE TABLE TARGET.result_result COMPUTE STATISTICS; + + +-- indicators +-- Sprint 1 ---- +INSERT INTO TARGET.indi_pub_green_oa select * from SOURCE.indi_pub_green_oa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_green_oa COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_grey_lit select * from SOURCE.indi_pub_grey_lit orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_grey_lit COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_doi_from_crossref select * from SOURCE.indi_pub_doi_from_crossref orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_doi_from_crossref COMPUTE STATISTICS; +-- Sprint 2 ---- +INSERT INTO TARGET.indi_result_has_cc_licence select * from SOURCE.indi_result_has_cc_licence orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_result_has_cc_licence COMPUTE STATISTICS; +INSERT INTO TARGET.indi_result_has_cc_licence_url select * from SOURCE.indi_result_has_cc_licence_url orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_result_has_cc_licence_url COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_has_abstract select * from SOURCE.indi_pub_has_abstract orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_has_abstract COMPUTE STATISTICS; +INSERT INTO TARGET.indi_result_with_orcid select * from SOURCE.indi_result_with_orcid orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_result_with_orcid COMPUTE STATISTICS; +---- Sprint 3 ---- +INSERT INTO TARGET.indi_funded_result_with_fundref select * from SOURCE.indi_funded_result_with_fundref orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_funded_result_with_fundref COMPUTE STATISTICS; + +---- Sprint 4 ---- +INSERT INTO TARGET.indi_pub_diamond select * from SOURCE.indi_pub_diamond orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_diamond COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_in_transformative select * from SOURCE.indi_pub_in_transformative orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_in_transformative COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_closed_other_open select * from SOURCE.indi_pub_closed_other_open orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_closed_other_open COMPUTE STATISTICS; +---- Sprint 5 ---- +INSERT INTO TARGET.indi_result_no_of_copies select * from SOURCE.indi_result_no_of_copies orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_result_no_of_copies COMPUTE STATISTICS; +---- Sprint 6 ---- +INSERT INTO TARGET.indi_pub_hybrid_oa_with_cc select * from SOURCE.indi_pub_hybrid_oa_with_cc orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_hybrid_oa_with_cc COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_bronze_oa select * from SOURCE.indi_pub_bronze_oa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_bronze_oa COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_downloads select * from SOURCE.indi_pub_downloads orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); +ANALYZE TABLE TARGET.indi_pub_downloads COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_downloads_datasource select * from SOURCE.indi_pub_downloads_datasource orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); +ANALYZE TABLE TARGET.indi_pub_downloads_datasource COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_downloads_year select * from SOURCE.indi_pub_downloads_year orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); +ANALYZE TABLE TARGET.indi_pub_downloads_year COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_downloads_datasource_year select * from SOURCE.indi_pub_downloads_datasource_year orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); +ANALYZE TABLE TARGET.indi_pub_downloads_datasource_year COMPUTE STATISTICS; +---- Sprint 7 ---- +INSERT INTO TARGET.indi_pub_gold_oa select * from SOURCE.indi_pub_gold_oa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_gold_oa COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_hybrid select * from SOURCE.indi_pub_hybrid orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_hybrid COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_has_preprint select * from SOURCE.indi_pub_has_preprint orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_has_preprint COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_in_subscribed select * from SOURCE.indi_pub_in_subscribed orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_in_subscribed COMPUTE STATISTICS; +INSERT INTO TARGET.indi_result_with_pid select * from SOURCE.indi_result_with_pid orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_result_with_pid COMPUTE STATISTICS; +INSERT INTO TARGET.indi_impact_measures select * from SOURCE.indi_impact_measures orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_impact_measures COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_interdisciplinarity select * from SOURCE.indi_pub_interdisciplinarity orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_interdisciplinarity COMPUTE STATISTICS; + +DROP TABLE IF EXISTS TARGET.result_new; \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDBAll.sql b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDBAll.sql new file mode 100644 index 000000000..478e3824e --- /dev/null +++ b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDBAll.sql @@ -0,0 +1,150 @@ +DROP TABLE IF EXISTS TARGET.result_new; + +create table TARGET.result_new as + select distinct * from ( + select * from SOURCE.result r where exists (select 1 from SOURCE.result_organization ro where ro.id=r.id and ro.organization in ( + 'openorgs____::4d4051b56708688235252f1d8fddb8c1', --Iscte - Instituto Universitário de Lisboa + 'openorgs____::ab4ac74c35fa5dada770cf08e5110fab' -- Universidade Católica Portuguesa + ) )) foo; + +INSERT INTO TARGET.result select * from TARGET.result_new; +ANALYZE TABLE TARGET.result_new COMPUTE STATISTICS; + +INSERT INTO TARGET.result select * from TARGET.result_new; +ANALYZE TABLE TARGET.result COMPUTE STATISTICS; + +INSERT INTO TARGET.result_citations select * from SOURCE.result_citations orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_citations COMPUTE STATISTICS; + +INSERT INTO TARGET.result_references_oc select * from SOURCE.result_references_oc orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_references_oc COMPUTE STATISTICS; + +INSERT INTO TARGET.result_classifications select * from SOURCE.result_classifications orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_classifications COMPUTE STATISTICS; + +INSERT INTO TARGET.result_apc select * from SOURCE.result_apc orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_apc COMPUTE STATISTICS; + +INSERT INTO TARGET.result_concepts select * from SOURCE.result_concepts orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_concepts COMPUTE STATISTICS; + +INSERT INTO TARGET.result_datasources select * from SOURCE.result_datasources orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_datasources COMPUTE STATISTICS; + +INSERT INTO TARGET.result_fundercount select * from SOURCE.result_fundercount orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_fundercount COMPUTE STATISTICS; + +INSERT INTO TARGET.result_gold select * from SOURCE.result_gold orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_gold COMPUTE STATISTICS; + +INSERT INTO TARGET.result_greenoa select * from SOURCE.result_greenoa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_greenoa COMPUTE STATISTICS; + +INSERT INTO TARGET.result_languages select * from SOURCE.result_languages orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_languages COMPUTE STATISTICS; + +INSERT INTO TARGET.result_licenses select * from SOURCE.result_licenses orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_licenses COMPUTE STATISTICS; + +INSERT INTO TARGET.result_oids select * from SOURCE.result_oids orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_oids COMPUTE STATISTICS; + +INSERT INTO TARGET.result_organization select * from SOURCE.result_organization orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_organization COMPUTE STATISTICS; + +INSERT INTO TARGET.result_peerreviewed select * from SOURCE.result_peerreviewed orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_peerreviewed COMPUTE STATISTICS; + +INSERT INTO TARGET.result_pids select * from SOURCE.result_pids orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_pids COMPUTE STATISTICS; + +INSERT INTO TARGET.result_projectcount select * from SOURCE.result_projectcount orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_projectcount COMPUTE STATISTICS; + +INSERT INTO TARGET.result_projects select * from SOURCE.result_projects orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_projects COMPUTE STATISTICS; + +INSERT INTO TARGET.result_refereed select * from SOURCE.result_refereed orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_refereed COMPUTE STATISTICS; + +INSERT INTO TARGET.result_sources select * from SOURCE.result_sources orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_sources COMPUTE STATISTICS; + +INSERT INTO TARGET.result_topics select * from SOURCE.result_topics orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_topics COMPUTE STATISTICS; + +INSERT INTO TARGET.result_fos select * from SOURCE.result_fos orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_fos COMPUTE STATISTICS; + +INSERT INTO TARGET.result_accessroute select * from SOURCE.result_accessroute orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_accessroute COMPUTE STATISTICS; + +create or replace view TARGET.foo1 as select * from SOURCE.result_result rr where rr.source in (select id from TARGET.result_new); +create or replace view TARGET.foo2 as select * from SOURCE.result_result rr where rr.target in (select id from TARGET.result_new); +insert into TARGET.result_result select distinct * from (select * from TARGET.foo1 union all select * from TARGET.foo2) foufou; +drop view TARGET.foo1; +drop view TARGET.foo2; +ANALYZE TABLE TARGET.result_result COMPUTE STATISTICS; + + +-- indicators +-- Sprint 1 ---- +INSERT INTO TARGET.indi_pub_green_oa select * from SOURCE.indi_pub_green_oa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_green_oa COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_grey_lit select * from SOURCE.indi_pub_grey_lit orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_grey_lit COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_doi_from_crossref select * from SOURCE.indi_pub_doi_from_crossref orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_doi_from_crossref COMPUTE STATISTICS; +-- Sprint 2 ---- +INSERT INTO TARGET.indi_result_has_cc_licence select * from SOURCE.indi_result_has_cc_licence orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_result_has_cc_licence COMPUTE STATISTICS; +INSERT INTO TARGET.indi_result_has_cc_licence_url select * from SOURCE.indi_result_has_cc_licence_url orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_result_has_cc_licence_url COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_has_abstract select * from SOURCE.indi_pub_has_abstract orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_has_abstract COMPUTE STATISTICS; +INSERT INTO TARGET.indi_result_with_orcid select * from SOURCE.indi_result_with_orcid orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_result_with_orcid COMPUTE STATISTICS; +---- Sprint 3 ---- +INSERT INTO TARGET.indi_funded_result_with_fundref select * from SOURCE.indi_funded_result_with_fundref orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_funded_result_with_fundref COMPUTE STATISTICS; + +---- Sprint 4 ---- +INSERT INTO TARGET.indi_pub_diamond select * from SOURCE.indi_pub_diamond orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_diamond COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_in_transformative select * from SOURCE.indi_pub_in_transformative orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_in_transformative COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_closed_other_open select * from SOURCE.indi_pub_closed_other_open orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_closed_other_open COMPUTE STATISTICS; +---- Sprint 5 ---- +INSERT INTO TARGET.indi_result_no_of_copies select * from SOURCE.indi_result_no_of_copies orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_result_no_of_copies COMPUTE STATISTICS; +---- Sprint 6 ---- +INSERT INTO TARGET.indi_pub_hybrid_oa_with_cc select * from SOURCE.indi_pub_hybrid_oa_with_cc orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_hybrid_oa_with_cc COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_bronze_oa select * from SOURCE.indi_pub_bronze_oa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_bronze_oa COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_downloads select * from SOURCE.indi_pub_downloads orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); +ANALYZE TABLE TARGET.indi_pub_downloads COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_downloads_datasource select * from SOURCE.indi_pub_downloads_datasource orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); +ANALYZE TABLE TARGET.indi_pub_downloads_datasource COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_downloads_year select * from SOURCE.indi_pub_downloads_year orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); +ANALYZE TABLE TARGET.indi_pub_downloads_year COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_downloads_datasource_year select * from SOURCE.indi_pub_downloads_datasource_year orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); +ANALYZE TABLE TARGET.indi_pub_downloads_datasource_year COMPUTE STATISTICS; +---- Sprint 7 ---- +INSERT INTO TARGET.indi_pub_gold_oa select * from SOURCE.indi_pub_gold_oa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_gold_oa COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_hybrid select * from SOURCE.indi_pub_hybrid orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_hybrid COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_has_preprint select * from SOURCE.indi_pub_has_preprint orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_has_preprint COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_in_subscribed select * from SOURCE.indi_pub_in_subscribed orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_in_subscribed COMPUTE STATISTICS; +INSERT INTO TARGET.indi_result_with_pid select * from SOURCE.indi_result_with_pid orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_result_with_pid COMPUTE STATISTICS; +INSERT INTO TARGET.indi_impact_measures select * from SOURCE.indi_impact_measures orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_impact_measures COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_interdisciplinarity select * from SOURCE.indi_pub_interdisciplinarity orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_interdisciplinarity COMPUTE STATISTICS; + +DROP TABLE IF EXISTS TARGET.result_new; diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB_institutions.sql b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB_institutions.sql new file mode 100644 index 000000000..236f3733f --- /dev/null +++ b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB_institutions.sql @@ -0,0 +1,12 @@ +DROP TABLE IF EXISTS TARGET.result_new; + +create table TARGET.result_new as + select distinct * from ( + select * from SOURCE.result r where exists (select 1 from SOURCE.result_organization ro where ro.id=r.id and ro.organization in ( + 'openorgs____::4d4051b56708688235252f1d8fddb8c1', --Iscte - Instituto Universitário de Lisboa + 'openorgs____::ab4ac74c35fa5dada770cf08e5110fab' -- Universidade Católica Portuguesa + ) )) foo; + +INSERT INTO TARGET.result select * from TARGET.result_new; +ANALYZE TABLE TARGET.result_new COMPUTE STATISTICS; + diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/workflow.xml b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/workflow.xml new file mode 100644 index 000000000..7b999a843 --- /dev/null +++ b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/workflow.xml @@ -0,0 +1,110 @@ + + + + stats_db_name + the target stats database name + + + monitor_db_name + the target monitor db name + + + monitor_db_shadow_name + the name of the shadow monitor db + + + hive_metastore_uris + hive server metastore URIs + + + hive_jdbc_url + hive server jdbc url + + + hive_timeout + the time period, in seconds, after which Hive fails a transaction if a Hive client has not sent a hearbeat. The default value is 300 seconds. + + + hadoop_user_name + user name of the wf owner + + + + + ${jobTracker} + ${nameNode} + + + hive.metastore.uris + ${hive_metastore_uris} + + + hive.txn.timeout + ${hive_timeout} + + + mapred.job.queue.name + analytics + + + + + + + + ${wf:conf('resumeFrom') eq 'Step1-updateMonitorDB'} + ${wf:conf('resumeFrom') eq 'Step2-copyDataToImpalaCluster'} + ${wf:conf('resumeFrom') eq 'Step3-finalizeImpalaCluster'} + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + ${jobTracker} + ${nameNode} + monitor.sh + ${stats_db_name} + ${monitor_db_name} + ${monitor_db_shadow_name} + ${wf:appPath()}/scripts/updateMonitorDB_institutions.sql + ${wf:appPath()}/scripts/updateMonitorDB.sql + ${wf:appPath()}/scripts/updateMonitorDBAll.sql + monitor.sh + + + + + + + + ${jobTracker} + ${nameNode} + copyDataToImpalaCluster.sh + ${monitor_db_name} + ${hadoop_user_name} + copyDataToImpalaCluster.sh + + + + + + + + ${jobTracker} + ${nameNode} + finalizeImpalaCluster.sh + ${monitor_db_name} + ${monitor_db_shadow_name} + finalizeImpalaCluster.sh + + + + + + + diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh index 87294f6e9..431978997 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh @@ -24,13 +24,13 @@ function copydb() { # drop tables from db for i in `impala-shell --user $HADOOP_USER_NAME -i impala-cluster-dn1.openaire.eu -d ${db} --delimited -q "show tables"`; do - `impala-shell -i impala-cluster-dn1.openaire.eu -d -d ${db} -q "drop table $i;"`; + `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "drop table $i;"`; done # drop views from db for i in `impala-shell --user $HADOOP_USER_NAME -i impala-cluster-dn1.openaire.eu -d ${db} --delimited -q "show tables"`; do - `impala-shell -i impala-cluster-dn1.openaire.eu -d -d ${db} -q "drop view $i;"`; + `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "drop view $i;"`; done # delete the database @@ -82,12 +82,12 @@ copydb $USAGE_STATS_DB copydb $PROD_USAGE_STATS_DB copydb $EXT_DB copydb $STATS_DB -#copydb $MONITOR_DB +copydb $MONITOR_DB copydb $OBSERVATORY_DB copydb $MONITOR_DB'_funded' copydb $MONITOR_DB'_institutions' -copydb $MONITOR_DB'_RIs_tail' +copydb $MONITOR_DB'_ris_tail' contexts="knowmad::other dh-ch::other enermaps::other gotriple::other neanias-atmospheric::other rural-digital-europe::other covid-19::other aurora::other neanias-space::other north-america-studies::other north-american-studies::other eutopia::other" for i in ${contexts} diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/finalizeImpalaCluster.sh b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/finalizeImpalaCluster.sh index 857635b6c..86a93216c 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/finalizeImpalaCluster.sh +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/finalizeImpalaCluster.sh @@ -13,7 +13,7 @@ function createShadowDB() { # drop views from db for i in `impala-shell -i impala-cluster-dn1.openaire.eu -d ${SHADOW} --delimited -q "show tables"`; do - `impala-shell -i impala-cluster-dn1.openaire.eu -d -d ${SHADOW} -q "drop view $i;"`; + `impala-shell -i impala-cluster-dn1.openaire.eu -d ${SHADOW} -q "drop view $i;"`; done impala-shell -i impala-cluster-dn1.openaire.eu -q "drop database ${SHADOW} CASCADE"; @@ -36,13 +36,13 @@ createShadowDB $MONITOR_DB $MONITOR_DB_SHADOW createShadowDB $OBSERVATORY_DB $OBSERVATORY_DB_SHADOW createShadowDB USAGE_STATS_DB USAGE_STATS_DB_SHADOW -createShadowDB $MONITOR_DB'_funded' $MONITOR_DB'_funded_shadow' -createShadowDB $MONITOR_DB'_institutions' $MONITOR_DB'_institutions_shadow' -createShadowDB $MONITOR_DB'_RIs_tail' $MONITOR_DB'_RIs_tail_shadow' +createShadowDB $MONITOR_DB'_funded' $MONITOR_DB_SHADOW'_shadow_funded' +createShadowDB $MONITOR_DB'_institutions' $MONITOR_DB_SHADOW'_shadow_institutions' +createShadowDB $MONITOR_DB'_ris_tail' $MONITOR_DB_SHADOW'_shadow_ris_tail' contexts="knowmad::other dh-ch::other enermaps::other gotriple::other neanias-atmospheric::other rural-digital-europe::other covid-19::other aurora::other neanias-space::other north-america-studies::other north-american-studies::other eutopia::other" for i in ${contexts} do tmp=`echo "$i" | sed 's/'-'/'_'/g' | sed 's/'::'/'_'/g'` - createShadowDB ${MONITOR_DB}'_'${tmp} ${MONITOR_DB}'_'${tmp}'_shadow' + createShadowDB ${MONITOR_DB}'_'${tmp} ${MONITOR_DB_SHADOW}'_shadow_'${tmp} done \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/monitor.sh b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/monitor.sh index 08f4c9232..014b19c6c 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/monitor.sh +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/monitor.sh @@ -14,6 +14,7 @@ export SCRIPT_PATH2=$5 export SCRIPT_PATH3=$6 export SCRIPT_PATH4=$7 export SCRIPT_PATH5=$8 +export SCRIPT_PATH6=$9 export HIVE_OPTS="-hiveconf mapred.job.queue.name=analytics -hiveconf hive.spark.client.connect.timeout=120000ms -hiveconf hive.spark.client.server.connect.timeout=300000ms -hiveconf spark.executor.memory=19166291558 -hiveconf spark.yarn.executor.memoryOverhead=3225 -hiveconf spark.driver.memory=11596411699 -hiveconf spark.yarn.driver.memoryOverhead=1228" export HADOOP_USER_NAME="oozie" @@ -33,12 +34,19 @@ hdfs dfs -copyToLocal $7 echo "Getting file from " $8 hdfs dfs -copyToLocal $8 +echo "Getting file from " $9 +hdfs dfs -copyToLocal $9 + + echo "Creating monitor database" +cat step20-createMonitorDBAll.sql | sed "s/SOURCE/openaire_prod_stats_20230707/g" | sed "s/TARGET/openaire_prod_stats_monitor_20230707/g1" > foo +hive $HIVE_OPTS -f foo + cat step20-createMonitorDB_funded.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2_funded/g1" > foo hive $HIVE_OPTS -f foo cat step20-createMonitorDB.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2_funded/g1" > foo hive $HIVE_OPTS -f foo -# + cat step20-createMonitorDB_institutions.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2_institutions/g1" > foo hive $HIVE_OPTS -f foo cat step20-createMonitorDB.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2_institutions/g1" > foo @@ -56,14 +64,20 @@ do hive $HIVE_OPTS -f foo done - -cat step20-createMonitorDB_RIs_tail.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2_RIs_tail/g1" | sed "s/CONTEXTS/\"'knowmad::other','dh-ch::other', 'enermaps::other', 'gotriple::other', 'neanias-atmospheric::other', 'rural-digital-europe::other', 'covid-19::other', 'aurora::other', 'neanias-space::other', 'north-america-studies::other', 'north-american-studies::other', 'eutopia::other'\"/g" > foo +cat step20-createMonitorDB_RIs_tail.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2_ris_tail/g1" | sed "s/CONTEXTS/\"'knowmad::other','dh-ch::other', 'enermaps::other', 'gotriple::other', 'neanias-atmospheric::other', 'rural-digital-europe::other', 'covid-19::other', 'aurora::other', 'neanias-space::other', 'north-america-studies::other', 'north-american-studies::other', 'eutopia::other'\"/g" > foo hive $HIVE_OPTS -f foo -cat step20-createMonitorDB.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2_RIs_tail/g1" > foo +cat step20-createMonitorDB.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2_ris_tail/g1" > foo hive $HIVE_OPTS -f foo echo "Hive shell finished" +echo "Updating shadow monitor all database" +hive -e "drop database if exists ${SHADOW} cascade" +hive -e "create database if not exists ${SHADOW}" +hive $HIVE_OPTS --database ${2} -e "show tables" | grep -v WARN | sed "s/\(.*\)/create view ${SHADOW}.\1 as select * from ${2}.\1;/" > foo +hive -f foo +echo "Updated shadow monitor all database" + echo "Updating shadow monitor funded database" hive -e "drop database if exists ${SHADOW}_funded cascade" hive -e "create database if not exists ${SHADOW}_funded" diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql index 132cb482e..75e8b001b 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql @@ -37,8 +37,17 @@ select * from ${stats_db_name}.otherresearchproduct_refereed; create table if not exists ${stats_db_name}.indi_impact_measures STORED AS PARQUET as select substr(id, 4) as id, measures_ids.id impactmetric, cast(measures_ids.unit.value[0] as double) score, -cast(measures_ids.unit.value[0] as decimal(6,3)) score_dec, measures_ids.unit.value[1] class +cast(measures_ids.unit.value[0] as decimal(6,3)) score_dec, measures_ids.unit.value[1] impact_class from ${openaire_db_name}.result lateral view explode(measures) measures as measures_ids where measures_ids.id!='views' and measures_ids.id!='downloads'; ANALYZE TABLE indi_impact_measures COMPUTE STATISTICS; + +create table if not exists ${stats_db_name}.result_apc_affiliations STORED AS PARQUET as +select distinct substr(rel.target,4) id, substr(rel.source,4) organization, o.legalname.value name, +cast(rel.properties[0].value as double) apc_amount, +rel.properties[1].value apc_currency +from ${openaire_db_name}.relation rel +join ${openaire_db_name}.organization o on o.id=rel.source +join ${openaire_db_name}.result r on r.id=rel.target +where rel.subreltype = 'affiliation' and rel.datainfo.deletedbyinference = false and size(rel.properties) > 0; diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql index 36b34cc3c..57c381875 100755 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql @@ -1,88 +1,88 @@ -- Sprint 1 ---- -create table if not exists indi_pub_green_oa stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_green_oa stored as parquet as select distinct p.id, coalesce(green_oa, 0) as green_oa -from publication p +from ${stats_db_name}.publication p left outer join ( select p.id, 1 as green_oa - from publication p - join result_instance ri on ri.id = p.id - join datasource on datasource.id = ri.hostedby + from ${stats_db_name}.publication p + join ${stats_db_name}.result_instance ri on ri.id = p.id + join ${stats_db_name}.datasource on datasource.id = ri.hostedby where datasource.type like '%Repository%' and (ri.accessright = 'Open Access' or ri.accessright = 'Embargo' or ri.accessright = 'Open Source')) tmp on p.id= tmp.id; -ANALYZE TABLE indi_pub_green_oa COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_green_oa COMPUTE STATISTICS; -create table if not exists indi_pub_grey_lit stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_grey_lit stored as parquet as select distinct p.id, coalesce(grey_lit, 0) as grey_lit -from publication p +from ${stats_db_name}.publication p left outer join ( select p.id, 1 as grey_lit - from publication p - join result_classifications rt on rt.id = p.id + from ${stats_db_name}.publication p + join ${stats_db_name}.result_classifications rt on rt.id = p.id where rt.type not in ('Article','Part of book or chapter of book','Book','Doctoral thesis','Master thesis','Data Paper', 'Thesis', 'Bachelor thesis', 'Conference object') and - not exists (select 1 from result_classifications rc where type ='Other literature type' + not exists (select 1 from ${stats_db_name}.result_classifications rc where type ='Other literature type' and rc.id=p.id)) tmp on p.id=tmp.id; -ANALYZE TABLE indi_pub_grey_lit COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_grey_lit COMPUTE STATISTICS; -create table if not exists indi_pub_doi_from_crossref stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_doi_from_crossref stored as parquet as select distinct p.id, coalesce(doi_from_crossref, 0) as doi_from_crossref -from publication p +from ${stats_db_name}.publication p left outer join - (select ri.id, 1 as doi_from_crossref from result_instance ri - join datasource d on d.id = ri.collectedfrom + (select ri.id, 1 as doi_from_crossref from ${stats_db_name}.result_instance ri + join ${stats_db_name}.datasource d on d.id = ri.collectedfrom where pidtype='Digital Object Identifier' and d.name ='Crossref') tmp on tmp.id=p.id; -ANALYZE TABLE indi_pub_doi_from_crossref COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_doi_from_crossref COMPUTE STATISTICS; -- Sprint 2 ---- -create table if not exists indi_result_has_cc_licence stored as parquet as +create table if not exists ${stats_db_name}.indi_result_has_cc_licence stored as parquet as select distinct r.id, (case when lic='' or lic is null then 0 else 1 end) as has_cc_license -from result r - left outer join (select r.id, license.type as lic from result r - join result_licenses as license on license.id = r.id +from ${stats_db_name}.result r +left outer join (select r.id, license.type as lic from ${stats_db_name}.result r + join ${stats_db_name}.result_licenses as license on license.id = r.id where lower(license.type) LIKE '%creativecommons.org%' OR lower(license.type) LIKE '%cc-%') tmp on r.id= tmp.id; -ANALYZE TABLE indi_result_has_cc_licence COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_result_has_cc_licence COMPUTE STATISTICS; -create table if not exists indi_result_has_cc_licence_url stored as parquet as +create table if not exists ${stats_db_name}.indi_result_has_cc_licence_url stored as parquet as select distinct r.id, case when lic_host='' or lic_host is null then 0 else 1 end as has_cc_license_url -from result r +from ${stats_db_name}.result r left outer join (select r.id, lower(parse_url(license.type, "HOST")) as lic_host - from result r - join result_licenses as license on license.id = r.id + from ${stats_db_name}.result r + join ${stats_db_name}.result_licenses as license on license.id = r.id WHERE lower(parse_url(license.type, "HOST")) = "creativecommons.org") tmp on r.id= tmp.id; -ANALYZE TABLE indi_result_has_cc_licence_url COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_result_has_cc_licence_url COMPUTE STATISTICS; -create table if not exists indi_pub_has_abstract stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_has_abstract stored as parquet as select distinct publication.id, cast(coalesce(abstract, true) as int) has_abstract -from publication; +from ${stats_db_name}.publication; -ANALYZE TABLE indi_pub_has_abstract COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_has_abstract COMPUTE STATISTICS; -create table if not exists indi_result_with_orcid stored as parquet as +create table if not exists ${stats_db_name}.indi_result_with_orcid stored as parquet as select distinct r.id, coalesce(has_orcid, 0) as has_orcid -from result r - left outer join (select id, 1 as has_orcid from result_orcid) tmp +from ${stats_db_name}.result r + left outer join (select id, 1 as has_orcid from ${stats_db_name}.result_orcid) tmp on r.id= tmp.id; -ANALYZE TABLE indi_result_with_orcid COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_result_with_orcid COMPUTE STATISTICS; ---- Sprint 3 ---- -create table if not exists indi_funded_result_with_fundref stored as parquet as +create table if not exists ${stats_db_name}.indi_funded_result_with_fundref stored as parquet as select distinct r.result as id, coalesce(fundref, 0) as fundref -from project_results r - left outer join (select distinct result, 1 as fundref from project_results +from ${stats_db_name}.project_results r + left outer join (select distinct result, 1 as fundref from ${stats_db_name}.project_results where provenance='Harvested') tmp on r.result= tmp.result; -ANALYZE TABLE indi_funded_result_with_fundref COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_funded_result_with_fundref COMPUTE STATISTICS; -- create table indi_result_org_collab stored as parquet as -- select o1.organization org1, o2.organization org2, count(distinct o1.id) as collaborations @@ -92,68 +92,68 @@ ANALYZE TABLE indi_funded_result_with_fundref COMPUTE STATISTICS; -- -- compute stats indi_result_org_collab; -- -create TEMPORARY TABLE tmp AS SELECT ro.organization organization, ro.id, o.name from result_organization ro -join organization o on o.id=ro.organization where o.name is not null; +create TEMPORARY TABLE ${stats_db_name}.tmp AS SELECT ro.organization organization, ro.id, o.name from ${stats_db_name}.result_organization ro +join ${stats_db_name}.organization o on o.id=ro.organization where o.name is not null; -create table if not exists indi_result_org_collab stored as parquet as +create table if not exists ${stats_db_name}.indi_result_org_collab stored as parquet as select o1.organization org1, o1.name org1name1, o2.organization org2, o2.name org2name2, count(o1.id) as collaborations -from tmp as o1 -join tmp as o2 where o1.id=o2.id and o1.organization!=o2.organization and o1.name!=o2.name +from ${stats_db_name}.tmp as o1 +join ${stats_db_name}.tmp as o2 where o1.id=o2.id and o1.organization!=o2.organization and o1.name!=o2.name group by o1.organization, o2.organization, o1.name, o2.name; -drop table tmp purge; +drop table ${stats_db_name}.tmp purge; -ANALYZE TABLE indi_result_org_collab COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_result_org_collab COMPUTE STATISTICS; -create TEMPORARY TABLE tmp AS -select distinct ro.organization organization, ro.id, o.name, o.country from result_organization ro -join organization o on o.id=ro.organization where country <> 'UNKNOWN' and o.name is not null; +create TEMPORARY TABLE ${stats_db_name}.tmp AS +select distinct ro.organization organization, ro.id, o.name, o.country from ${stats_db_name}.result_organization ro +join ${stats_db_name}.organization o on o.id=ro.organization where country <> 'UNKNOWN' and o.name is not null; -create table if not exists indi_result_org_country_collab stored as parquet as +create table if not exists ${stats_db_name}.indi_result_org_country_collab stored as parquet as select o1.organization org1,o1.name org1name1, o2.country country2, count(o1.id) as collaborations -from tmp as o1 join tmp as o2 on o1.id=o2.id +from ${stats_db_name}.tmp as o1 join ${stats_db_name}.tmp as o2 on o1.id=o2.id where o1.id=o2.id and o1.country!=o2.country group by o1.organization, o1.id, o1.name, o2.country; -drop table tmp purge; +drop table ${stats_db_name}.tmp purge; -ANALYZE TABLE indi_result_org_country_collab COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_result_org_country_collab COMPUTE STATISTICS; -create TEMPORARY TABLE tmp AS -select o.id organization, o.name, ro.project as project from organization o - join organization_projects ro on o.id=ro.id where o.name is not null; +create TEMPORARY TABLE ${stats_db_name}.tmp AS +select o.id organization, o.name, ro.project as project from ${stats_db_name}.organization o + join ${stats_db_name}.organization_projects ro on o.id=ro.id where o.name is not null; -create table if not exists indi_project_collab_org stored as parquet as +create table if not exists ${stats_db_name}.indi_project_collab_org stored as parquet as select o1.organization org1,o1.name orgname1, o2.organization org2, o2.name orgname2, count(distinct o1.project) as collaborations -from tmp as o1 - join tmp as o2 on o1.project=o2.project +from ${stats_db_name}.tmp as o1 + join ${stats_db_name}.tmp as o2 on o1.project=o2.project where o1.organization<>o2.organization and o1.name<>o2.name group by o1.name,o2.name, o1.organization, o2.organization; -drop table tmp purge; +drop table ${stats_db_name}.tmp purge; -ANALYZE TABLE indi_project_collab_org COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_project_collab_org COMPUTE STATISTICS; -create TEMPORARY TABLE tmp AS -select o.id organization, o.name, o.country , ro.project as project from organization o - join organization_projects ro on o.id=ro.id +create TEMPORARY TABLE ${stats_db_name}.tmp AS +select o.id organization, o.name, o.country , ro.project as project from ${stats_db_name}.organization o + join ${stats_db_name}.organization_projects ro on o.id=ro.id and o.country <> 'UNKNOWN' and o.name is not null; -create table if not exists indi_project_collab_org_country stored as parquet as +create table if not exists ${stats_db_name}.indi_project_collab_org_country stored as parquet as select o1.organization org1,o1.name org1name, o2.country country2, count(distinct o1.project) as collaborations -from tmp as o1 - join tmp as o2 on o1.project=o2.project +from ${stats_db_name}.tmp as o1 + join ${stats_db_name}.tmp as o2 on o1.project=o2.project where o1.organization<>o2.organization and o1.country<>o2.country group by o1.organization, o2.country, o1.name; -drop table tmp purge; +drop table ${stats_db_name}.tmp purge; -ANALYZE TABLE indi_project_collab_org_country COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_project_collab_org_country COMPUTE STATISTICS; -create table if not exists indi_funder_country_collab stored as parquet as - with tmp as (select funder, project, country from organization_projects op - join organization o on o.id=op.id - join project p on p.id=op.project +create table if not exists ${stats_db_name}.indi_funder_country_collab stored as parquet as + with tmp as (select funder, project, country from ${stats_db_name}.organization_projects op + join ${stats_db_name}.organization o on o.id=op.id + join ${stats_db_name}.project p on p.id=op.project where country <> 'UNKNOWN') select f1.funder, f1.country as country1, f2.country as country2, count(distinct f1.project) as collaborations from tmp as f1 @@ -161,104 +161,104 @@ from tmp as f1 where f1.country<>f2.country group by f1.funder, f2.country, f1.country; -ANALYZE TABLE indi_funder_country_collab COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_funder_country_collab COMPUTE STATISTICS; -create TEMPORARY TABLE tmp AS -select distinct country, ro.id as result from organization o - join result_organization ro on o.id=ro.organization +create TEMPORARY TABLE ${stats_db_name}.tmp AS +select distinct country, ro.id as result from ${stats_db_name}.organization o + join ${stats_db_name}.result_organization ro on o.id=ro.organization where country <> 'UNKNOWN' and o.name is not null; -create table if not exists indi_result_country_collab stored as parquet as +create table if not exists ${stats_db_name}.indi_result_country_collab stored as parquet as select o1.country country1, o2.country country2, count(o1.result) as collaborations -from tmp as o1 - join tmp as o2 on o1.result=o2.result +from ${stats_db_name}.tmp as o1 + join ${stats_db_name}.tmp as o2 on o1.result=o2.result where o1.country<>o2.country group by o1.country, o2.country; -drop table tmp purge; +drop table ${stats_db_name}.tmp purge; -ANALYZE TABLE indi_result_country_collab COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_result_country_collab COMPUTE STATISTICS; ---- Sprint 4 ---- -create table if not exists indi_pub_diamond stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_diamond stored as parquet as select distinct pd.id, coalesce(in_diamond_journal, 0) as in_diamond_journal -from publication_datasources pd +from ${stats_db_name}.publication_datasources pd left outer join ( - select pd.id, 1 as in_diamond_journal from publication_datasources pd - join datasource d on d.id=pd.datasource + select pd.id, 1 as in_diamond_journal from ${stats_db_name}.publication_datasources pd + join ${stats_db_name}.datasource d on d.id=pd.datasource join STATS_EXT.plan_s_jn ps where (ps.issn_print=d.issn_printed and ps.issn_online=d.issn_online) and (ps.journal_is_in_doaj=true or ps.journal_is_oa=true) and ps.has_apc=false) tmp on pd.id=tmp.id; -ANALYZE TABLE indi_pub_diamond COMPUTE STATISTICS; +----ANALYZE TABLE ${stats_db_name}.indi_pub_diamond COMPUTE STATISTICS; -create table if not exists indi_pub_in_transformative stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_in_transformative stored as parquet as select distinct pd.id, coalesce(is_transformative, 0) as is_transformative -from publication pd +from ${stats_db_name}.publication pd left outer join ( - select pd.id, 1 as is_transformative from publication_datasources pd - join datasource d on d.id=pd.datasource + select pd.id, 1 as is_transformative from ${stats_db_name}.publication_datasources pd + join ${stats_db_name}.datasource d on d.id=pd.datasource join STATS_EXT.plan_s_jn ps where (ps.issn_print=d.issn_printed and ps.issn_online=d.issn_online) and ps.is_transformative_journal=true) tmp on pd.id=tmp.id; -ANALYZE TABLE indi_pub_in_transformative COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_in_transformative COMPUTE STATISTICS; -create table if not exists indi_pub_closed_other_open stored as parquet as -select distinct ri.id, coalesce(pub_closed_other_open, 0) as pub_closed_other_open from result_instance ri +create table if not exists ${stats_db_name}.indi_pub_closed_other_open stored as parquet as +select distinct ri.id, coalesce(pub_closed_other_open, 0) as pub_closed_other_open from ${stats_db_name}.result_instance ri left outer join - (select ri.id, 1 as pub_closed_other_open from result_instance ri - join publication p on p.id=ri.id - join datasource d on ri.hostedby=d.id + (select ri.id, 1 as pub_closed_other_open from ${stats_db_name}.result_instance ri + join ${stats_db_name}.publication p on p.id=ri.id + join ${stats_db_name}.datasource d on ri.hostedby=d.id where d.type like '%Journal%' and ri.accessright='Closed Access' and (p.bestlicence='Open Access' or p.bestlicence='Open Source')) tmp on tmp.id=ri.id; -ANALYZE TABLE indi_pub_closed_other_open COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_closed_other_open COMPUTE STATISTICS; ---- Sprint 5 ---- -create table if not exists indi_result_no_of_copies stored as parquet as -select id, count(id) as number_of_copies from result_instance group by id; +create table if not exists ${stats_db_name}.indi_result_no_of_copies stored as parquet as +select id, count(id) as number_of_copies from ${stats_db_name}.result_instance group by id; -ANALYZE TABLE indi_result_no_of_copies COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_result_no_of_copies COMPUTE STATISTICS; ---- Sprint 6 ---- -create table if not exists indi_pub_downloads stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_downloads stored as parquet as SELECT result_id, sum(downloads) no_downloads from openaire_prod_usage_stats.usage_stats - join publication on result_id=id + join ${stats_db_name}.publication on result_id=id where downloads>0 GROUP BY result_id order by no_downloads desc; -ANALYZE TABLE indi_pub_downloads COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_downloads COMPUTE STATISTICS; -create table if not exists indi_pub_downloads_datasource stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_downloads_datasource stored as parquet as SELECT result_id, repository_id, sum(downloads) no_downloads from openaire_prod_usage_stats.usage_stats - join publication on result_id=id + join ${stats_db_name}.publication on result_id=id where downloads>0 GROUP BY result_id, repository_id order by result_id; -ANALYZE TABLE indi_pub_downloads_datasource COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_downloads_datasource COMPUTE STATISTICS; -create table if not exists indi_pub_downloads_year stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_downloads_year stored as parquet as SELECT result_id, substring(us.`date`, 1,4) as `year`, sum(downloads) no_downloads from openaire_prod_usage_stats.usage_stats us -join publication on result_id=id where downloads>0 +join ${stats_db_name}.publication on result_id=id where downloads>0 GROUP BY result_id, substring(us.`date`, 1,4); -ANALYZE TABLE indi_pub_downloads_year COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_downloads_year COMPUTE STATISTICS; -create table if not exists indi_pub_downloads_datasource_year stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_downloads_datasource_year stored as parquet as SELECT result_id, substring(us.`date`, 1,4) as `year`, repository_id, sum(downloads) no_downloads from openaire_prod_usage_stats.usage_stats us -join publication on result_id=id +join ${stats_db_name}.publication on result_id=id where downloads>0 GROUP BY result_id, repository_id, substring(us.`date`, 1,4); -ANALYZE TABLE indi_pub_downloads_datasource_year COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_downloads_datasource_year COMPUTE STATISTICS; ---- Sprint 7 ---- -create table if not exists indi_pub_gold_oa stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_gold_oa stored as parquet as WITH gold_oa AS ( SELECT issn_l, journal_is_in_doaj, @@ -284,7 +284,7 @@ create table if not exists indi_pub_gold_oa stored as parquet as id, issn_printed as issn FROM - datasource + ${stats_db_name}.datasource WHERE issn_printed IS NOT NULL UNION ALL @@ -292,7 +292,7 @@ create table if not exists indi_pub_gold_oa stored as parquet as id, issn_online as issn FROM - datasource + ${stats_db_name}.datasource WHERE issn_online IS NOT NULL or id like '%doajarticles%') as issn WHERE @@ -300,16 +300,16 @@ create table if not exists indi_pub_gold_oa stored as parquet as SELECT DISTINCT pd.id, coalesce(is_gold, 0) as is_gold FROM - publication_datasources pd + ${stats_db_name}.publication_datasources pd left outer join( - select pd.id, 1 as is_gold FROM publication_datasources pd + select pd.id, 1 as is_gold FROM ${stats_db_name}.publication_datasources pd JOIN issn on issn.id=pd.datasource JOIN gold_oa on issn.issn = gold_oa.issn) tmp on pd.id=tmp.id; -ANALYZE TABLE indi_pub_gold_oa COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_gold_oa COMPUTE STATISTICS; -create table if not exists indi_pub_hybrid_oa_with_cc stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_hybrid_oa_with_cc stored as parquet as WITH hybrid_oa AS ( SELECT issn_l, journal_is_in_doaj, journal_is_oa, issn_print as issn FROM STATS_EXT.plan_s_jn @@ -322,27 +322,27 @@ create table if not exists indi_pub_hybrid_oa_with_cc stored as parquet as SELECT * FROM ( SELECT id, issn_printed as issn - FROM datasource + FROM ${stats_db_name}.datasource WHERE issn_printed IS NOT NULL UNION ALL SELECT id,issn_online as issn - FROM datasource + FROM ${stats_db_name}.datasource WHERE issn_online IS NOT NULL ) as issn WHERE LENGTH(issn) > 7) SELECT DISTINCT pd.id, coalesce(is_hybrid_oa, 0) as is_hybrid_oa -FROM publication_datasources pd +FROM ${stats_db_name}.publication_datasources pd LEFT OUTER JOIN ( - SELECT pd.id, 1 as is_hybrid_oa from publication_datasources pd - JOIN datasource d on d.id=pd.datasource + SELECT pd.id, 1 as is_hybrid_oa from ${stats_db_name}.publication_datasources pd + JOIN ${stats_db_name}.datasource d on d.id=pd.datasource JOIN issn on issn.id=pd.datasource JOIN hybrid_oa ON issn.issn = hybrid_oa.issn - JOIN indi_result_has_cc_licence cc on pd.id=cc.id - JOIN indi_pub_gold_oa ga on pd.id=ga.id + JOIN ${stats_db_name}.indi_result_has_cc_licence cc on pd.id=cc.id + JOIN ${stats_db_name}.indi_pub_gold_oa ga on pd.id=ga.id where cc.has_cc_license=1 and ga.is_gold=0) tmp on pd.id=tmp.id; -ANALYZE TABLE indi_pub_hybrid_oa_with_cc COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_hybrid_oa_with_cc COMPUTE STATISTICS; -create table if not exists indi_pub_hybrid stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_hybrid stored as parquet as WITH gold_oa AS ( SELECT issn_l, journal_is_in_doaj, @@ -370,7 +370,7 @@ create table if not exists indi_pub_hybrid stored as parquet as id, issn_printed as issn FROM - datasource + ${stats_db_name}.datasource WHERE issn_printed IS NOT NULL UNION ALL @@ -378,424 +378,398 @@ create table if not exists indi_pub_hybrid stored as parquet as id, issn_online as issn FROM - datasource + ${stats_db_name}.datasource WHERE issn_online IS NOT NULL or id like '%doajarticles%') as issn WHERE LENGTH(issn) > 7) select distinct pd.id, coalesce(is_hybrid, 0) as is_hybrid -from publication_datasources pd +from ${stats_db_name}.publication_datasources pd left outer join ( - select pd.id, 1 as is_hybrid from publication_datasources pd - join datasource d on d.id=pd.datasource + select pd.id, 1 as is_hybrid from ${stats_db_name}.publication_datasources pd + join ${stats_db_name}.datasource d on d.id=pd.datasource join issn on issn.id=pd.datasource join gold_oa on issn.issn=gold_oa.issn where (gold_oa.journal_is_in_doaj=false or gold_oa.journal_is_oa=false))tmp on pd.id=tmp.id; -ANALYZE TABLE indi_pub_hybrid COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_hybrid COMPUTE STATISTICS; -create table if not exists indi_org_fairness stored as parquet as +create table if not exists ${stats_db_name}.indi_org_fairness stored as parquet as --return results with PIDs, and rich metadata group by organization with result_fair as - (select ro.organization organization, count(distinct ro.id) no_result_fair from result_organization ro - join result r on r.id=ro.id + (select ro.organization organization, count(distinct ro.id) no_result_fair from ${stats_db_name}.result_organization ro + join ${stats_db_name}.result r on r.id=ro.id --join result_pids rp on r.id=rp.id where (title is not null) and (publisher is not null) and (abstract=true) and (year is not null) and (authors>0) and cast(year as int)>2003 group by ro.organization), --return all results group by organization - allresults as (select organization, count(distinct ro.id) no_allresults from result_organization ro - join result r on r.id=ro.id + allresults as (select ro.organization, count(distinct ro.id) no_allresults from ${stats_db_name}.result_organization ro + join ${stats_db_name}.result r on r.id=ro.id where cast(year as int)>2003 - group by organization) + group by ro.organization) --return results_fair/all_results select allresults.organization, result_fair.no_result_fair/allresults.no_allresults org_fairness from allresults join result_fair on result_fair.organization=allresults.organization; -ANALYZE TABLE indi_org_fairness COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_org_fairness COMPUTE STATISTICS; -create table if not exists indi_org_fairness_pub_pr stored as parquet as - with result_fair as - (select ro.organization organization, count(distinct ro.id) no_result_fair - from result_organization ro - join publication p on p.id=ro.id - join indi_pub_doi_from_crossref dc on dc.id=p.id - join indi_pub_grey_lit gl on gl.id=p.id +CREATE TEMPORARY table ${stats_db_name}.result_fair as +select ro.organization organization, count(distinct ro.id) no_result_fair + from ${stats_db_name}.result_organization ro + join ${stats_db_name}.publication p on p.id=ro.id + join ${stats_db_name}.indi_pub_doi_from_crossref dc on dc.id=p.id + join ${stats_db_name}.indi_pub_grey_lit gl on gl.id=p.id where (title is not null) and (publisher is not null) and (abstract=true) and (year is not null) and (authors>0) and cast(year as int)>2003 and dc.doi_from_crossref=1 and gl.grey_lit=0 - group by ro.organization), - allresults as (select organization, count(distinct ro.id) no_allresults from result_organization ro - join publication p on p.id=ro.id + group by ro.organization; + +CREATE TEMPORARY TABLE ${stats_db_name}.allresults as +select ro.organization, count(distinct ro.id) no_allresults from ${stats_db_name}.result_organization ro + join ${stats_db_name}.publication p on p.id=ro.id where cast(year as int)>2003 - group by organization) ---return results_fair/all_results -select allresults.organization, result_fair.no_result_fair/allresults.no_allresults org_fairness -from allresults - join result_fair on result_fair.organization=allresults.organization; + group by ro.organization; -ANALYZE TABLE indi_org_fairness_pub_pr COMPUTE STATISTICS; +create table if not exists ${stats_db_name}.indi_org_fairness_pub_pr stored as parquet as +select ar.organization, rf.no_result_fair/ar.no_allresults org_fairness +from ${stats_db_name}.allresults ar + join ${stats_db_name}.result_fair rf on rf.organization=ar.organization; -CREATE TEMPORARY table result_fair as - select year, ro.organization organization, count(distinct ro.id) no_result_fair from result_organization ro - join result p on p.id=ro.id +DROP table ${stats_db_name}.result_fair purge; +DROP table ${stats_db_name}.allresults purge; + +--ANALYZE TABLE ${stats_db_name}.indi_org_fairness_pub_pr COMPUTE STATISTICS; + +CREATE TEMPORARY table ${stats_db_name}.result_fair as + select year, ro.organization organization, count(distinct ro.id) no_result_fair from ${stats_db_name}.result_organization ro + join ${stats_db_name}.result p on p.id=ro.id where (title is not null) and (publisher is not null) and (abstract=true) and (year is not null) and (authors>0) and cast(year as int)>2003 group by ro.organization, year; -CREATE TEMPORARY TABLE allresults as select year, organization, count(distinct ro.id) no_allresults from result_organization ro - join result p on p.id=ro.id +CREATE TEMPORARY TABLE ${stats_db_name}.allresults as select year, ro.organization, count(distinct ro.id) no_allresults from ${stats_db_name}.result_organization ro + join ${stats_db_name}.result p on p.id=ro.id where cast(year as int)>2003 - group by organization, year; + group by ro.organization, year; -create table if not exists indi_org_fairness_pub_year stored as parquet as +create table if not exists ${stats_db_name}.indi_org_fairness_pub_year stored as parquet as select allresults.year, allresults.organization, result_fair.no_result_fair/allresults.no_allresults org_fairness -from allresults - join result_fair on result_fair.organization=allresults.organization and result_fair.year=allresults.year; +from ${stats_db_name}.allresults + join ${stats_db_name}.result_fair on result_fair.organization=allresults.organization and result_fair.year=allresults.year; -DROP table result_fair purge; -DROP table allresults purge; +DROP table ${stats_db_name}.result_fair purge; +DROP table ${stats_db_name}.allresults purge; -ANALYZE TABLE indi_org_fairness_pub_year COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_org_fairness_pub_year COMPUTE STATISTICS; -CREATE TEMPORARY TABLE result_fair as +CREATE TEMPORARY TABLE ${stats_db_name}.result_fair as select ro.organization organization, count(distinct ro.id) no_result_fair - from result_organization ro - join result p on p.id=ro.id + from ${stats_db_name}.result_organization ro + join ${stats_db_name}.result p on p.id=ro.id where (title is not null) and (publisher is not null) and (abstract=true) and (year is not null) and (authors>0) and cast(year as int)>2003 group by ro.organization; -CREATE TEMPORARY TABLE allresults as - select organization, count(distinct ro.id) no_allresults from result_organization ro - join result p on p.id=ro.id +CREATE TEMPORARY TABLE ${stats_db_name}.allresults as + select ro.organization, count(distinct ro.id) no_allresults from ${stats_db_name}.result_organization ro + join ${stats_db_name}.result p on p.id=ro.id where cast(year as int)>2003 - group by organization; + group by ro.organization; -create table if not exists indi_org_fairness_pub as -select allresults.organization, result_fair.no_result_fair/allresults.no_allresults org_fairness -from allresults join result_fair on result_fair.organization=allresults.organization; +create table if not exists ${stats_db_name}.indi_org_fairness_pub as +select ar.organization, rf.no_result_fair/ar.no_allresults org_fairness +from ${stats_db_name}.allresults ar join ${stats_db_name}.result_fair rf +on rf.organization=ar.organization; -DROP table result_fair purge; -DROP table allresults purge; +DROP table ${stats_db_name}.result_fair purge; +DROP table ${stats_db_name}.allresults purge; -ANALYZE TABLE indi_org_fairness_pub COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_org_fairness_pub COMPUTE STATISTICS; -CREATE TEMPORARY TABLE result_fair as - select year, ro.organization organization, count(distinct ro.id) no_result_fair from result_organization ro - join result r on r.id=ro.id - join result_pids rp on r.id=rp.id +CREATE TEMPORARY TABLE ${stats_db_name}.result_fair as + select year, ro.organization organization, count(distinct ro.id) no_result_fair from ${stats_db_name}.result_organization ro + join ${stats_db_name}.result r on r.id=ro.id + join ${stats_db_name}.result_pids rp on r.id=rp.id where (title is not null) and (publisher is not null) and (abstract=true) and (year is not null) and (authors>0) and cast(year as int)>2003 group by ro.organization, year; -CREATE TEMPORARY TABLE allresults as - select year, organization, count(distinct ro.id) no_allresults from result_organization ro - join result r on r.id=ro.id +CREATE TEMPORARY TABLE ${stats_db_name}.allresults as + select year, ro.organization, count(distinct ro.id) no_allresults from ${stats_db_name}.result_organization ro + join ${stats_db_name}.result r on r.id=ro.id where cast(year as int)>2003 - group by organization, year; + group by ro.organization, year; -create table if not exists indi_org_fairness_year stored as parquet as +create table if not exists ${stats_db_name}.indi_org_fairness_year stored as parquet as select allresults.year, allresults.organization, result_fair.no_result_fair/allresults.no_allresults org_fairness - from allresults - join result_fair on result_fair.organization=allresults.organization and result_fair.year=allresults.year; + from ${stats_db_name}.allresults + join ${stats_db_name}.result_fair on result_fair.organization=allresults.organization and result_fair.year=allresults.year; -DROP table result_fair purge; -DROP table allresults purge; +DROP table ${stats_db_name}.result_fair purge; +DROP table ${stats_db_name}.allresults purge; -ANALYZE TABLE indi_org_fairness_year COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_org_fairness_year COMPUTE STATISTICS; -CREATE TEMPORARY TABLE result_with_pid as - select year, ro.organization organization, count(distinct rp.id) no_result_with_pid from result_organization ro - join result_pids rp on rp.id=ro.id - join result r on r.id=rp.id +CREATE TEMPORARY TABLE ${stats_db_name}.result_with_pid as + select year, ro.organization, count(distinct rp.id) no_result_with_pid from ${stats_db_name}.result_organization ro + join ${stats_db_name}.result_pids rp on rp.id=ro.id + join ${stats_db_name}.result r on r.id=rp.id where cast(year as int) >2003 group by ro.organization, year; -CREATE TEMPORARY TABLE allresults as - select year, organization, count(distinct ro.id) no_allresults from result_organization ro - join result r on r.id=ro.id +CREATE TEMPORARY TABLE ${stats_db_name}.allresults as + select year, ro.organization, count(distinct ro.id) no_allresults from ${stats_db_name}.result_organization ro + join ${stats_db_name}.result r on r.id=ro.id where cast(year as int) >2003 - group by organization, year; + group by ro.organization, year; -create table if not exists indi_org_findable_year stored as parquet as +create table if not exists ${stats_db_name}.indi_org_findable_year stored as parquet as select allresults.year, allresults.organization, result_with_pid.no_result_with_pid/allresults.no_allresults org_findable -from allresults - join result_with_pid on result_with_pid.organization=allresults.organization and result_with_pid.year=allresults.year; +from ${stats_db_name}.allresults + join ${stats_db_name}.result_with_pid on result_with_pid.organization=allresults.organization and result_with_pid.year=allresults.year; -DROP table result_with_pid purge; -DROP table allresults purge; +DROP table ${stats_db_name}.result_with_pid purge; +DROP table ${stats_db_name}.allresults purge; -ANALYZE TABLE indi_org_findable_year COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_org_findable_year COMPUTE STATISTICS; -CREATE TEMPORARY TABLE result_with_pid as -select ro.organization organization, count(distinct rp.id) no_result_with_pid from result_organization ro - join result_pids rp on rp.id=ro.id - join result r on r.id=rp.id +CREATE TEMPORARY TABLE ${stats_db_name}.result_with_pid as +select ro.organization, count(distinct rp.id) no_result_with_pid from ${stats_db_name}.result_organization ro + join ${stats_db_name}.result_pids rp on rp.id=ro.id + join ${stats_db_name}.result r on r.id=rp.id where cast(year as int) >2003 group by ro.organization; -CREATE TEMPORARY TABLE allresults as -select organization, count(distinct ro.id) no_allresults from result_organization ro - join result r on r.id=ro.id +CREATE TEMPORARY TABLE ${stats_db_name}.allresults as +select ro.organization, count(distinct ro.id) no_allresults from ${stats_db_name}.result_organization ro + join ${stats_db_name}.result r on r.id=ro.id where cast(year as int) >2003 - group by organization; + group by ro.organization; -create table if not exists indi_org_findable stored as parquet as +create table if not exists ${stats_db_name}.indi_org_findable stored as parquet as select allresults.organization, result_with_pid.no_result_with_pid/allresults.no_allresults org_findable -from allresults - join result_with_pid on result_with_pid.organization=allresults.organization; +from ${stats_db_name}.allresults + join ${stats_db_name}.result_with_pid on result_with_pid.organization=allresults.organization; -DROP table result_with_pid purge; -DROP table allresults purge; +DROP table ${stats_db_name}.result_with_pid purge; +DROP table ${stats_db_name}.allresults purge; -ANALYZE TABLE indi_org_findable COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_org_findable COMPUTE STATISTICS; -CREATE TEMPORARY TABLE pubs_oa as -SELECT ro.organization, count(distinct r.id) no_oapubs FROM publication r - join result_organization ro on ro.id=r.id - join result_instance ri on ri.id=r.id +CREATE TEMPORARY TABLE ${stats_db_name}.pubs_oa as +SELECT ro.organization, count(distinct r.id) no_oapubs FROM ${stats_db_name}.publication r + join ${stats_db_name}.result_organization ro on ro.id=r.id + join ${stats_db_name}.result_instance ri on ri.id=r.id where (ri.accessright = 'Open Access' or ri.accessright = 'Embargo' or ri.accessright = 'Open Source') and cast(r.year as int)>2003 group by ro.organization; -CREATE TEMPORARY TABLE datasets_oa as -SELECT ro.organization, count(distinct r.id) no_oadatasets FROM dataset r - join result_organization ro on ro.id=r.id - join result_instance ri on ri.id=r.id +CREATE TEMPORARY TABLE ${stats_db_name}.datasets_oa as +SELECT ro.organization, count(distinct r.id) no_oadatasets FROM ${stats_db_name}.dataset r + join ${stats_db_name}.result_organization ro on ro.id=r.id + join ${stats_db_name}.result_instance ri on ri.id=r.id where (ri.accessright = 'Open Access' or ri.accessright = 'Embargo' or ri.accessright = 'Open Source') and cast(r.year as int)>2003 group by ro.organization; -CREATE TEMPORARY TABLE software_oa as -SELECT ro.organization, count(distinct r.id) no_oasoftware FROM software r - join result_organization ro on ro.id=r.id - join result_instance ri on ri.id=r.id +CREATE TEMPORARY TABLE ${stats_db_name}.software_oa as +SELECT ro.organization, count(distinct r.id) no_oasoftware FROM ${stats_db_name}.software r + join ${stats_db_name}.result_organization ro on ro.id=r.id + join ${stats_db_name}.result_instance ri on ri.id=r.id where (ri.accessright = 'Open Access' or ri.accessright = 'Embargo' or ri.accessright = 'Open Source') and cast(r.year as int)>2003 group by ro.organization; -CREATE TEMPORARY TABLE allpubs as -SELECT ro.organization organization, count(ro.id) no_allpubs FROM result_organization ro - join publication ps on ps.id=ro.id +CREATE TEMPORARY TABLE ${stats_db_name}.allpubs as +SELECT ro.organization, count(ro.id) no_allpubs FROM ${stats_db_name}.result_organization ro + join ${stats_db_name}.publication ps on ps.id=ro.id where cast(ps.year as int)>2003 group by ro.organization; -CREATE TEMPORARY TABLE alldatasets as -SELECT ro.organization organization, count(ro.id) no_alldatasets FROM result_organization ro - join dataset ps on ps.id=ro.id +CREATE TEMPORARY TABLE ${stats_db_name}.alldatasets as +SELECT ro.organization, count(ro.id) no_alldatasets FROM ${stats_db_name}.result_organization ro + join ${stats_db_name}.dataset ps on ps.id=ro.id where cast(ps.year as int)>2003 group by ro.organization; -CREATE TEMPORARY TABLE allsoftware as -SELECT ro.organization organization, count(ro.id) no_allsoftware FROM result_organization ro - join software ps on ps.id=ro.id +CREATE TEMPORARY TABLE ${stats_db_name}.allsoftware as +SELECT ro.organization, count(ro.id) no_allsoftware FROM ${stats_db_name}.result_organization ro + join ${stats_db_name}.software ps on ps.id=ro.id where cast(ps.year as int)>2003 group by ro.organization; -CREATE TEMPORARY TABLE allpubsshare as -select pubs_oa.organization, pubs_oa.no_oapubs/allpubs.no_allpubs p from allpubs - join pubs_oa on allpubs.organization=pubs_oa.organization; +CREATE TEMPORARY TABLE ${stats_db_name}.allpubsshare as +select pubs_oa.organization, pubs_oa.no_oapubs/allpubs.no_allpubs p from ${stats_db_name}.allpubs + join ${stats_db_name}.pubs_oa on allpubs.organization=pubs_oa.organization; -CREATE TEMPORARY TABLE alldatasetssshare as +CREATE TEMPORARY TABLE ${stats_db_name}.alldatasetssshare as select datasets_oa.organization, datasets_oa.no_oadatasets/alldatasets.no_alldatasets d - from alldatasets - join datasets_oa on alldatasets.organization=datasets_oa.organization; + from ${stats_db_name}.alldatasets + join ${stats_db_name}.datasets_oa on alldatasets.organization=datasets_oa.organization; -CREATE TEMPORARY TABLE allsoftwaresshare as +CREATE TEMPORARY TABLE ${stats_db_name}.allsoftwaresshare as select software_oa.organization, software_oa.no_oasoftware/allsoftware.no_allsoftware s - from allsoftware - join software_oa on allsoftware.organization=software_oa.organization; + from ${stats_db_name}.allsoftware + join ${stats_db_name}.software_oa on allsoftware.organization=software_oa.organization; -create table if not exists indi_org_openess stored as parquet as +create table if not exists ${stats_db_name}.indi_org_openess stored as parquet as select allpubsshare.organization, (p+if(isnull(s),0,s)+if(isnull(d),0,d))/(1+(case when s is null then 0 else 1 end) +(case when d is null then 0 else 1 end)) - org_openess FROM allpubsshare + org_openess FROM ${stats_db_name}.allpubsshare left outer join (select organization,d from - alldatasetssshare) tmp1 + ${stats_db_name}.alldatasetssshare) tmp1 on tmp1.organization=allpubsshare.organization left outer join (select organization,s from - allsoftwaresshare) tmp2 + ${stats_db_name}.allsoftwaresshare) tmp2 on tmp2.organization=allpubsshare.organization; -DROP TABLE pubs_oa purge; -DROP TABLE datasets_oa purge; -DROP TABLE software_oa purge; -DROP TABLE allpubs purge; -DROP TABLE alldatasets purge; -DROP TABLE allsoftware purge; -DROP TABLE allpubsshare purge; -DROP TABLE alldatasetssshare purge; -DROP TABLE allsoftwaresshare purge; +DROP TABLE ${stats_db_name}.pubs_oa purge; +DROP TABLE ${stats_db_name}.datasets_oa purge; +DROP TABLE ${stats_db_name}.software_oa purge; +DROP TABLE ${stats_db_name}.allpubs purge; +DROP TABLE ${stats_db_name}.alldatasets purge; +DROP TABLE ${stats_db_name}.allsoftware purge; +DROP TABLE ${stats_db_name}.allpubsshare purge; +DROP TABLE ${stats_db_name}.alldatasetssshare purge; +DROP TABLE ${stats_db_name}.allsoftwaresshare purge; -ANALYZE TABLE indi_org_openess COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_org_openess COMPUTE STATISTICS; -CREATE TEMPORARY TABLE pubs_oa AS -SELECT r.year, ro.organization, count(distinct r.id) no_oapubs FROM publication r - join result_organization ro on ro.id=r.id - join result_instance ri on ri.id=r.id +CREATE TEMPORARY TABLE ${stats_db_name}.pubs_oa AS +SELECT r.year, ro.organization, count(distinct r.id) no_oapubs FROM ${stats_db_name}.publication r + join ${stats_db_name}.result_organization ro on ro.id=r.id + join ${stats_db_name}.result_instance ri on ri.id=r.id where (ri.accessright = 'Open Access' or ri.accessright = 'Embargo' or ri.accessright = 'Open Source') and cast(r.year as int)>2003 group by ro.organization,r.year; -CREATE TEMPORARY TABLE datasets_oa AS -SELECT r.year,ro.organization, count(distinct r.id) no_oadatasets FROM dataset r - join result_organization ro on ro.id=r.id - join result_instance ri on ri.id=r.id +CREATE TEMPORARY TABLE ${stats_db_name}.datasets_oa AS +SELECT r.year,ro.organization, count(distinct r.id) no_oadatasets FROM ${stats_db_name}.dataset r + join ${stats_db_name}.result_organization ro on ro.id=r.id + join ${stats_db_name}.result_instance ri on ri.id=r.id where (ri.accessright = 'Open Access' or ri.accessright = 'Embargo' or ri.accessright = 'Open Source') and cast(r.year as int)>2003 group by ro.organization, r.year; -CREATE TEMPORARY TABLE software_oa AS -SELECT r.year,ro.organization, count(distinct r.id) no_oasoftware FROM software r - join result_organization ro on ro.id=r.id - join result_instance ri on ri.id=r.id +CREATE TEMPORARY TABLE ${stats_db_name}.software_oa AS +SELECT r.year,ro.organization, count(distinct r.id) no_oasoftware FROM ${stats_db_name}.software r + join ${stats_db_name}.result_organization ro on ro.id=r.id + join ${stats_db_name}.result_instance ri on ri.id=r.id where (ri.accessright = 'Open Access' or ri.accessright = 'Embargo' or ri.accessright = 'Open Source') and cast(r.year as int)>2003 group by ro.organization, r.year; -CREATE TEMPORARY TABLE allpubs as -SELECT p.year,ro.organization organization, count(ro.id) no_allpubs FROM result_organization ro - join publication p on p.id=ro.id where cast(p.year as int)>2003 +CREATE TEMPORARY TABLE ${stats_db_name}.allpubs as +SELECT p.year,ro.organization organization, count(ro.id) no_allpubs FROM ${stats_db_name}.result_organization ro + join ${stats_db_name}.publication p on p.id=ro.id where cast(p.year as int)>2003 group by ro.organization, p.year; -CREATE TEMPORARY TABLE alldatasets as -SELECT d.year, ro.organization organization, count(ro.id) no_alldatasets FROM result_organization ro - join dataset d on d.id=ro.id where cast(d.year as int)>2003 +CREATE TEMPORARY TABLE ${stats_db_name}.alldatasets as +SELECT d.year, ro.organization organization, count(ro.id) no_alldatasets FROM ${stats_db_name}.result_organization ro + join ${stats_db_name}.dataset d on d.id=ro.id where cast(d.year as int)>2003 group by ro.organization, d.year; -CREATE TEMPORARY TABLE allsoftware as -SELECT s.year,ro.organization organization, count(ro.id) no_allsoftware FROM result_organization ro - join software s on s.id=ro.id where cast(s.year as int)>2003 +CREATE TEMPORARY TABLE ${stats_db_name}.allsoftware as +SELECT s.year,ro.organization organization, count(ro.id) no_allsoftware FROM ${stats_db_name}.result_organization ro + join ${stats_db_name}.software s on s.id=ro.id where cast(s.year as int)>2003 group by ro.organization, s.year; -CREATE TEMPORARY TABLE allpubsshare as -select allpubs.year, pubs_oa.organization, pubs_oa.no_oapubs/allpubs.no_allpubs p from allpubs - join pubs_oa on allpubs.organization=pubs_oa.organization where cast(allpubs.year as INT)=cast(pubs_oa.year as int); +CREATE TEMPORARY TABLE ${stats_db_name}.allpubsshare as +select allpubs.year, pubs_oa.organization, pubs_oa.no_oapubs/allpubs.no_allpubs p from ${stats_db_name}.allpubs + join ${stats_db_name}.pubs_oa on allpubs.organization=pubs_oa.organization where cast(allpubs.year as INT)=cast(pubs_oa.year as int); -CREATE TEMPORARY TABLE alldatasetssshare as +CREATE TEMPORARY TABLE ${stats_db_name}.alldatasetssshare as select alldatasets.year, datasets_oa.organization, datasets_oa.no_oadatasets/alldatasets.no_alldatasets d - from alldatasets - join datasets_oa on alldatasets.organization=datasets_oa.organization where cast(alldatasets.year as INT)=cast(datasets_oa.year as int); + from ${stats_db_name}.alldatasets + join ${stats_db_name}.datasets_oa on alldatasets.organization=datasets_oa.organization where cast(alldatasets.year as INT)=cast(datasets_oa.year as int); -CREATE TEMPORARY TABLE allsoftwaresshare as +CREATE TEMPORARY TABLE ${stats_db_name}.allsoftwaresshare as select allsoftware.year, software_oa.organization, software_oa.no_oasoftware/allsoftware.no_allsoftware s - from allsoftware - join software_oa on allsoftware.organization=software_oa.organization where cast(allsoftware.year as INT)=cast(software_oa.year as int); + from ${stats_db_name}.allsoftware + join ${stats_db_name}.software_oa on allsoftware.organization=software_oa.organization where cast(allsoftware.year as INT)=cast(software_oa.year as int); -create table if not exists indi_org_openess_year stored as parquet as +create table if not exists ${stats_db_name}.indi_org_openess_year stored as parquet as select allpubsshare.year, allpubsshare.organization, (p+if(isnull(s),0,s)+if(isnull(d),0,d))/(1+(case when s is null then 0 else 1 end) +(case when d is null then 0 else 1 end)) - org_openess FROM allpubsshare + org_openess FROM ${stats_db_name}.allpubsshare left outer join (select year, organization,d from - alldatasetssshare) tmp1 + ${stats_db_name}.alldatasetssshare) tmp1 on tmp1.organization=allpubsshare.organization and tmp1.year=allpubsshare.year left outer join (select year, organization,s from - allsoftwaresshare) tmp2 + ${stats_db_name}.allsoftwaresshare) tmp2 on tmp2.organization=allpubsshare.organization and tmp2.year=allpubsshare.year; -DROP TABLE pubs_oa purge; -DROP TABLE datasets_oa purge; -DROP TABLE software_oa purge; -DROP TABLE allpubs purge; -DROP TABLE alldatasets purge; -DROP TABLE allsoftware purge; -DROP TABLE allpubsshare purge; -DROP TABLE alldatasetssshare purge; -DROP TABLE allsoftwaresshare purge; +DROP TABLE ${stats_db_name}.pubs_oa purge; +DROP TABLE ${stats_db_name}.datasets_oa purge; +DROP TABLE ${stats_db_name}.software_oa purge; +DROP TABLE ${stats_db_name}.allpubs purge; +DROP TABLE ${stats_db_name}.alldatasets purge; +DROP TABLE ${stats_db_name}.allsoftware purge; +DROP TABLE ${stats_db_name}.allpubsshare purge; +DROP TABLE ${stats_db_name}.alldatasetssshare purge; +DROP TABLE ${stats_db_name}.allsoftwaresshare purge; -ANALYZE TABLE indi_org_openess_year COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_org_openess_year COMPUTE STATISTICS; -create table if not exists indi_pub_has_preprint stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_has_preprint stored as parquet as select distinct p.id, coalesce(has_preprint, 0) as has_preprint -from publication_classifications p +from ${stats_db_name}.publication_classifications p left outer join ( select p.id, 1 as has_preprint - from publication_classifications p + from ${stats_db_name}.publication_classifications p where p.type='Preprint') tmp on p.id= tmp.id; -ANALYZE TABLE indi_pub_has_preprint COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_has_preprint COMPUTE STATISTICS; -create table if not exists indi_pub_in_subscribed stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_in_subscribed stored as parquet as select distinct p.id, coalesce(is_subscription, 0) as is_subscription -from publication p +from ${stats_db_name}.publication p left outer join( - select p.id, 1 as is_subscription from publication p - join indi_pub_gold_oa g on p.id=g.id - join indi_pub_hybrid h on p.id=h.id - join indi_pub_in_transformative t on p.id=t.id + select p.id, 1 as is_subscription from ${stats_db_name}.publication p + join ${stats_db_name}.indi_pub_gold_oa g on p.id=g.id + join ${stats_db_name}.indi_pub_hybrid h on p.id=h.id + join ${stats_db_name}.indi_pub_in_transformative t on p.id=t.id where g.is_gold=0 and h.is_hybrid=0 and t.is_transformative=0) tmp on p.id=tmp.id; -ANALYZE TABLE indi_pub_in_subscribed COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_in_subscribed COMPUTE STATISTICS; -create table if not exists indi_result_with_pid as +create table if not exists ${stats_db_name}.indi_result_with_pid as select distinct p.id, coalesce(result_with_pid, 0) as result_with_pid -from result p +from ${stats_db_name}.result p left outer join ( select p.id, 1 as result_with_pid - from result_pids p) tmp + from ${stats_db_name}.result_pids p) tmp on p.id= tmp.id; -ANALYZE TABLE indi_result_with_pid COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_result_with_pid COMPUTE STATISTICS; -CREATE TEMPORARY TABLE pub_fos_totals as -select rf.id, count(distinct lvl3) totals from result_fos rf +CREATE TEMPORARY TABLE ${stats_db_name}.pub_fos_totals as +select rf.id, count(distinct lvl3) totals from ${stats_db_name}.result_fos rf group by rf.id; -create table if not exists indi_pub_interdisciplinarity as +create table if not exists ${stats_db_name}.indi_pub_interdisciplinarity as select distinct p.id as id, coalesce(is_interdisciplinary, 0) as is_interdisciplinary -from pub_fos_totals p +from ${stats_db_name}.pub_fos_totals p left outer join ( -select pub_fos_totals.id, 1 as is_interdisciplinary from pub_fos_totals +select pub_fos_totals.id, 1 as is_interdisciplinary from ${stats_db_name}.pub_fos_totals where totals>1) tmp on p.id=tmp.id; -drop table pub_fos_totals purge; +drop table ${stats_db_name}.pub_fos_totals purge; -ANALYZE TABLE indi_pub_interdisciplinarity COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_interdisciplinarity COMPUTE STATISTICS; -create table if not exists indi_pub_bronze_oa stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_bronze_oa stored as parquet as select distinct p.id, coalesce(is_bronze_oa,0) as is_bronze_oa -from publication p +from ${stats_db_name}.publication p left outer join -(select p.id, 1 as is_bronze_oa from publication p -join indi_result_has_cc_licence cc on cc.id=p.id -join indi_pub_gold_oa ga on ga.id=p.id +(select p.id, 1 as is_bronze_oa from ${stats_db_name}.publication p +join ${stats_db_name}.indi_result_has_cc_licence cc on cc.id=p.id +join ${stats_db_name}.indi_pub_gold_oa ga on ga.id=p.id where cc.has_cc_license=0 and ga.is_gold=0) tmp on tmp.id=p.id; --- create table if not exists indi_pub_bronze_oa stored as parquet as --- WITH hybrid_oa AS ( --- SELECT issn_l, journal_is_in_doaj, journal_is_oa, issn_print as issn --- FROM STATS_EXT.plan_s_jn --- WHERE issn_print != "" --- UNION ALL --- SELECT issn_l, journal_is_in_doaj, journal_is_oa, issn_online as issn --- FROM STATS_EXT.plan_s_jn --- WHERE issn_online != "" and (journal_is_in_doaj = FALSE OR journal_is_oa = FALSE)), --- issn AS ( --- SELECT * --- FROM ( --- SELECT id, issn_printed as issn --- FROM datasource --- WHERE issn_printed IS NOT NULL --- UNION ALL --- SELECT id,issn_online as issn --- FROM datasource --- WHERE issn_online IS NOT NULL ) as issn --- WHERE LENGTH(issn) > 7) ---SELECT DISTINCT pd.id, coalesce(is_bronze_oa, 0) as is_bronze_oa ---FROM publication_datasources pd --- LEFT OUTER JOIN ( --- SELECT pd.id, 1 as is_bronze_oa from publication_datasources pd --- JOIN datasource d on d.id=pd.datasource --- JOIN issn on issn.id=pd.datasource --- JOIN hybrid_oa ON issn.issn = hybrid_oa.issn --- JOIN indi_result_has_cc_licence cc on pd.id=cc.id --- JOIN indi_pub_gold_oa ga on pd.id=ga.id --- JOIN indi_pub_hybrid_oa_with_cc hy on hy.id=pd.id --- where cc.has_cc_license=0 and ga.is_gold=0 and hy.is_hybrid_oa=0) tmp on pd.id=tmp.id; - -ANALYZE TABLE indi_pub_bronze_oa COMPUTE STATISTICS; \ No newline at end of file +--ANALYZE TABLE ${stats_db_name}.indi_pub_bronze_oa COMPUTE STATISTICS; \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql index 9744d5aae..3eeb792c7 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql @@ -90,83 +90,83 @@ create view if not exists TARGET.totalresearchersft as select * from SOURCE.tota create view if not exists TARGET.hrrst as select * from SOURCE.hrrst; create table TARGET.result_citations stored as parquet as select * from SOURCE.result_citations orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_citations COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_citations COMPUTE STATISTICS; create table TARGET.result_references_oc stored as parquet as select * from SOURCE.result_references_oc orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_references_oc COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_references_oc COMPUTE STATISTICS; create table TARGET.result_citations_oc stored as parquet as select * from SOURCE.result_citations_oc orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_citations_oc COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_citations_oc COMPUTE STATISTICS; create table TARGET.result_classifications stored as parquet as select * from SOURCE.result_classifications orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_classifications COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_classifications COMPUTE STATISTICS; create table TARGET.result_apc stored as parquet as select * from SOURCE.result_apc orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_apc COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_apc COMPUTE STATISTICS; create table TARGET.result_concepts stored as parquet as select * from SOURCE.result_concepts orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_concepts COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_concepts COMPUTE STATISTICS; create table TARGET.result_datasources stored as parquet as select * from SOURCE.result_datasources orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_datasources COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_datasources COMPUTE STATISTICS; create table TARGET.result_fundercount stored as parquet as select * from SOURCE.result_fundercount orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_fundercount COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_fundercount COMPUTE STATISTICS; create table TARGET.result_gold stored as parquet as select * from SOURCE.result_gold orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_gold COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_gold COMPUTE STATISTICS; create table TARGET.result_greenoa stored as parquet as select * from SOURCE.result_greenoa orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_greenoa COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_greenoa COMPUTE STATISTICS; create table TARGET.result_languages stored as parquet as select * from SOURCE.result_languages orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_languages COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_languages COMPUTE STATISTICS; create table TARGET.result_licenses stored as parquet as select * from SOURCE.result_licenses orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_licenses COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_licenses COMPUTE STATISTICS; create table TARGET.licenses_normalized STORED AS PARQUET as select * from SOURCE.licenses_normalized; -ANALYZE TABLE TARGET.licenses_normalized COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.licenses_normalized COMPUTE STATISTICS; create table TARGET.result_oids stored as parquet as select * from SOURCE.result_oids orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_oids COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_oids COMPUTE STATISTICS; create table TARGET.result_organization stored as parquet as select * from SOURCE.result_organization orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_organization COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_organization COMPUTE STATISTICS; create table TARGET.result_peerreviewed stored as parquet as select * from SOURCE.result_peerreviewed orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_peerreviewed COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_peerreviewed COMPUTE STATISTICS; create table TARGET.result_pids stored as parquet as select * from SOURCE.result_pids orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_pids COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_pids COMPUTE STATISTICS; create table TARGET.result_projectcount stored as parquet as select * from SOURCE.result_projectcount orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_projectcount COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_projectcount COMPUTE STATISTICS; create table TARGET.result_projects stored as parquet as select * from SOURCE.result_projects orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_projects COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_projects COMPUTE STATISTICS; create table TARGET.result_refereed stored as parquet as select * from SOURCE.result_refereed orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_refereed COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_refereed COMPUTE STATISTICS; create table TARGET.result_sources stored as parquet as select * from SOURCE.result_sources orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_sources COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_sources COMPUTE STATISTICS; create table TARGET.result_topics stored as parquet as select * from SOURCE.result_topics orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_topics COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_topics COMPUTE STATISTICS; create table TARGET.result_fos stored as parquet as select * from SOURCE.result_fos orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_fos COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_fos COMPUTE STATISTICS; create table TARGET.result_accessroute stored as parquet as select * from SOURCE.result_accessroute orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_accessroute COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_accessroute COMPUTE STATISTICS; create view TARGET.foo1 as select * from SOURCE.result_result rr where rr.source in (select id from TARGET.result); create view TARGET.foo2 as select * from SOURCE.result_result rr where rr.target in (select id from TARGET.result); create table TARGET.result_result STORED AS PARQUET as select distinct * from (select * from TARGET.foo1 union all select * from TARGET.foo2) foufou; drop view TARGET.foo1; drop view TARGET.foo2; -ANALYZE TABLE TARGET.result_result COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_result COMPUTE STATISTICS; -- datasources create view if not exists TARGET.datasource as select * from SOURCE.datasource; @@ -175,7 +175,7 @@ create view if not exists TARGET.datasource_organizations as select * from SOURC create view if not exists TARGET.datasource_sources as select * from SOURCE.datasource_sources; create table TARGET.datasource_results stored as parquet as select id as result, datasource as id from TARGET.result_datasources; -ANALYZE TABLE TARGET.datasource_results COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.datasource_results COMPUTE STATISTICS; -- organizations create view if not exists TARGET.organization as select * from SOURCE.organization; @@ -193,28 +193,28 @@ create view if not exists TARGET.project_classification as select * from SOURCE. create view if not exists TARGET.project_organization_contribution as select * from SOURCE.project_organization_contribution; create table TARGET.project_results stored as parquet as select id as result, project as id from TARGET.result_projects; -ANALYZE TABLE TARGET.project_results COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.project_results COMPUTE STATISTICS; -- indicators -- Sprint 1 ---- create table TARGET.indi_pub_green_oa stored as parquet as select * from SOURCE.indi_pub_green_oa orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_green_oa COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_green_oa COMPUTE STATISTICS; create table TARGET.indi_pub_grey_lit stored as parquet as select * from SOURCE.indi_pub_grey_lit orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_grey_lit COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_grey_lit COMPUTE STATISTICS; create table TARGET.indi_pub_doi_from_crossref stored as parquet as select * from SOURCE.indi_pub_doi_from_crossref orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_doi_from_crossref COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_doi_from_crossref COMPUTE STATISTICS; -- Sprint 2 ---- create table TARGET.indi_result_has_cc_licence stored as parquet as select * from SOURCE.indi_result_has_cc_licence orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_has_cc_licence COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_result_has_cc_licence COMPUTE STATISTICS; create table TARGET.indi_result_has_cc_licence_url stored as parquet as select * from SOURCE.indi_result_has_cc_licence_url orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_has_cc_licence_url COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_result_has_cc_licence_url COMPUTE STATISTICS; create table TARGET.indi_pub_has_abstract stored as parquet as select * from SOURCE.indi_pub_has_abstract orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_has_abstract COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_has_abstract COMPUTE STATISTICS; create table TARGET.indi_result_with_orcid stored as parquet as select * from SOURCE.indi_result_with_orcid orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_with_orcid COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_result_with_orcid COMPUTE STATISTICS; ---- Sprint 3 ---- create table TARGET.indi_funded_result_with_fundref stored as parquet as select * from SOURCE.indi_funded_result_with_fundref orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_funded_result_with_fundref COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_funded_result_with_fundref COMPUTE STATISTICS; create view TARGET.indi_result_org_collab as select * from SOURCE.indi_result_org_collab; create view TARGET.indi_result_org_country_collab as select * from SOURCE.indi_result_org_country_collab; create view TARGET.indi_project_collab_org as select * from SOURCE.indi_project_collab_org; @@ -223,32 +223,32 @@ create view TARGET.indi_funder_country_collab as select * from SOURCE.indi_funde create view TARGET.indi_result_country_collab as select * from SOURCE.indi_result_country_collab; ---- Sprint 4 ---- create table TARGET.indi_pub_diamond stored as parquet as select * from SOURCE.indi_pub_diamond orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_diamond COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_diamond COMPUTE STATISTICS; create table TARGET.indi_pub_in_transformative stored as parquet as select * from SOURCE.indi_pub_in_transformative orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_in_transformative COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_in_transformative COMPUTE STATISTICS; create table TARGET.indi_pub_closed_other_open stored as parquet as select * from SOURCE.indi_pub_closed_other_open orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_closed_other_open COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_closed_other_open COMPUTE STATISTICS; ---- Sprint 5 ---- create table TARGET.indi_result_no_of_copies stored as parquet as select * from SOURCE.indi_result_no_of_copies orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_no_of_copies COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_result_no_of_copies COMPUTE STATISTICS; ---- Sprint 6 ---- create table TARGET.indi_pub_hybrid_oa_with_cc stored as parquet as select * from SOURCE.indi_pub_hybrid_oa_with_cc orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_hybrid_oa_with_cc COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_hybrid_oa_with_cc COMPUTE STATISTICS; create table TARGET.indi_pub_bronze_oa stored as parquet as select * from SOURCE.indi_pub_bronze_oa orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_bronze_oa COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_bronze_oa COMPUTE STATISTICS; create table TARGET.indi_pub_downloads stored as parquet as select * from SOURCE.indi_pub_downloads orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); -ANALYZE TABLE TARGET.indi_pub_downloads COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_downloads COMPUTE STATISTICS; create table TARGET.indi_pub_downloads_datasource stored as parquet as select * from SOURCE.indi_pub_downloads_datasource orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); -ANALYZE TABLE TARGET.indi_pub_downloads_datasource COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_downloads_datasource COMPUTE STATISTICS; create table TARGET.indi_pub_downloads_year stored as parquet as select * from SOURCE.indi_pub_downloads_year orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); -ANALYZE TABLE TARGET.indi_pub_downloads_year COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_downloads_year COMPUTE STATISTICS; create table TARGET.indi_pub_downloads_datasource_year stored as parquet as select * from SOURCE.indi_pub_downloads_datasource_year orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); -ANALYZE TABLE TARGET.indi_pub_downloads_datasource_year COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_downloads_datasource_year COMPUTE STATISTICS; ---- Sprint 7 ---- create table TARGET.indi_pub_gold_oa stored as parquet as select * from SOURCE.indi_pub_gold_oa orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_gold_oa COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_gold_oa COMPUTE STATISTICS; create table TARGET.indi_pub_hybrid stored as parquet as select * from SOURCE.indi_pub_hybrid orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_hybrid COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_hybrid COMPUTE STATISTICS; create view TARGET.indi_org_fairness as select * from SOURCE.indi_org_fairness; create view TARGET.indi_org_fairness_pub_pr as select * from SOURCE.indi_org_fairness_pub_pr; create view TARGET.indi_org_fairness_pub_year as select * from SOURCE.indi_org_fairness_pub_year; @@ -259,12 +259,14 @@ create view TARGET.indi_org_findable as select * from SOURCE.indi_org_findable; create view TARGET.indi_org_openess as select * from SOURCE.indi_org_openess; create view TARGET.indi_org_openess_year as select * from SOURCE.indi_org_openess_year; create table TARGET.indi_pub_has_preprint stored as parquet as select * from SOURCE.indi_pub_has_preprint orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_has_preprint COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_has_preprint COMPUTE STATISTICS; create table TARGET.indi_pub_in_subscribed stored as parquet as select * from SOURCE.indi_pub_in_subscribed orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_in_subscribed COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_in_subscribed COMPUTE STATISTICS; create table TARGET.indi_result_with_pid stored as parquet as select * from SOURCE.indi_result_with_pid orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_with_pid COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_result_with_pid COMPUTE STATISTICS; create table TARGET.indi_impact_measures stored as parquet as select * from SOURCE.indi_impact_measures orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_impact_measures COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_impact_measures COMPUTE STATISTICS; create table TARGET.indi_pub_interdisciplinarity stored as parquet as select * from SOURCE.indi_pub_interdisciplinarity orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_interdisciplinarity COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_interdisciplinarity COMPUTE STATISTICS; +create table TARGET.result_apc_affiliations stored as parquet as select * from SOURCE.result_apc_affiliations orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_apc_affiliations COMPUTE STATISTICS; diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql new file mode 100644 index 000000000..a59791084 --- /dev/null +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql @@ -0,0 +1,276 @@ +drop database if exists TARGET cascade; +create database if not exists TARGET; + +create view if not exists TARGET.category as select * from SOURCE.category; +create view if not exists TARGET.concept as select * from SOURCE.concept; +create view if not exists TARGET.context as select * from SOURCE.context; +create view if not exists TARGET.country as select * from SOURCE.country; +create view if not exists TARGET.countrygdp as select * from SOURCE.countrygdp; +create view if not exists TARGET.creation_date as select * from SOURCE.creation_date; +create view if not exists TARGET.funder as select * from SOURCE.funder; +create view if not exists TARGET.fundref as select * from SOURCE.fundref; +create view if not exists TARGET.rndexpenditure as select * from SOURCE.rndexpediture; +create view if not exists TARGET.rndgdpexpenditure as select * from SOURCE.rndgdpexpenditure; +create view if not exists TARGET.doctoratestudents as select * from SOURCE.doctoratestudents; +create view if not exists TARGET.totalresearchers as select * from SOURCE.totalresearchers; +create view if not exists TARGET.totalresearchersft as select * from SOURCE.totalresearchersft; +create view if not exists TARGET.hrrst as select * from SOURCE.hrrst; + +create table TARGET.result stored as parquet as + select distinct * from ( + select * from SOURCE.result r where exists (select 1 from SOURCE.result_projects rp join SOURCE.project p on rp.project=p.id where rp.id=r.id) + union all + select * from SOURCE.result r where exists (select 1 from SOURCE.result_concepts rc where rc.id=r.id) + union all + select * from SOURCE.result r where exists (select 1 from SOURCE.result_organization ro where ro.id=r.id and ro.organization in ( + 'openorgs____::b84450f9864182c67b8611b5593f4250', --"Athena Research and Innovation Center In Information Communication & Knowledge Technologies', --ARC" + 'openorgs____::d41cf6bd4ab1b1362a44397e0b95c975', --National Research Council + 'openorgs____::d2a09b9d5eabb10c95f9470e172d05d2', --??? Not exists ?? + 'openorgs____::d169c7407dd417152596908d48c11460', --Masaryk University + 'openorgs____::1ec924b1759bb16d0a02f2dad8689b21', --University of Belgrade + 'openorgs____::0ae431b820e4c33db8967fbb2b919150', --University of Helsinki + 'openorgs____::759d59f05d77188faee99b7493b46805', --University of Minho + 'openorgs____::cad284878801b9465fa51a95b1d779db', --Universidad Politécnica de Madrid + 'openorgs____::eadc8da90a546e98c03f896661a2e4d4', --University of Göttingen + 'openorgs____::c0286313e36479eff8676dba9b724b40', --National and Kapodistrian University of Athens + -- 'openorgs____::c80a8243a5e5c620d7931c88d93bf17a', --Université Paris Diderot + 'openorgs____::c08634f0a6b0081c3dc6e6c93a4314f3', --Bielefeld University + 'openorgs____::6fc85e4a8f7ecaf4b0c738d010e967ea', --University of Southern Denmark + 'openorgs____::3d6122f87f9a97a99d8f6e3d73313720', --Humboldt-Universität zu Berlin + 'openorgs____::16720ada63d0fa8ca41601feae7d1aa5', --TU Darmstadt + 'openorgs____::ccc0a066b56d2cfaf90c2ae369df16f5', --KU Leuven + 'openorgs____::4c6f119632adf789746f0a057ed73e90', --University of the Western Cape + 'openorgs____::ec3665affa01aeafa28b7852c4176dbd', --Rudjer Boskovic Institute + 'openorgs____::5f31346d444a7f06a28c880fb170b0f6', --Ghent University + 'openorgs____::2dbe47117fd5409f9c61620813456632', --University of Luxembourg + 'openorgs____::6445d7758d3a40c4d997953b6632a368', --National Institute of Informatics (NII) + 'openorgs____::b77c01aa15de3675da34277d48de2ec1', -- Valencia Catholic University Saint Vincent Martyr + 'openorgs____::7fe2f66cdc43983c6b24816bfe9cf6a0', -- Unviersity of Warsaw + 'openorgs____::15e7921fc50d9aa1229a82a84429419e', -- University Of Thessaly + 'openorgs____::11f7919dadc8f8a7251af54bba60c956', -- Technical University of Crete + 'openorgs____::84f0c5f5dbb6daf42748485924efde4b', -- University of Piraeus + 'openorgs____::4ac562f0376fce3539504567649cb373', -- University of Patras + 'openorgs____::3e8d1f8c3f6cd7f418b09f1f58b4873b', -- Aristotle University of Thessaloniki + 'openorgs____::3fcef6e1c469c10f2a84b281372c9814', -- World Bank + 'openorgs____::1698a2eb1885ef8adb5a4a969e745ad3', -- École des Ponts ParisTech + 'openorgs____::e15adb13c4dadd49de4d35c39b5da93a', -- Nanyang Technological University + 'openorgs____::4b34103bde246228fcd837f5f1bf4212', -- Autonomous University of Barcelona + 'openorgs____::72ec75fcfc4e0df1a76dc4c49007fceb', -- McMaster University + 'openorgs____::51c7fc556e46381734a25a6fbc3fd398', -- University of Modena and Reggio Emilia + 'openorgs____::235d7f9ad18ecd7e6dc62ea4990cb9db', -- Bilkent University + 'openorgs____::31f2fa9e05b49d4cf40a19c3fed8eb06', -- Saints Cyril and Methodius University of Skopje + 'openorgs____::db7686f30f22cbe73a4fde872ce812a6', -- University of Milan + 'openorgs____::b8b8ca674452579f3f593d9f5e557483', -- University College Cork + 'openorgs____::38d7097854736583dde879d12dacafca', -- Brown University + 'openorgs____::57784c9e047e826fefdb1ef816120d92', --Arts et Métiers ParisTech + 'openorgs____::2530baca8a15936ba2e3297f2bce2e7e', -- University of Cape Town + 'openorgs____::d11f981828c485cd23d93f7f24f24db1', -- Technological University Dublin + 'openorgs____::5e6bf8962665cdd040341171e5c631d8', -- Delft University of Technology + 'openorgs____::846cb428d3f52a445f7275561a7beb5d', -- University of Manitoba + 'openorgs____::eb391317ed0dc684aa81ac16265de041', -- Universitat Rovira i Virgili + 'openorgs____::66aa9fc2fceb271423dfabcc38752dc0', -- Lund University + 'openorgs____::3cff625a4370d51e08624cc586138b2f', -- IMT Atlantique + 'openorgs____::c0b262bd6eab819e4c994914f9c010e2', -- National Institute of Geophysics and Volcanology + 'openorgs____::1624ff7c01bb641b91f4518539a0c28a', -- Vrije Universiteit Amsterdam + 'openorgs____::4d4051b56708688235252f1d8fddb8c1', --Iscte - Instituto Universitário de Lisboa + 'openorgs____::ab4ac74c35fa5dada770cf08e5110fab' -- Universidade Católica Portuguesa + ) )) foo; + +--ANALYZE TABLE TARGET.result COMPUTE STATISTICS; + +create view if not exists TARGET.category as select * from SOURCE.category; +create view if not exists TARGET.concept as select * from SOURCE.concept; +create view if not exists TARGET.context as select * from SOURCE.context; +create view if not exists TARGET.country as select * from SOURCE.country; +create view if not exists TARGET.countrygdp as select * from SOURCE.countrygdp; +create view if not exists TARGET.creation_date as select * from SOURCE.creation_date; +create view if not exists TARGET.funder as select * from SOURCE.funder; +create view if not exists TARGET.fundref as select * from SOURCE.fundref; +create view if not exists TARGET.rndexpenditure as select * from SOURCE.rndexpediture; +create view if not exists TARGET.rndgdpexpenditure as select * from SOURCE.rndgdpexpenditure; +create view if not exists TARGET.doctoratestudents as select * from SOURCE.doctoratestudents; +create view if not exists TARGET.totalresearchers as select * from SOURCE.totalresearchers; +create view if not exists TARGET.totalresearchersft as select * from SOURCE.totalresearchersft; +create view if not exists TARGET.hrrst as select * from SOURCE.hrrst; + +create table TARGET.result_citations stored as parquet as select * from SOURCE.result_citations orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_citations COMPUTE STATISTICS; + +create table TARGET.result_references_oc stored as parquet as select * from SOURCE.result_references_oc orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_references_oc COMPUTE STATISTICS; + +create table TARGET.result_citations_oc stored as parquet as select * from SOURCE.result_citations_oc orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_citations_oc COMPUTE STATISTICS; + +create table TARGET.result_classifications stored as parquet as select * from SOURCE.result_classifications orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_classifications COMPUTE STATISTICS; + +create table TARGET.result_apc stored as parquet as select * from SOURCE.result_apc orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_apc COMPUTE STATISTICS; + +create table TARGET.result_concepts stored as parquet as select * from SOURCE.result_concepts orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_concepts COMPUTE STATISTICS; + +create table TARGET.result_datasources stored as parquet as select * from SOURCE.result_datasources orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_datasources COMPUTE STATISTICS; + +create table TARGET.result_fundercount stored as parquet as select * from SOURCE.result_fundercount orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_fundercount COMPUTE STATISTICS; + +create table TARGET.result_gold stored as parquet as select * from SOURCE.result_gold orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_gold COMPUTE STATISTICS; + +create table TARGET.result_greenoa stored as parquet as select * from SOURCE.result_greenoa orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_greenoa COMPUTE STATISTICS; + +create table TARGET.result_languages stored as parquet as select * from SOURCE.result_languages orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_languages COMPUTE STATISTICS; + +create table TARGET.result_licenses stored as parquet as select * from SOURCE.result_licenses orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_licenses COMPUTE STATISTICS; + +create table TARGET.licenses_normalized STORED AS PARQUET as select * from SOURCE.licenses_normalized; +--ANALYZE TABLE TARGET.licenses_normalized COMPUTE STATISTICS; + +create table TARGET.result_oids stored as parquet as select * from SOURCE.result_oids orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_oids COMPUTE STATISTICS; + +create table TARGET.result_organization stored as parquet as select * from SOURCE.result_organization orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_organization COMPUTE STATISTICS; + +create table TARGET.result_peerreviewed stored as parquet as select * from SOURCE.result_peerreviewed orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_peerreviewed COMPUTE STATISTICS; + +create table TARGET.result_pids stored as parquet as select * from SOURCE.result_pids orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_pids COMPUTE STATISTICS; + +create table TARGET.result_projectcount stored as parquet as select * from SOURCE.result_projectcount orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_projectcount COMPUTE STATISTICS; + +create table TARGET.result_projects stored as parquet as select * from SOURCE.result_projects orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_projects COMPUTE STATISTICS; + +create table TARGET.result_refereed stored as parquet as select * from SOURCE.result_refereed orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_refereed COMPUTE STATISTICS; + +create table TARGET.result_sources stored as parquet as select * from SOURCE.result_sources orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_sources COMPUTE STATISTICS; + +create table TARGET.result_topics stored as parquet as select * from SOURCE.result_topics orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_topics COMPUTE STATISTICS; + +create table TARGET.result_fos stored as parquet as select * from SOURCE.result_fos orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_fos COMPUTE STATISTICS; + +create table TARGET.result_accessroute stored as parquet as select * from SOURCE.result_accessroute orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_accessroute COMPUTE STATISTICS; + +create view TARGET.foo1 as select * from SOURCE.result_result rr where rr.source in (select id from TARGET.result); +create view TARGET.foo2 as select * from SOURCE.result_result rr where rr.target in (select id from TARGET.result); +create table TARGET.result_result STORED AS PARQUET as select distinct * from (select * from TARGET.foo1 union all select * from TARGET.foo2) foufou; +drop view TARGET.foo1; +drop view TARGET.foo2; +--ANALYZE TABLE TARGET.result_result COMPUTE STATISTICS; + +-- datasources +create view if not exists TARGET.datasource as select * from SOURCE.datasource; +create view if not exists TARGET.datasource_oids as select * from SOURCE.datasource_oids; +create view if not exists TARGET.datasource_organizations as select * from SOURCE.datasource_organizations; +create view if not exists TARGET.datasource_sources as select * from SOURCE.datasource_sources; + +create table TARGET.datasource_results stored as parquet as select id as result, datasource as id from TARGET.result_datasources; +--ANALYZE TABLE TARGET.datasource_results COMPUTE STATISTICS; + +-- organizations +create view if not exists TARGET.organization as select * from SOURCE.organization; +create view if not exists TARGET.organization_datasources as select * from SOURCE.organization_datasources; +create view if not exists TARGET.organization_pids as select * from SOURCE.organization_pids; +create view if not exists TARGET.organization_projects as select * from SOURCE.organization_projects; +create view if not exists TARGET.organization_sources as select * from SOURCE.organization_sources; + +-- projects +create view if not exists TARGET.project as select * from SOURCE.project; +create view if not exists TARGET.project_oids as select * from SOURCE.project_oids; +create view if not exists TARGET.project_organizations as select * from SOURCE.project_organizations; +create view if not exists TARGET.project_resultcount as select * from SOURCE.project_resultcount; +create view if not exists TARGET.project_classification as select * from SOURCE.project_classification; +create view if not exists TARGET.project_organization_contribution as select * from SOURCE.project_organization_contribution; + +create table TARGET.project_results stored as parquet as select id as result, project as id from TARGET.result_projects; +--ANALYZE TABLE TARGET.project_results COMPUTE STATISTICS; + +-- indicators +-- Sprint 1 ---- +create table TARGET.indi_pub_green_oa stored as parquet as select * from SOURCE.indi_pub_green_oa orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_green_oa COMPUTE STATISTICS; +create table TARGET.indi_pub_grey_lit stored as parquet as select * from SOURCE.indi_pub_grey_lit orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_grey_lit COMPUTE STATISTICS; +create table TARGET.indi_pub_doi_from_crossref stored as parquet as select * from SOURCE.indi_pub_doi_from_crossref orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_doi_from_crossref COMPUTE STATISTICS; +-- Sprint 2 ---- +create table TARGET.indi_result_has_cc_licence stored as parquet as select * from SOURCE.indi_result_has_cc_licence orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_result_has_cc_licence COMPUTE STATISTICS; +create table TARGET.indi_result_has_cc_licence_url stored as parquet as select * from SOURCE.indi_result_has_cc_licence_url orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_result_has_cc_licence_url COMPUTE STATISTICS; +create table TARGET.indi_pub_has_abstract stored as parquet as select * from SOURCE.indi_pub_has_abstract orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_has_abstract COMPUTE STATISTICS; +create table TARGET.indi_result_with_orcid stored as parquet as select * from SOURCE.indi_result_with_orcid orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_result_with_orcid COMPUTE STATISTICS; +---- Sprint 3 ---- +create table TARGET.indi_funded_result_with_fundref stored as parquet as select * from SOURCE.indi_funded_result_with_fundref orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_funded_result_with_fundref COMPUTE STATISTICS; +create view TARGET.indi_result_org_collab as select * from SOURCE.indi_result_org_collab; +create view TARGET.indi_result_org_country_collab as select * from SOURCE.indi_result_org_country_collab; +create view TARGET.indi_project_collab_org as select * from SOURCE.indi_project_collab_org; +create view TARGET.indi_project_collab_org_country as select * from SOURCE.indi_project_collab_org_country; +create view TARGET.indi_funder_country_collab as select * from SOURCE.indi_funder_country_collab; +create view TARGET.indi_result_country_collab as select * from SOURCE.indi_result_country_collab; +---- Sprint 4 ---- +create table TARGET.indi_pub_diamond stored as parquet as select * from SOURCE.indi_pub_diamond orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_diamond COMPUTE STATISTICS; +create table TARGET.indi_pub_in_transformative stored as parquet as select * from SOURCE.indi_pub_in_transformative orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_in_transformative COMPUTE STATISTICS; +create table TARGET.indi_pub_closed_other_open stored as parquet as select * from SOURCE.indi_pub_closed_other_open orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_closed_other_open COMPUTE STATISTICS; +---- Sprint 5 ---- +create table TARGET.indi_result_no_of_copies stored as parquet as select * from SOURCE.indi_result_no_of_copies orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_result_no_of_copies COMPUTE STATISTICS; +---- Sprint 6 ---- +create table TARGET.indi_pub_hybrid_oa_with_cc stored as parquet as select * from SOURCE.indi_pub_hybrid_oa_with_cc orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_hybrid_oa_with_cc COMPUTE STATISTICS; +create table TARGET.indi_pub_bronze_oa stored as parquet as select * from SOURCE.indi_pub_bronze_oa orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_bronze_oa COMPUTE STATISTICS; +create table TARGET.indi_pub_downloads stored as parquet as select * from SOURCE.indi_pub_downloads orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); +--ANALYZE TABLE TARGET.indi_pub_downloads COMPUTE STATISTICS; +create table TARGET.indi_pub_downloads_datasource stored as parquet as select * from SOURCE.indi_pub_downloads_datasource orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); +--ANALYZE TABLE TARGET.indi_pub_downloads_datasource COMPUTE STATISTICS; +create table TARGET.indi_pub_downloads_year stored as parquet as select * from SOURCE.indi_pub_downloads_year orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); +--ANALYZE TABLE TARGET.indi_pub_downloads_year COMPUTE STATISTICS; +create table TARGET.indi_pub_downloads_datasource_year stored as parquet as select * from SOURCE.indi_pub_downloads_datasource_year orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); +--ANALYZE TABLE TARGET.indi_pub_downloads_datasource_year COMPUTE STATISTICS; +---- Sprint 7 ---- +create table TARGET.indi_pub_gold_oa stored as parquet as select * from SOURCE.indi_pub_gold_oa orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_gold_oa COMPUTE STATISTICS; +create table TARGET.indi_pub_hybrid stored as parquet as select * from SOURCE.indi_pub_hybrid orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_hybrid COMPUTE STATISTICS; +create view TARGET.indi_org_fairness as select * from SOURCE.indi_org_fairness; +create view TARGET.indi_org_fairness_pub_pr as select * from SOURCE.indi_org_fairness_pub_pr; +create view TARGET.indi_org_fairness_pub_year as select * from SOURCE.indi_org_fairness_pub_year; +create view TARGET.indi_org_fairness_pub as select * from SOURCE.indi_org_fairness_pub; +create view TARGET.indi_org_fairness_year as select * from SOURCE.indi_org_fairness_year; +create view TARGET.indi_org_findable_year as select * from SOURCE.indi_org_findable_year; +create view TARGET.indi_org_findable as select * from SOURCE.indi_org_findable; +create view TARGET.indi_org_openess as select * from SOURCE.indi_org_openess; +create view TARGET.indi_org_openess_year as select * from SOURCE.indi_org_openess_year; +create table TARGET.indi_pub_has_preprint stored as parquet as select * from SOURCE.indi_pub_has_preprint orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_has_preprint COMPUTE STATISTICS; +create table TARGET.indi_pub_in_subscribed stored as parquet as select * from SOURCE.indi_pub_in_subscribed orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_in_subscribed COMPUTE STATISTICS; +create table TARGET.indi_result_with_pid stored as parquet as select * from SOURCE.indi_result_with_pid orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_result_with_pid COMPUTE STATISTICS; +create table TARGET.indi_impact_measures stored as parquet as select * from SOURCE.indi_impact_measures orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_impact_measures COMPUTE STATISTICS; +create table TARGET.indi_pub_interdisciplinarity stored as parquet as select * from SOURCE.indi_pub_interdisciplinarity orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_interdisciplinarity COMPUTE STATISTICS; +create table TARGET.result_apc_affiliations stored as parquet as select * from SOURCE.result_apc_affiliations orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_apc_affiliations COMPUTE STATISTICS; diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs.sql index 92b40405d..9a9407c2d 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs.sql @@ -12,4 +12,4 @@ create table TARGET.result stored as parquet as -- join SOURCE.result where rc.id=r.id and conc.category like CONTEXT) ) foo; -ANALYZE TABLE TARGET.result COMPUTE STATISTICS; \ No newline at end of file +--ANALYZE TABLE TARGET.result COMPUTE STATISTICS; \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs_tail.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs_tail.sql index ef6d08d79..bad18efde 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs_tail.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs_tail.sql @@ -12,4 +12,4 @@ create table TARGET.result stored as parquet as -- join SOURCE.result where rc.id=r.id and conc.category not in (CONTEXTS)) ) foo; -ANALYZE TABLE TARGET.result COMPUTE STATISTICS; \ No newline at end of file +--ANALYZE TABLE TARGET.result COMPUTE STATISTICS; \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_funded.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_funded.sql index 8d8739c74..b8d3c0242 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_funded.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_funded.sql @@ -6,4 +6,4 @@ create table TARGET.result stored as parquet as select * from SOURCE.result r where exists (select 1 from SOURCE.result_projects rp join SOURCE.project p on rp.project=p.id where rp.id=r.id) ) foo; -ANALYZE TABLE TARGET.result COMPUTE STATISTICS; \ No newline at end of file +--ANALYZE TABLE TARGET.result COMPUTE STATISTICS; \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql index 442e623cd..1f75c3cd1 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql @@ -42,7 +42,7 @@ create table TARGET.result stored as parquet as 'openorgs____::31f2fa9e05b49d4cf40a19c3fed8eb06', -- Saints Cyril and Methodius University of Skopje 'openorgs____::db7686f30f22cbe73a4fde872ce812a6', -- University of Milan 'openorgs____::b8b8ca674452579f3f593d9f5e557483', -- University College Cork - 'openorgs____::38d7097854736583dde879d12dacafca' -- Brown University + 'openorgs____::38d7097854736583dde879d12dacafca', -- Brown University 'openorgs____::57784c9e047e826fefdb1ef816120d92', --Arts et Métiers ParisTech 'openorgs____::2530baca8a15936ba2e3297f2bce2e7e', -- University of Cape Town 'openorgs____::d11f981828c485cd23d93f7f24f24db1', -- Technological University Dublin @@ -52,7 +52,10 @@ create table TARGET.result stored as parquet as 'openorgs____::66aa9fc2fceb271423dfabcc38752dc0', -- Lund University 'openorgs____::3cff625a4370d51e08624cc586138b2f', -- IMT Atlantique 'openorgs____::c0b262bd6eab819e4c994914f9c010e2', -- National Institute of Geophysics and Volcanology - 'openorgs____::1624ff7c01bb641b91f4518539a0c28a' -- Vrije Universiteit Amsterdam + 'openorgs____::1624ff7c01bb641b91f4518539a0c28a', -- Vrije Universiteit Amsterdam + 'openorgs____::4d4051b56708688235252f1d8fddb8c1', --Iscte - Instituto Universitário de Lisboa + 'openorgs____::ab4ac74c35fa5dada770cf08e5110fab' -- Universidade Católica Portuguesa + ))) foo; -ANALYZE TABLE TARGET.result COMPUTE STATISTICS; \ No newline at end of file +--ANALYZE TABLE TARGET.result COMPUTE STATISTICS; \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step21-createObservatoryDB.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step21-createObservatoryDB.sql index 2d7d572b3..b7e421813 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step21-createObservatoryDB.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step21-createObservatoryDB.sql @@ -8,7 +8,7 @@ from ${stats_db_name}.result r group by rl.id ) rln on rln.id=r.id; -ANALYZE TABLE ${observatory_db_name}.result_cc_licence COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_cc_licence COMPUTE STATISTICS; create table ${observatory_db_name}.result_affiliated_country stored as parquet as select @@ -39,7 +39,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, c.code, c.name; -ANALYZE TABLE ${observatory_db_name}.result_affiliated_country COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_affiliated_country COMPUTE STATISTICS; create table ${observatory_db_name}.result_affiliated_year stored as parquet as select @@ -70,7 +70,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, r.year; -ANALYZE TABLE ${observatory_db_name}.result_affiliated_year COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_affiliated_year COMPUTE STATISTICS; create table ${observatory_db_name}.result_affiliated_year_country stored as parquet as select @@ -101,7 +101,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, r.year, c.code, c.name; -ANALYZE TABLE ${observatory_db_name}.result_affiliated_year_country COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_affiliated_year_country COMPUTE STATISTICS; create table ${observatory_db_name}.result_affiliated_datasource stored as parquet as select @@ -134,7 +134,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, d.name; -ANALYZE TABLE ${observatory_db_name}.result_affiliated_datasource COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_affiliated_datasource COMPUTE STATISTICS; create table ${observatory_db_name}.result_affiliated_datasource_country stored as parquet as select @@ -167,7 +167,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, d.name, c.code, c.name; -ANALYZE TABLE ${observatory_db_name}.result_affiliated_datasource_country COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_affiliated_datasource_country COMPUTE STATISTICS; create table ${observatory_db_name}.result_affiliated_organization stored as parquet as select @@ -198,7 +198,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, o.name; -ANALYZE TABLE ${observatory_db_name}.result_affiliated_organization COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_affiliated_organization COMPUTE STATISTICS; create table ${observatory_db_name}.result_affiliated_organization_country stored as parquet as select @@ -229,7 +229,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, o.name, c.code, c.name; -ANALYZE TABLE ${observatory_db_name}.result_affiliated_organization_country COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_affiliated_organization_country COMPUTE STATISTICS; create table ${observatory_db_name}.result_affiliated_funder stored as parquet as select @@ -262,7 +262,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, p.funder; -ANALYZE TABLE ${observatory_db_name}.result_affiliated_funder COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_affiliated_funder COMPUTE STATISTICS; create table ${observatory_db_name}.result_affiliated_funder_country stored as parquet as select @@ -295,7 +295,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, p.funder, c.code, c.name; -ANALYZE TABLE ${observatory_db_name}.result_affiliated_funder_country COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_affiliated_funder_country COMPUTE STATISTICS; create table ${observatory_db_name}.result_deposited_country stored as parquet as select @@ -328,7 +328,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, c.code, c.name; -ANALYZE TABLE ${observatory_db_name}.result_deposited_country COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_deposited_country COMPUTE STATISTICS; create table ${observatory_db_name}.result_deposited_year stored as parquet as select @@ -361,7 +361,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, r.year; -ANALYZE TABLE ${observatory_db_name}.result_deposited_year COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_deposited_year COMPUTE STATISTICS; create table ${observatory_db_name}.result_deposited_year_country stored as parquet as select @@ -394,7 +394,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, r.year, c.code, c.name; -ANALYZE TABLE ${observatory_db_name}.result_deposited_year_country COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_deposited_year_country COMPUTE STATISTICS; create table ${observatory_db_name}.result_deposited_datasource stored as parquet as select @@ -427,7 +427,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, d.name; -ANALYZE TABLE ${observatory_db_name}.result_deposited_datasource COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_deposited_datasource COMPUTE STATISTICS; create table ${observatory_db_name}.result_deposited_datasource_country stored as parquet as select @@ -460,7 +460,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, d.name, c.code, c.name; -ANALYZE TABLE ${observatory_db_name}.result_deposited_datasource_country COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_deposited_datasource_country COMPUTE STATISTICS; create table ${observatory_db_name}.result_deposited_organization stored as parquet as select @@ -493,7 +493,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, o.name; -ANALYZE TABLE ${observatory_db_name}.result_deposited_organization COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_deposited_organization COMPUTE STATISTICS; create table ${observatory_db_name}.result_deposited_organization_country stored as parquet as select @@ -526,7 +526,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, o.name, c.code, c.name; -ANALYZE TABLE ${observatory_db_name}.result_deposited_organization_country COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_deposited_organization_country COMPUTE STATISTICS; create table ${observatory_db_name}.result_deposited_funder stored as parquet as select @@ -561,7 +561,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, p.funder; -ANALYZE TABLE ${observatory_db_name}.result_deposited_funder COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_deposited_funder COMPUTE STATISTICS; create table ${observatory_db_name}.result_deposited_funder_country stored as parquet as select @@ -596,4 +596,4 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, p.funder, c.code, c.name; -ANALYZE TABLE ${observatory_db_name}.result_deposited_funder_country COMPUTE STATISTICS; \ No newline at end of file +--ANALYZE TABLE ${observatory_db_name}.result_deposited_funder_country COMPUTE STATISTICS; \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml index 2ab50fb29..c03520e48 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml @@ -317,15 +317,12 @@ - - ${jobTracker} - ${nameNode} - indicators.sh - ${stats_db_name} - ${external_stats_db_name} - ${wf:appPath()}/scripts/step16-createIndicatorsTables.sql - indicators.sh - + + ${hive_jdbc_url} + + stats_db_name=${stats_db_name} + external_stats_db_name=${external_stats_db_name} + @@ -378,6 +375,7 @@ ${wf:appPath()}/scripts/step20-createMonitorDB_institutions.sql ${wf:appPath()}/scripts/step20-createMonitorDB_RIs.sql ${wf:appPath()}/scripts/step20-createMonitorDB_RIs_tail.sql + ${wf:appPath()}/scripts/step20-createMonitorDBAll.sql monitor.sh @@ -469,7 +467,7 @@ ${usage_stats_db_shadow_name} finalizeImpalaCluster.sh - + From be4856ef35401dc7a6e969763839254e645456fb Mon Sep 17 00:00:00 2001 From: dimitrispie Date: Mon, 17 Jul 2023 15:33:58 +0300 Subject: [PATCH 006/148] Update step15.sql --- .../eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql index 75e8b001b..d1cbde438 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql @@ -41,8 +41,6 @@ cast(measures_ids.unit.value[0] as decimal(6,3)) score_dec, measures_ids.unit.va from ${openaire_db_name}.result lateral view explode(measures) measures as measures_ids where measures_ids.id!='views' and measures_ids.id!='downloads'; -ANALYZE TABLE indi_impact_measures COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.result_apc_affiliations STORED AS PARQUET as select distinct substr(rel.target,4) id, substr(rel.source,4) organization, o.legalname.value name, cast(rel.properties[0].value as double) apc_amount, From 6cc7d8ca7b04f8aff859fda0ff04294b830608b5 Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Thu, 24 Aug 2023 21:48:07 +0200 Subject: [PATCH 007/148] GroupEntities and DispatchEntites are now merged in GroupEntitiesSparkJob --- .../oa/merge/DispatchEntitiesSparkJob.java | 98 -------- .../dhp/oa/merge/GroupEntitiesSparkJob.java | 221 ++++++++---------- .../merge/dispatch_entities_parameters.json | 26 --- .../group_graph_entities_parameters.json | 16 +- .../dedup/consistency/oozie_app/workflow.xml | 26 +-- .../dhp/oa/graph/group/oozie_app/workflow.xml | 25 +- .../group/GroupEntitiesSparkJobTest.java | 201 ++++++++-------- 7 files changed, 203 insertions(+), 410 deletions(-) delete mode 100644 dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/DispatchEntitiesSparkJob.java delete mode 100644 dhp-common/src/main/resources/eu/dnetlib/dhp/oa/merge/dispatch_entities_parameters.json diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/DispatchEntitiesSparkJob.java b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/DispatchEntitiesSparkJob.java deleted file mode 100644 index 4d2ccb178..000000000 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/DispatchEntitiesSparkJob.java +++ /dev/null @@ -1,98 +0,0 @@ - -package eu.dnetlib.dhp.oa.merge; - -import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; - -import java.util.Objects; -import java.util.Optional; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.spark.SparkConf; -import org.apache.spark.api.java.function.FilterFunction; -import org.apache.spark.api.java.function.MapFunction; -import org.apache.spark.sql.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import eu.dnetlib.dhp.application.ArgumentApplicationParser; -import eu.dnetlib.dhp.common.HdfsSupport; -import eu.dnetlib.dhp.schema.common.ModelSupport; - -public class DispatchEntitiesSparkJob { - - private static final Logger log = LoggerFactory.getLogger(DispatchEntitiesSparkJob.class); - - public static void main(String[] args) throws Exception { - - String jsonConfiguration = IOUtils - .toString( - Objects - .requireNonNull( - DispatchEntitiesSparkJob.class - .getResourceAsStream( - "/eu/dnetlib/dhp/oa/merge/dispatch_entities_parameters.json"))); - final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); - parser.parseArgument(args); - - Boolean isSparkSessionManaged = Optional - .ofNullable(parser.get("isSparkSessionManaged")) - .map(Boolean::valueOf) - .orElse(Boolean.TRUE); - log.info("isSparkSessionManaged: {}", isSparkSessionManaged); - - String inputPath = parser.get("inputPath"); - log.info("inputPath: {}", inputPath); - - String outputPath = parser.get("outputPath"); - log.info("outputPath: {}", outputPath); - - boolean filterInvisible = Boolean.valueOf(parser.get("filterInvisible")); - log.info("filterInvisible: {}", filterInvisible); - - SparkConf conf = new SparkConf(); - runWithSparkSession( - conf, - isSparkSessionManaged, - spark -> { - HdfsSupport.remove(outputPath, spark.sparkContext().hadoopConfiguration()); - dispatchEntities(spark, inputPath, outputPath, filterInvisible); - }); - } - - private static void dispatchEntities( - SparkSession spark, - String inputPath, - String outputPath, - boolean filterInvisible) { - - Dataset df = spark.read().textFile(inputPath); - - ModelSupport.oafTypes.entrySet().parallelStream().forEach(entry -> { - String entityType = entry.getKey(); - Class clazz = entry.getValue(); - - if (!entityType.equalsIgnoreCase("relation")) { - Dataset entityDF = spark - .read() - .schema(Encoders.bean(clazz).schema()) - .json( - df - .filter((FilterFunction) s -> s.startsWith(clazz.getName())) - .map( - (MapFunction) s -> StringUtils.substringAfter(s, "|"), - Encoders.STRING())); - - if (filterInvisible) { - entityDF = entityDF.filter("dataInfo.invisible != true"); - } - - entityDF - .write() - .mode(SaveMode.Overwrite) - .option("compression", "gzip") - .json(outputPath + "/" + entityType); - } - }); - } -} diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java index e652bd5b6..87510c108 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java @@ -2,36 +2,28 @@ package eu.dnetlib.dhp.oa.merge; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; -import static eu.dnetlib.dhp.utils.DHPUtils.toSeq; +import static org.apache.spark.sql.functions.col; +import static org.apache.spark.sql.functions.when; -import java.io.IOException; -import java.util.List; -import java.util.Objects; +import java.util.Map; import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ForkJoinPool; import java.util.stream.Collectors; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.spark.SparkConf; -import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.api.java.function.FilterFunction; import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.api.java.function.ReduceFunction; import org.apache.spark.sql.*; -import org.apache.spark.sql.expressions.Aggregator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.jayway.jsonpath.Configuration; -import com.jayway.jsonpath.DocumentContext; -import com.jayway.jsonpath.JsonPath; -import com.jayway.jsonpath.Option; - import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.common.HdfsSupport; +import eu.dnetlib.dhp.schema.common.EntityType; import eu.dnetlib.dhp.schema.common.ModelSupport; -import eu.dnetlib.dhp.schema.oaf.*; +import eu.dnetlib.dhp.schema.oaf.OafEntity; import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils; import scala.Tuple2; @@ -39,13 +31,9 @@ import scala.Tuple2; * Groups the graph content by entity identifier to ensure ID uniqueness */ public class GroupEntitiesSparkJob { - private static final Logger log = LoggerFactory.getLogger(GroupEntitiesSparkJob.class); - private static final String ID_JPATH = "$.id"; - - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper() - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + private static final Encoder OAFENTITY_KRYO_ENC = Encoders.kryo(OafEntity.class); public static void main(String[] args) throws Exception { @@ -66,9 +54,15 @@ public class GroupEntitiesSparkJob { String graphInputPath = parser.get("graphInputPath"); log.info("graphInputPath: {}", graphInputPath); + String checkpointPath = parser.get("checkpointPath"); + log.info("checkpointPath: {}", checkpointPath); + String outputPath = parser.get("outputPath"); log.info("outputPath: {}", outputPath); + boolean filterInvisible = Boolean.valueOf(parser.get("filterInvisible")); + log.info("filterInvisible: {}", filterInvisible); + SparkConf conf = new SparkConf(); conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); conf.registerKryoClasses(ModelSupport.getOafModelClasses()); @@ -78,126 +72,95 @@ public class GroupEntitiesSparkJob { isSparkSessionManaged, spark -> { HdfsSupport.remove(outputPath, spark.sparkContext().hadoopConfiguration()); - groupEntities(spark, graphInputPath, outputPath); + groupEntities(spark, graphInputPath, checkpointPath, outputPath, filterInvisible); }); } private static void groupEntities( SparkSession spark, String inputPath, - String outputPath) { + String checkpointPath, + String outputPath, + boolean filterInvisible) { - final TypedColumn aggregator = new GroupingAggregator().toColumn(); - final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); - spark - .read() - .textFile(toSeq(listEntityPaths(inputPath, sc))) - .map((MapFunction) GroupEntitiesSparkJob::parseOaf, Encoders.kryo(OafEntity.class)) - .filter((FilterFunction) e -> StringUtils.isNotBlank(ModelSupport.idFn().apply(e))) - .groupByKey((MapFunction) oaf -> ModelSupport.idFn().apply(oaf), Encoders.STRING()) - .agg(aggregator) + Dataset allEntities = spark.emptyDataset(OAFENTITY_KRYO_ENC); + + for (Map.Entry e : ModelSupport.entityTypes.entrySet()) { + String entity = e.getKey().name(); + Class entityClass = e.getValue(); + String entityInputPath = inputPath + "/" + entity; + + if (!HdfsSupport.exists(entityInputPath, spark.sparkContext().hadoopConfiguration())) { + continue; + } + + allEntities = allEntities + .union( + ((Dataset) spark + .read() + .schema(Encoders.bean(entityClass).schema()) + .json(entityInputPath) + .filter("length(id) > 0") + .as(Encoders.bean(entityClass))) + .map((MapFunction) r -> r, OAFENTITY_KRYO_ENC)); + } + + Dataset groupedEntities = allEntities + .groupByKey((MapFunction) OafEntity::getId, Encoders.STRING()) + .reduceGroups((ReduceFunction) (b, a) -> OafMapperUtils.mergeEntities(b, a)) .map( - (MapFunction, String>) t -> t._2().getClass().getName() + - "|" + OBJECT_MAPPER.writeValueAsString(t._2()), - Encoders.STRING()) + (MapFunction, Tuple2>) t -> new Tuple2( + t._2().getClass().getName(), t._2()), + Encoders.tuple(Encoders.STRING(), OAFENTITY_KRYO_ENC)); + + // pivot on "_1" (classname of the entity) + // created columns containing only entities of the same class + for (Map.Entry e : ModelSupport.entityTypes.entrySet()) { + String entity = e.getKey().name(); + Class entityClass = e.getValue(); + + groupedEntities = groupedEntities + .withColumn( + entity, + when(col("_1").equalTo(entityClass.getName()), col("_2"))); + } + + groupedEntities + .drop("_1", "_2") .write() - .option("compression", "gzip") .mode(SaveMode.Overwrite) - .text(outputPath); - } + .option("compression", "gzip") + .save(checkpointPath); - public static class GroupingAggregator extends Aggregator { + ForkJoinPool parPool = new ForkJoinPool(ModelSupport.entityTypes.size()); - @Override - public OafEntity zero() { - return null; - } - - @Override - public OafEntity reduce(OafEntity b, OafEntity a) { - return mergeAndGet(b, a); - } - - private OafEntity mergeAndGet(OafEntity b, OafEntity a) { - if (Objects.nonNull(a) && Objects.nonNull(b)) { - return OafMapperUtils.mergeEntities(b, a); - } - return Objects.isNull(a) ? b : a; - } - - @Override - public OafEntity merge(OafEntity b, OafEntity a) { - return mergeAndGet(b, a); - } - - @Override - public OafEntity finish(OafEntity j) { - return j; - } - - @Override - public Encoder bufferEncoder() { - return Encoders.kryo(OafEntity.class); - } - - @Override - public Encoder outputEncoder() { - return Encoders.kryo(OafEntity.class); - } - - } - - private static OafEntity parseOaf(String s) { - - DocumentContext dc = JsonPath - .parse(s, Configuration.defaultConfiguration().addOptions(Option.SUPPRESS_EXCEPTIONS)); - final String id = dc.read(ID_JPATH); - if (StringUtils.isNotBlank(id)) { - - String prefix = StringUtils.substringBefore(id, "|"); - switch (prefix) { - case "10": - return parse(s, Datasource.class); - case "20": - return parse(s, Organization.class); - case "40": - return parse(s, Project.class); - case "50": - String resultType = dc.read("$.resulttype.classid"); - switch (resultType) { - case "publication": - return parse(s, Publication.class); - case "dataset": - return parse(s, eu.dnetlib.dhp.schema.oaf.Dataset.class); - case "software": - return parse(s, Software.class); - case "other": - return parse(s, OtherResearchProduct.class); - default: - throw new IllegalArgumentException(String.format("invalid resultType: '%s'", resultType)); - } - default: - throw new IllegalArgumentException(String.format("invalid id prefix: '%s'", prefix)); - } - } else { - throw new IllegalArgumentException(String.format("invalid oaf: '%s'", s)); - } - } - - private static OafEntity parse(String s, Class clazz) { - try { - return OBJECT_MAPPER.readValue(s, clazz); - } catch (IOException e) { - throw new IllegalArgumentException(e); - } - } - - private static List listEntityPaths(String inputPath, JavaSparkContext sc) { - return HdfsSupport - .listFiles(inputPath, sc.hadoopConfiguration()) + ModelSupport.entityTypes + .entrySet() .stream() - .filter(f -> !f.toLowerCase().contains("relation")) - .collect(Collectors.toList()); - } + .map(e -> parPool.submit(() -> { + String entity = e.getKey().name(); + Class entityClass = e.getValue(); + spark + .read() + .load(checkpointPath) + .select(col(entity).as("value")) + .filter("value IS NOT NULL") + .as(OAFENTITY_KRYO_ENC) + .map((MapFunction) r -> r, (Encoder) Encoders.bean(entityClass)) + .filter(filterInvisible ? "dataInfo.invisible != TRUE" : "TRUE") + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(outputPath + "/" + entity); + })) + .collect(Collectors.toList()) + .forEach(t -> { + try { + t.get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + }); + } } diff --git a/dhp-common/src/main/resources/eu/dnetlib/dhp/oa/merge/dispatch_entities_parameters.json b/dhp-common/src/main/resources/eu/dnetlib/dhp/oa/merge/dispatch_entities_parameters.json deleted file mode 100644 index 60f11ac84..000000000 --- a/dhp-common/src/main/resources/eu/dnetlib/dhp/oa/merge/dispatch_entities_parameters.json +++ /dev/null @@ -1,26 +0,0 @@ -[ - { - "paramName": "issm", - "paramLongName": "isSparkSessionManaged", - "paramDescription": "when true will stop SparkSession after job execution", - "paramRequired": false - }, - { - "paramName": "i", - "paramLongName": "inputPath", - "paramDescription": "the source path", - "paramRequired": true - }, - { - "paramName": "o", - "paramLongName": "outputPath", - "paramDescription": "path of the output graph", - "paramRequired": true - }, - { - "paramName": "fi", - "paramLongName": "filterInvisible", - "paramDescription": "if true filters out invisible entities", - "paramRequired": true - } -] \ No newline at end of file diff --git a/dhp-common/src/main/resources/eu/dnetlib/dhp/oa/merge/group_graph_entities_parameters.json b/dhp-common/src/main/resources/eu/dnetlib/dhp/oa/merge/group_graph_entities_parameters.json index e65acb3c4..58e3ca711 100644 --- a/dhp-common/src/main/resources/eu/dnetlib/dhp/oa/merge/group_graph_entities_parameters.json +++ b/dhp-common/src/main/resources/eu/dnetlib/dhp/oa/merge/group_graph_entities_parameters.json @@ -8,13 +8,25 @@ { "paramName": "gin", "paramLongName": "graphInputPath", - "paramDescription": "the graph root path", + "paramDescription": "the input graph root path", + "paramRequired": true + }, + { + "paramName": "cp", + "paramLongName": "checkpointPath", + "paramDescription": "checkpoint directory", "paramRequired": true }, { "paramName": "out", "paramLongName": "outputPath", - "paramDescription": "the output merged graph root path", + "paramDescription": "the output graph root path", + "paramRequired": true + }, + { + "paramName": "fi", + "paramLongName": "filterInvisible", + "paramDescription": "if true filters out invisible entities", "paramRequired": true } ] \ No newline at end of file diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/workflow.xml b/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/workflow.xml index b724e5d0b..3640ee6ca 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/workflow.xml @@ -152,31 +152,7 @@ --conf spark.sql.shuffle.partitions=15000 --graphInputPath${graphBasePath} - --outputPath${workingPath}/grouped_entities - - - - - - - - yarn - cluster - Dispatch grouped entitities - eu.dnetlib.dhp.oa.merge.DispatchEntitiesSparkJob - dhp-dedup-openaire-${projectVersion}.jar - - --executor-memory=${sparkExecutorMemory} - --conf spark.executor.memoryOverhead=${sparkExecutorMemoryOverhead} - --executor-cores=${sparkExecutorCores} - --driver-memory=${sparkDriverMemory} - --conf spark.extraListeners=${spark2ExtraListeners} - --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} - --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} - --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.sql.shuffle.partitions=7680 - - --inputPath${workingPath}/grouped_entities + --checkpointPath${workingPath}/grouped_entities --outputPath${graphOutputPath} --filterInvisible${filterInvisible} diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/group/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/group/oozie_app/workflow.xml index 219dc7331..190788c9d 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/group/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/group/oozie_app/workflow.xml @@ -96,30 +96,7 @@ --conf spark.sql.shuffle.partitions=15000 --graphInputPath${graphBasePath} - --outputPath${workingPath}/grouped_entities - - - - - - - - yarn - cluster - Dispatch grouped entities - eu.dnetlib.dhp.oa.merge.DispatchEntitiesSparkJob - dhp-graph-mapper-${projectVersion}.jar - - --executor-cores=${sparkExecutorCores} - --executor-memory=${sparkExecutorMemory} - --driver-memory=${sparkDriverMemory} - --conf spark.extraListeners=${spark2ExtraListeners} - --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} - --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} - --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.sql.shuffle.partitions=7680 - - --inputPath${workingPath}/grouped_entities + --checkpointPath${workingPath}/grouped_entities --outputPath${graphOutputPath} --filterInvisible${filterInvisible} diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/group/GroupEntitiesSparkJobTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/group/GroupEntitiesSparkJobTest.java index 61baf80dc..b878e778e 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/group/GroupEntitiesSparkJobTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/group/GroupEntitiesSparkJobTest.java @@ -1,16 +1,15 @@ package eu.dnetlib.dhp.oa.graph.group; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import java.io.IOException; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; - +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import eu.dnetlib.dhp.common.HdfsSupport; +import eu.dnetlib.dhp.oa.merge.GroupEntitiesSparkJob; +import eu.dnetlib.dhp.schema.common.ModelSupport; +import eu.dnetlib.dhp.schema.oaf.OafEntity; +import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.dhp.utils.DHPUtils; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.function.FilterFunction; import org.apache.spark.api.java.function.MapFunction; @@ -19,118 +18,108 @@ import org.apache.spark.sql.Encoders; import org.apache.spark.sql.SparkSession; import org.junit.jupiter.api.*; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; -import eu.dnetlib.dhp.common.HdfsSupport; -import eu.dnetlib.dhp.oa.merge.DispatchEntitiesSparkJob; -import eu.dnetlib.dhp.oa.merge.GroupEntitiesSparkJob; -import eu.dnetlib.dhp.schema.common.ModelSupport; -import eu.dnetlib.dhp.schema.oaf.Result; -import eu.dnetlib.dhp.utils.DHPUtils; +import static org.junit.jupiter.api.Assertions.assertEquals; @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class GroupEntitiesSparkJobTest { - private static SparkSession spark; + private static SparkSession spark; - private static ObjectMapper mapper = new ObjectMapper() - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + private static ObjectMapper mapper = new ObjectMapper() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - private static Path workingDir; - private Path dataInputPath; + private static Path workingDir; + private Path dataInputPath; - private Path groupEntityPath; - private Path dispatchEntityPath; + private Path checkpointPath; - @BeforeAll - public static void beforeAll() throws IOException { - workingDir = Files.createTempDirectory(GroupEntitiesSparkJob.class.getSimpleName()); + private Path outputPath; - SparkConf conf = new SparkConf(); - conf.setAppName(GroupEntitiesSparkJob.class.getSimpleName()); - conf.setMaster("local"); - conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); - conf.registerKryoClasses(ModelSupport.getOafModelClasses()); - spark = SparkSession.builder().config(conf).getOrCreate(); - } + @BeforeAll + public static void beforeAll() throws IOException { + workingDir = Files.createTempDirectory(GroupEntitiesSparkJob.class.getSimpleName()); - @BeforeEach - public void beforeEach() throws IOException, URISyntaxException { - dataInputPath = Paths.get(ClassLoader.getSystemResource("eu/dnetlib/dhp/oa/graph/group").toURI()); - groupEntityPath = workingDir.resolve("grouped_entity"); - dispatchEntityPath = workingDir.resolve("dispatched_entity"); - } + SparkConf conf = new SparkConf(); + conf.setAppName(GroupEntitiesSparkJob.class.getSimpleName()); + conf.setMaster("local"); + conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); + conf.registerKryoClasses(ModelSupport.getOafModelClasses()); + spark = SparkSession.builder().config(conf).getOrCreate(); + } - @AfterAll - public static void afterAll() throws IOException { - spark.stop(); - FileUtils.deleteDirectory(workingDir.toFile()); - } + @BeforeEach + public void beforeEach() throws IOException, URISyntaxException { + dataInputPath = Paths.get(ClassLoader.getSystemResource("eu/dnetlib/dhp/oa/graph/group").toURI()); + checkpointPath = workingDir.resolve("grouped_entity"); + outputPath = workingDir.resolve("dispatched_entity"); + } - @Test - @Order(1) - void testGroupEntities() throws Exception { - GroupEntitiesSparkJob.main(new String[] { - "-isSparkSessionManaged", - Boolean.FALSE.toString(), - "-graphInputPath", - dataInputPath.toString(), - "-outputPath", - groupEntityPath.toString() - }); + @AfterAll + public static void afterAll() throws IOException { + spark.stop(); + FileUtils.deleteDirectory(workingDir.toFile()); + } - Dataset output = spark - .read() - .textFile(groupEntityPath.toString()) - .map((MapFunction) s -> StringUtils.substringAfter(s, "|"), Encoders.STRING()) - .map((MapFunction) s -> mapper.readValue(s, Result.class), Encoders.bean(Result.class)); + @Test + @Order(1) + void testGroupEntities() throws Exception { + GroupEntitiesSparkJob.main(new String[]{ + "-isSparkSessionManaged", + Boolean.FALSE.toString(), + "-graphInputPath", + dataInputPath.toString(), + "-checkpointPath", + checkpointPath.toString(), + "-outputPath", + outputPath.toString(), + "-filterInvisible", + Boolean.FALSE.toString() + }); - assertEquals( - 1, - output - .filter( - (FilterFunction) r -> "50|doi_________::09821844208a5cd6300b2bfb13bca1b9" - .equals(r.getId()) && - r.getCollectedfrom().stream().anyMatch(kv -> kv.getValue().equalsIgnoreCase("zenodo"))) - .count()); - } + Dataset checkpointTable = spark + .read() + .load(checkpointPath.toString()) + .selectExpr("COALESCE(*)") + .as(Encoders.kryo(OafEntity.class)); - @Test - @Order(2) - void testDispatchEntities() throws Exception { - DispatchEntitiesSparkJob.main(new String[] { - "-isSparkSessionManaged", - Boolean.FALSE.toString(), - "-inputPath", - groupEntityPath.toString(), - "-outputPath", - dispatchEntityPath.resolve(".").toString(), - "-filterInvisible", - Boolean.TRUE.toString() - }); - Dataset output = spark - .read() - .textFile( - DHPUtils - .toSeq( - HdfsSupport - .listFiles(dispatchEntityPath.toString(), spark.sparkContext().hadoopConfiguration()))) - .map((MapFunction) s -> mapper.readValue(s, Result.class), Encoders.bean(Result.class)); + assertEquals( + 1, + checkpointTable + .filter( + (FilterFunction) r -> "50|doi_________::09821844208a5cd6300b2bfb13bca1b9" + .equals(r.getId()) && + r.getCollectedfrom().stream().anyMatch(kv -> kv.getValue().equalsIgnoreCase("zenodo"))) + .count()); - assertEquals(3, output.count()); - assertEquals( - 2, - output - .map((MapFunction) r -> r.getResulttype().getClassid(), Encoders.STRING()) - .filter((FilterFunction) s -> s.equals("publication")) - .count()); - assertEquals( - 1, - output - .map((MapFunction) r -> r.getResulttype().getClassid(), Encoders.STRING()) - .filter((FilterFunction) s -> s.equals("dataset")) - .count()); - } -} + + Dataset output = spark + .read() + .textFile( + DHPUtils + .toSeq( + HdfsSupport + .listFiles(outputPath.toString(), spark.sparkContext().hadoopConfiguration()))) + .map((MapFunction) s -> mapper.readValue(s, Result.class), Encoders.bean(Result.class)); + + assertEquals(3, output.count()); + assertEquals( + 2, + output + .map((MapFunction) r -> r.getResulttype().getClassid(), Encoders.STRING()) + .filter((FilterFunction) s -> s.equals("publication")) + .count()); + assertEquals( + 1, + output + .map((MapFunction) r -> r.getResulttype().getClassid(), Encoders.STRING()) + .filter((FilterFunction) s -> s.equals("dataset")) + .count()); + } +} \ No newline at end of file From 964c2f553e43438cedbfe44d2b1ae5e4d4d3d4f6 Mon Sep 17 00:00:00 2001 From: dimitrispie Date: Fri, 1 Sep 2023 10:57:02 +0300 Subject: [PATCH 008/148] Changes in indicators step, monitor step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - graduatedoctorates for observatory - result_apc_affiliations table - new indicators indi_is_funder_plan_s indi_funder_fairness indi_ris_fairness indi_funder_openess indi_ris_openess indi_funder_findable indi_ris_findable indi_is_project_result_after - cast year to int in composite indicators - new institutions -- Universidade Católica Portuguesa -- Iscte - Instituto Universitário de Lisboa -- Munster Technological University -- Cardiff University -- Leibniz Institute of Ecological Urban and Regional Development --- .../dhp/oa/graph/stats/oozie_app/monitor.sh | 2 +- .../graph/stats/oozie_app/scripts/step13.sql | 0 .../graph/stats/oozie_app/scripts/step15.sql | 2 +- .../stats/oozie_app/scripts/step15_5.sql | 1 + .../scripts/step16-createIndicatorsTables.sql | 358 ++++++++++++++---- .../scripts/step20-createMonitorDB.sql | 9 + .../scripts/step20-createMonitorDBAll.sql | 18 +- .../step20-createMonitorDB_institutions.sql | 25 +- .../graph/stats/oozie_app/scripts/step5.sql | 0 .../dhp/oa/graph/stats/oozie_app/workflow.xml | 2 +- 10 files changed, 324 insertions(+), 93 deletions(-) mode change 100644 => 100755 dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/monitor.sh mode change 100644 => 100755 dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step13.sql mode change 100644 => 100755 dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step5.sql diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/monitor.sh b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/monitor.sh old mode 100644 new mode 100755 index 014b19c6c..872456973 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/monitor.sh +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/monitor.sh @@ -39,7 +39,7 @@ hdfs dfs -copyToLocal $9 echo "Creating monitor database" -cat step20-createMonitorDBAll.sql | sed "s/SOURCE/openaire_prod_stats_20230707/g" | sed "s/TARGET/openaire_prod_stats_monitor_20230707/g1" > foo +cat step20-createMonitorDBAll.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2/g1" > foo hive $HIVE_OPTS -f foo cat step20-createMonitorDB_funded.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2_funded/g1" > foo diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step13.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step13.sql old mode 100644 new mode 100755 diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql index d1cbde438..4a8f81943 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql @@ -48,4 +48,4 @@ rel.properties[1].value apc_currency from ${openaire_db_name}.relation rel join ${openaire_db_name}.organization o on o.id=rel.source join ${openaire_db_name}.result r on r.id=rel.target -where rel.subreltype = 'affiliation' and rel.datainfo.deletedbyinference = false and size(rel.properties) > 0; +where rel.subreltype = 'affiliation' and rel.datainfo.deletedbyinference = false and size(rel.properties)>0; diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15_5.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15_5.sql index f39ff2afd..615f523ce 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15_5.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15_5.sql @@ -35,6 +35,7 @@ create or replace view ${stats_db_name}.doctoratestudents as select * from stats create or replace view ${stats_db_name}.totalresearchers as select * from stats_ext.totalresearchers; create or replace view ${stats_db_name}.totalresearchersft as select * from stats_ext.totalresearchersft; create or replace view ${stats_db_name}.hrrst as select * from stats_ext.hrrst; +create or replace view ${stats_db_name}.graduatedoctorates as select * from stats_ext.graduatedoctorates; create table if not exists ${stats_db_name}.result_instance stored as parquet as select distinct r.* diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql index 57c381875..1c80f6757 100755 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql @@ -12,8 +12,6 @@ from ${stats_db_name}.publication p or ri.accessright = 'Embargo' or ri.accessright = 'Open Source')) tmp on p.id= tmp.id; ---ANALYZE TABLE ${stats_db_name}.indi_pub_green_oa COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_pub_grey_lit stored as parquet as select distinct p.id, coalesce(grey_lit, 0) as grey_lit from ${stats_db_name}.publication p @@ -25,8 +23,6 @@ from ${stats_db_name}.publication p not exists (select 1 from ${stats_db_name}.result_classifications rc where type ='Other literature type' and rc.id=p.id)) tmp on p.id=tmp.id; ---ANALYZE TABLE ${stats_db_name}.indi_pub_grey_lit COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_pub_doi_from_crossref stored as parquet as select distinct p.id, coalesce(doi_from_crossref, 0) as doi_from_crossref from ${stats_db_name}.publication p @@ -36,8 +32,6 @@ from ${stats_db_name}.publication p where pidtype='Digital Object Identifier' and d.name ='Crossref') tmp on tmp.id=p.id; ---ANALYZE TABLE ${stats_db_name}.indi_pub_doi_from_crossref COMPUTE STATISTICS; - -- Sprint 2 ---- create table if not exists ${stats_db_name}.indi_result_has_cc_licence stored as parquet as select distinct r.id, (case when lic='' or lic is null then 0 else 1 end) as has_cc_license @@ -47,8 +41,6 @@ left outer join (select r.id, license.type as lic from ${stats_db_name}.result r where lower(license.type) LIKE '%creativecommons.org%' OR lower(license.type) LIKE '%cc-%') tmp on r.id= tmp.id; ---ANALYZE TABLE ${stats_db_name}.indi_result_has_cc_licence COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_result_has_cc_licence_url stored as parquet as select distinct r.id, case when lic_host='' or lic_host is null then 0 else 1 end as has_cc_license_url from ${stats_db_name}.result r @@ -58,22 +50,16 @@ from ${stats_db_name}.result r WHERE lower(parse_url(license.type, "HOST")) = "creativecommons.org") tmp on r.id= tmp.id; ---ANALYZE TABLE ${stats_db_name}.indi_result_has_cc_licence_url COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_pub_has_abstract stored as parquet as select distinct publication.id, cast(coalesce(abstract, true) as int) has_abstract from ${stats_db_name}.publication; ---ANALYZE TABLE ${stats_db_name}.indi_pub_has_abstract COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_result_with_orcid stored as parquet as select distinct r.id, coalesce(has_orcid, 0) as has_orcid from ${stats_db_name}.result r left outer join (select id, 1 as has_orcid from ${stats_db_name}.result_orcid) tmp on r.id= tmp.id; ---ANALYZE TABLE ${stats_db_name}.indi_result_with_orcid COMPUTE STATISTICS; - ---- Sprint 3 ---- create table if not exists ${stats_db_name}.indi_funded_result_with_fundref stored as parquet as select distinct r.result as id, coalesce(fundref, 0) as fundref @@ -82,8 +68,6 @@ from ${stats_db_name}.project_results r where provenance='Harvested') tmp on r.result= tmp.result; ---ANALYZE TABLE ${stats_db_name}.indi_funded_result_with_fundref COMPUTE STATISTICS; - -- create table indi_result_org_collab stored as parquet as -- select o1.organization org1, o2.organization org2, count(distinct o1.id) as collaborations -- from result_organization as o1 @@ -103,8 +87,6 @@ group by o1.organization, o2.organization, o1.name, o2.name; drop table ${stats_db_name}.tmp purge; ---ANALYZE TABLE ${stats_db_name}.indi_result_org_collab COMPUTE STATISTICS; - create TEMPORARY TABLE ${stats_db_name}.tmp AS select distinct ro.organization organization, ro.id, o.name, o.country from ${stats_db_name}.result_organization ro join ${stats_db_name}.organization o on o.id=ro.organization where country <> 'UNKNOWN' and o.name is not null; @@ -117,8 +99,6 @@ group by o1.organization, o1.id, o1.name, o2.country; drop table ${stats_db_name}.tmp purge; ---ANALYZE TABLE ${stats_db_name}.indi_result_org_country_collab COMPUTE STATISTICS; - create TEMPORARY TABLE ${stats_db_name}.tmp AS select o.id organization, o.name, ro.project as project from ${stats_db_name}.organization o join ${stats_db_name}.organization_projects ro on o.id=ro.id where o.name is not null; @@ -132,8 +112,6 @@ group by o1.name,o2.name, o1.organization, o2.organization; drop table ${stats_db_name}.tmp purge; ---ANALYZE TABLE ${stats_db_name}.indi_project_collab_org COMPUTE STATISTICS; - create TEMPORARY TABLE ${stats_db_name}.tmp AS select o.id organization, o.name, o.country , ro.project as project from ${stats_db_name}.organization o join ${stats_db_name}.organization_projects ro on o.id=ro.id @@ -148,8 +126,6 @@ group by o1.organization, o2.country, o1.name; drop table ${stats_db_name}.tmp purge; ---ANALYZE TABLE ${stats_db_name}.indi_project_collab_org_country COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_funder_country_collab stored as parquet as with tmp as (select funder, project, country from ${stats_db_name}.organization_projects op join ${stats_db_name}.organization o on o.id=op.id @@ -161,8 +137,6 @@ from tmp as f1 where f1.country<>f2.country group by f1.funder, f2.country, f1.country; ---ANALYZE TABLE ${stats_db_name}.indi_funder_country_collab COMPUTE STATISTICS; - create TEMPORARY TABLE ${stats_db_name}.tmp AS select distinct country, ro.id as result from ${stats_db_name}.organization o join ${stats_db_name}.result_organization ro on o.id=ro.organization @@ -177,8 +151,6 @@ group by o1.country, o2.country; drop table ${stats_db_name}.tmp purge; ---ANALYZE TABLE ${stats_db_name}.indi_result_country_collab COMPUTE STATISTICS; - ---- Sprint 4 ---- create table if not exists ${stats_db_name}.indi_pub_diamond stored as parquet as select distinct pd.id, coalesce(in_diamond_journal, 0) as in_diamond_journal @@ -190,8 +162,6 @@ from ${stats_db_name}.publication_datasources pd and (ps.journal_is_in_doaj=true or ps.journal_is_oa=true) and ps.has_apc=false) tmp on pd.id=tmp.id; -----ANALYZE TABLE ${stats_db_name}.indi_pub_diamond COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_pub_in_transformative stored as parquet as select distinct pd.id, coalesce(is_transformative, 0) as is_transformative from ${stats_db_name}.publication pd @@ -202,8 +172,6 @@ from ${stats_db_name}.publication pd and ps.is_transformative_journal=true) tmp on pd.id=tmp.id; ---ANALYZE TABLE ${stats_db_name}.indi_pub_in_transformative COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_pub_closed_other_open stored as parquet as select distinct ri.id, coalesce(pub_closed_other_open, 0) as pub_closed_other_open from ${stats_db_name}.result_instance ri left outer join @@ -214,14 +182,10 @@ select distinct ri.id, coalesce(pub_closed_other_open, 0) as pub_closed_other_op (p.bestlicence='Open Access' or p.bestlicence='Open Source')) tmp on tmp.id=ri.id; ---ANALYZE TABLE ${stats_db_name}.indi_pub_closed_other_open COMPUTE STATISTICS; - ---- Sprint 5 ---- create table if not exists ${stats_db_name}.indi_result_no_of_copies stored as parquet as select id, count(id) as number_of_copies from ${stats_db_name}.result_instance group by id; ---ANALYZE TABLE ${stats_db_name}.indi_result_no_of_copies COMPUTE STATISTICS; - ---- Sprint 6 ---- create table if not exists ${stats_db_name}.indi_pub_downloads stored as parquet as SELECT result_id, sum(downloads) no_downloads from openaire_prod_usage_stats.usage_stats @@ -239,24 +203,18 @@ where downloads>0 GROUP BY result_id, repository_id order by result_id; ---ANALYZE TABLE ${stats_db_name}.indi_pub_downloads_datasource COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_pub_downloads_year stored as parquet as -SELECT result_id, substring(us.`date`, 1,4) as `year`, sum(downloads) no_downloads +SELECT result_id, cast(substring(us.`date`, 1,4) as int) as `year`, sum(downloads) no_downloads from openaire_prod_usage_stats.usage_stats us join ${stats_db_name}.publication on result_id=id where downloads>0 GROUP BY result_id, substring(us.`date`, 1,4); ---ANALYZE TABLE ${stats_db_name}.indi_pub_downloads_year COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_pub_downloads_datasource_year stored as parquet as -SELECT result_id, substring(us.`date`, 1,4) as `year`, repository_id, sum(downloads) no_downloads from openaire_prod_usage_stats.usage_stats us +SELECT result_id, cast(substring(us.`date`, 1,4) as int) as `year`, repository_id, sum(downloads) no_downloads from openaire_prod_usage_stats.usage_stats us join ${stats_db_name}.publication on result_id=id where downloads>0 GROUP BY result_id, repository_id, substring(us.`date`, 1,4); ---ANALYZE TABLE ${stats_db_name}.indi_pub_downloads_datasource_year COMPUTE STATISTICS; - ---- Sprint 7 ---- create table if not exists ${stats_db_name}.indi_pub_gold_oa stored as parquet as WITH gold_oa AS ( SELECT @@ -307,8 +265,6 @@ FROM JOIN gold_oa on issn.issn = gold_oa.issn) tmp on pd.id=tmp.id; ---ANALYZE TABLE ${stats_db_name}.indi_pub_gold_oa COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_pub_hybrid_oa_with_cc stored as parquet as WITH hybrid_oa AS ( SELECT issn_l, journal_is_in_doaj, journal_is_oa, issn_print as issn @@ -340,8 +296,6 @@ FROM ${stats_db_name}.publication_datasources pd JOIN ${stats_db_name}.indi_pub_gold_oa ga on pd.id=ga.id where cc.has_cc_license=1 and ga.is_gold=0) tmp on pd.id=tmp.id; ---ANALYZE TABLE ${stats_db_name}.indi_pub_hybrid_oa_with_cc COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_pub_hybrid stored as parquet as WITH gold_oa AS ( SELECT issn_l, @@ -393,8 +347,6 @@ from ${stats_db_name}.publication_datasources pd where (gold_oa.journal_is_in_doaj=false or gold_oa.journal_is_oa=false))tmp on pd.id=tmp.id; ---ANALYZE TABLE ${stats_db_name}.indi_pub_hybrid COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_org_fairness stored as parquet as --return results with PIDs, and rich metadata group by organization with result_fair as @@ -413,8 +365,6 @@ select allresults.organization, result_fair.no_result_fair/allresults.no_allresu from allresults join result_fair on result_fair.organization=allresults.organization; ---ANALYZE TABLE ${stats_db_name}.indi_org_fairness COMPUTE STATISTICS; - CREATE TEMPORARY table ${stats_db_name}.result_fair as select ro.organization organization, count(distinct ro.id) no_result_fair from ${stats_db_name}.result_organization ro @@ -439,8 +389,6 @@ from ${stats_db_name}.allresults ar DROP table ${stats_db_name}.result_fair purge; DROP table ${stats_db_name}.allresults purge; ---ANALYZE TABLE ${stats_db_name}.indi_org_fairness_pub_pr COMPUTE STATISTICS; - CREATE TEMPORARY table ${stats_db_name}.result_fair as select year, ro.organization organization, count(distinct ro.id) no_result_fair from ${stats_db_name}.result_organization ro join ${stats_db_name}.result p on p.id=ro.id @@ -460,8 +408,6 @@ from ${stats_db_name}.allresults DROP table ${stats_db_name}.result_fair purge; DROP table ${stats_db_name}.allresults purge; ---ANALYZE TABLE ${stats_db_name}.indi_org_fairness_pub_year COMPUTE STATISTICS; - CREATE TEMPORARY TABLE ${stats_db_name}.result_fair as select ro.organization organization, count(distinct ro.id) no_result_fair from ${stats_db_name}.result_organization ro @@ -484,8 +430,6 @@ on rf.organization=ar.organization; DROP table ${stats_db_name}.result_fair purge; DROP table ${stats_db_name}.allresults purge; ---ANALYZE TABLE ${stats_db_name}.indi_org_fairness_pub COMPUTE STATISTICS; - CREATE TEMPORARY TABLE ${stats_db_name}.result_fair as select year, ro.organization organization, count(distinct ro.id) no_result_fair from ${stats_db_name}.result_organization ro join ${stats_db_name}.result r on r.id=ro.id @@ -507,8 +451,6 @@ create table if not exists ${stats_db_name}.indi_org_fairness_year stored as par DROP table ${stats_db_name}.result_fair purge; DROP table ${stats_db_name}.allresults purge; ---ANALYZE TABLE ${stats_db_name}.indi_org_fairness_year COMPUTE STATISTICS; - CREATE TEMPORARY TABLE ${stats_db_name}.result_with_pid as select year, ro.organization, count(distinct rp.id) no_result_with_pid from ${stats_db_name}.result_organization ro join ${stats_db_name}.result_pids rp on rp.id=ro.id @@ -530,8 +472,6 @@ from ${stats_db_name}.allresults DROP table ${stats_db_name}.result_with_pid purge; DROP table ${stats_db_name}.allresults purge; ---ANALYZE TABLE ${stats_db_name}.indi_org_findable_year COMPUTE STATISTICS; - CREATE TEMPORARY TABLE ${stats_db_name}.result_with_pid as select ro.organization, count(distinct rp.id) no_result_with_pid from ${stats_db_name}.result_organization ro join ${stats_db_name}.result_pids rp on rp.id=ro.id @@ -553,8 +493,6 @@ from ${stats_db_name}.allresults DROP table ${stats_db_name}.result_with_pid purge; DROP table ${stats_db_name}.allresults purge; ---ANALYZE TABLE ${stats_db_name}.indi_org_findable COMPUTE STATISTICS; - CREATE TEMPORARY TABLE ${stats_db_name}.pubs_oa as SELECT ro.organization, count(distinct r.id) no_oapubs FROM ${stats_db_name}.publication r join ${stats_db_name}.result_organization ro on ro.id=r.id @@ -633,8 +571,6 @@ DROP TABLE ${stats_db_name}.allpubsshare purge; DROP TABLE ${stats_db_name}.alldatasetssshare purge; DROP TABLE ${stats_db_name}.allsoftwaresshare purge; ---ANALYZE TABLE ${stats_db_name}.indi_org_openess COMPUTE STATISTICS; - CREATE TEMPORARY TABLE ${stats_db_name}.pubs_oa AS SELECT r.year, ro.organization, count(distinct r.id) no_oapubs FROM ${stats_db_name}.publication r join ${stats_db_name}.result_organization ro on ro.id=r.id @@ -690,7 +626,7 @@ select allsoftware.year, software_oa.organization, software_oa.no_oasoftware/all create table if not exists ${stats_db_name}.indi_org_openess_year stored as parquet as -select allpubsshare.year, allpubsshare.organization, +select cast(allpubsshare.year as int), allpubsshare.organization, (p+if(isnull(s),0,s)+if(isnull(d),0,d))/(1+(case when s is null then 0 else 1 end) +(case when d is null then 0 else 1 end)) org_openess FROM ${stats_db_name}.allpubsshare @@ -711,8 +647,6 @@ DROP TABLE ${stats_db_name}.allpubsshare purge; DROP TABLE ${stats_db_name}.alldatasetssshare purge; DROP TABLE ${stats_db_name}.allsoftwaresshare purge; ---ANALYZE TABLE ${stats_db_name}.indi_org_openess_year COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_pub_has_preprint stored as parquet as select distinct p.id, coalesce(has_preprint, 0) as has_preprint from ${stats_db_name}.publication_classifications p @@ -722,8 +656,6 @@ from ${stats_db_name}.publication_classifications p where p.type='Preprint') tmp on p.id= tmp.id; ---ANALYZE TABLE ${stats_db_name}.indi_pub_has_preprint COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_pub_in_subscribed stored as parquet as select distinct p.id, coalesce(is_subscription, 0) as is_subscription from ${stats_db_name}.publication p @@ -735,8 +667,6 @@ from ${stats_db_name}.publication p where g.is_gold=0 and h.is_hybrid=0 and t.is_transformative=0) tmp on p.id=tmp.id; ---ANALYZE TABLE ${stats_db_name}.indi_pub_in_subscribed COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_result_with_pid as select distinct p.id, coalesce(result_with_pid, 0) as result_with_pid from ${stats_db_name}.result p @@ -745,8 +675,6 @@ from ${stats_db_name}.result p from ${stats_db_name}.result_pids p) tmp on p.id= tmp.id; ---ANALYZE TABLE ${stats_db_name}.indi_result_with_pid COMPUTE STATISTICS; - CREATE TEMPORARY TABLE ${stats_db_name}.pub_fos_totals as select rf.id, count(distinct lvl3) totals from ${stats_db_name}.result_fos rf group by rf.id; @@ -761,8 +689,6 @@ where totals>1) tmp on p.id=tmp.id; drop table ${stats_db_name}.pub_fos_totals purge; ---ANALYZE TABLE ${stats_db_name}.indi_pub_interdisciplinarity COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_pub_bronze_oa stored as parquet as select distinct p.id, coalesce(is_bronze_oa,0) as is_bronze_oa from ${stats_db_name}.publication p @@ -772,4 +698,280 @@ join ${stats_db_name}.indi_result_has_cc_licence cc on cc.id=p.id join ${stats_db_name}.indi_pub_gold_oa ga on ga.id=p.id where cc.has_cc_license=0 and ga.is_gold=0) tmp on tmp.id=p.id; ---ANALYZE TABLE ${stats_db_name}.indi_pub_bronze_oa COMPUTE STATISTICS; \ No newline at end of file +CREATE TEMPORARY TABLE ${stats_db_name}.project_year_result_year as +select p.id project_id, acronym, r.id result_id, r.year, p.end_year +from ${stats_db_name}.project p +join ${stats_db_name}.result_projects rp on p.id=rp.project +join ${stats_db_name}.result r on r.id=rp.id +where p.end_year is NOT NULL and r.year is not null; + +create table if not exists ${stats_db_name}.indi_is_project_result_after stored as parquet as +select pry.project_id, pry.acronym, pry.result_id, +coalesce(is_project_result_after, 0) as is_project_result_after +from ${stats_db_name}.project_year_result_year pry +left outer join (select pry.project_id, pry.acronym, pry.result_id, 1 as is_project_result_after +from ${stats_db_name}.project_year_result_year pry +where pry.year>pry.end_year) tmp on pry.result_id=tmp.result_id; + +drop table ${stats_db_name}.project_year_result_year purge; + +create table if not exists ${stats_db_name}.indi_is_funder_plan_s stored as parquet as +select distinct f.id, f.name, coalesce(is_funder_plan_s, 0) as is_funder_plan_s +from ${stats_db_name}.funder f + left outer join (select id, name, 1 as is_funder_plan_s from ${stats_db_name}.funder + join stats_ext.plan_s_short on c_o_alition_s_organisation_funder=name) tmp + on f.name= tmp.name; + +--Funder Fairness + +create table if not exists ${stats_db_name}.indi_funder_fairness stored as parquet as + with result_fair as + (select p.funder funder, count(distinct rp.id) no_result_fair from ${stats_db_name}.result_projects rp + join ${stats_db_name}.result r on r.id=rp.id + join ${stats_db_name}.project p on p.id=rp.project + where (r.title is not null) and (publisher is not null) and (abstract=true) and (year is not null) and (authors>0) and cast(year as int)>2003 + group by p.funder), + allresults as (select p.funder funder, count(distinct rp.id) no_allresults from ${stats_db_name}.result_projects rp + join ${stats_db_name}.result r on r.id=rp.id + join ${stats_db_name}.project p on p.id=rp.project + where cast(year as int)>2003 + group by p.funder) +select allresults.funder, result_fair.no_result_fair/allresults.no_allresults funder_fairness +from allresults + join result_fair on result_fair.funder=allresults.funder; + +--RIs Fairness +create table if not exists ${stats_db_name}.indi_ris_fairness stored as parquet as +with result_contexts as +(select distinct rc.id, context.name ri_initiative from ${stats_db_name}.result_concepts rc +join ${stats_db_name}.concept on concept.id=rc.concept +join ${stats_db_name}.category on category.id=concept.category +join ${stats_db_name}.context on context.id=category.context), +result_fair as + (select rc.ri_initiative ri_initiative, count(distinct rc.id) no_result_fair from result_contexts rc + join ${stats_db_name}.result r on r.id=rc.id + where (title is not null) and (publisher is not null) and (abstract=true) and (year is not null) and (authors>0) and cast(year as int)>2003 + group by rc.ri_initiative), +allresults as +(select rc.ri_initiative ri_initiative, count(distinct rc.id) no_allresults from result_contexts rc + join ${stats_db_name}.result r on r.id=rc.id + where cast(year as int)>2003 + group by rc.ri_initiative) +select allresults.ri_initiative, result_fair.no_result_fair/allresults.no_allresults ris_fairness +from allresults + join result_fair on result_fair.ri_initiative=allresults.ri_initiative; + +--Funder Openess + +CREATE TEMPORARY TABLE ${stats_db_name}.pubs_oa as +select p.funder funder, count(distinct rp.id) no_oapubs from ${stats_db_name}.result_projects rp +join ${stats_db_name}.project p on p.id=rp.project +join ${stats_db_name}.publication r on r.id=rp.id +join ${stats_db_name}.result_instance ri on ri.id=r.id +where (ri.accessright = 'Open Access' or ri.accessright = 'Embargo' or ri.accessright = 'Open Source') +and cast(r.year as int)>2003 +group by p.funder; + + +CREATE TEMPORARY TABLE ${stats_db_name}.datasets_oa as +select p.funder funder, count(distinct rp.id) no_oadatasets from ${stats_db_name}.result_projects rp +join ${stats_db_name}.project p on p.id=rp.project +join ${stats_db_name}.dataset r on r.id=rp.id +join ${stats_db_name}.result_instance ri on ri.id=r.id +where (ri.accessright = 'Open Access' or ri.accessright = 'Embargo' or ri.accessright = 'Open Source') +and cast(r.year as int)>2003 +group by p.funder; + +CREATE TEMPORARY TABLE ${stats_db_name}.software_oa as +select p.funder funder, count(distinct rp.id) no_oasoftware from ${stats_db_name}.result_projects rp +join ${stats_db_name}.project p on p.id=rp.project +join ${stats_db_name}.software r on r.id=rp.id +join ${stats_db_name}.result_instance ri on ri.id=r.id +where (ri.accessright = 'Open Access' or ri.accessright = 'Embargo' or ri.accessright = 'Open Source') +and cast(r.year as int)>2003 +group by p.funder; + +CREATE TEMPORARY TABLE ${stats_db_name}.allpubs as +select p.funder funder, count(distinct rp.id) no_allpubs from ${stats_db_name}.result_projects rp +join ${stats_db_name}.project p on p.id=rp.project +join ${stats_db_name}.publication r on r.id=rp.id +where cast(r.year as int)>2003 +group by p.funder; + +CREATE TEMPORARY TABLE ${stats_db_name}.alldatasets as +select p.funder funder, count(distinct rp.id) no_alldatasets from ${stats_db_name}.result_projects rp +join ${stats_db_name}.project p on p.id=rp.project +join ${stats_db_name}.dataset r on r.id=rp.id +where cast(r.year as int)>2003 +group by p.funder; + +CREATE TEMPORARY TABLE ${stats_db_name}.allsoftware as +select p.funder funder, count(distinct rp.id) no_allsoftware from ${stats_db_name}.result_projects rp +join ${stats_db_name}.project p on p.id=rp.project +join ${stats_db_name}.software r on r.id=rp.id +where cast(r.year as int)>2003 +group by p.funder; + +CREATE TEMPORARY TABLE ${stats_db_name}.allpubsshare as +select pubs_oa.funder, pubs_oa.no_oapubs/allpubs.no_allpubs p from ${stats_db_name}.allpubs + join ${stats_db_name}.pubs_oa on allpubs.funder=pubs_oa.funder; + +CREATE TEMPORARY TABLE ${stats_db_name}.alldatasetssshare as +select datasets_oa.funder, datasets_oa.no_oadatasets/alldatasets.no_alldatasets d + from ${stats_db_name}.alldatasets + join ${stats_db_name}.datasets_oa on alldatasets.funder=datasets_oa.funder; + +CREATE TEMPORARY TABLE ${stats_db_name}.allsoftwaresshare as +select software_oa.funder, software_oa.no_oasoftware/allsoftware.no_allsoftware s + from ${stats_db_name}.allsoftware + join ${stats_db_name}.software_oa on allsoftware.funder=software_oa.funder; + +create table if not exists ${stats_db_name}.indi_funder_openess stored as parquet as +select allpubsshare.funder, + (p+if(isnull(s),0,s)+if(isnull(d),0,d))/(1+(case when s is null then 0 else 1 end) + +(case when d is null then 0 else 1 end)) + funder_openess FROM ${stats_db_name}.allpubsshare + left outer join (select funder,d from + ${stats_db_name}.alldatasetssshare) tmp1 + on tmp1.funder=allpubsshare.funder + left outer join (select funder,s from + ${stats_db_name}.allsoftwaresshare) tmp2 + on tmp2.funder=allpubsshare.funder; + +DROP TABLE ${stats_db_name}.pubs_oa purge; +DROP TABLE ${stats_db_name}.datasets_oa purge; +DROP TABLE ${stats_db_name}.software_oa purge; +DROP TABLE ${stats_db_name}.allpubs purge; +DROP TABLE ${stats_db_name}.alldatasets purge; +DROP TABLE ${stats_db_name}.allsoftware purge; +DROP TABLE ${stats_db_name}.allpubsshare purge; +DROP TABLE ${stats_db_name}.alldatasetssshare purge; +DROP TABLE ${stats_db_name}.allsoftwaresshare purge; + +--RIs Openess + +CREATE TEMPORARY TABLE ${stats_db_name}.result_contexts as +select distinct rc.id, context.name ri_initiative from ${stats_db_name}.result_concepts rc +join ${stats_db_name}.concept on concept.id=rc.concept +join ${stats_db_name}.category on category.id=concept.category +join ${stats_db_name}.context on context.id=category.context; + +CREATE TEMPORARY TABLE ${stats_db_name}.pubs_oa as +select rp.ri_initiative ri_initiative, count(distinct rp.id) no_oapubs from ${stats_db_name}.result_contexts rp +join ${stats_db_name}.publication r on r.id=rp.id +join ${stats_db_name}.result_instance ri on ri.id=r.id +where (ri.accessright = 'Open Access' or ri.accessright = 'Embargo' or ri.accessright = 'Open Source') +and cast(r.year as int)>2003 +group by rp.ri_initiative; + +CREATE TEMPORARY TABLE ${stats_db_name}.datasets_oa as +select rp.ri_initiative ri_initiative, count(distinct rp.id) no_oadatasets from ${stats_db_name}.result_contexts rp +join ${stats_db_name}.dataset r on r.id=rp.id +join ${stats_db_name}.result_instance ri on ri.id=r.id +where (ri.accessright = 'Open Access' or ri.accessright = 'Embargo' or ri.accessright = 'Open Source') +and cast(r.year as int)>2003 +group by rp.ri_initiative; + +CREATE TEMPORARY TABLE ${stats_db_name}.software_oa as +select rp.ri_initiative ri_initiative, count(distinct rp.id) no_oasoftware from ${stats_db_name}.result_contexts rp +join ${stats_db_name}.software r on r.id=rp.id +join ${stats_db_name}.result_instance ri on ri.id=r.id +where (ri.accessright = 'Open Access' or ri.accessright = 'Embargo' or ri.accessright = 'Open Source') +and cast(r.year as int)>2003 +group by rp.ri_initiative; + +CREATE TEMPORARY TABLE ${stats_db_name}.allpubs as +select rp.ri_initiative ri_initiative, count(distinct rp.id) no_allpubs from ${stats_db_name}.result_contexts rp +join ${stats_db_name}.publication r on r.id=rp.id +where cast(r.year as int)>2003 +group by rp.ri_initiative; + +CREATE TEMPORARY TABLE ${stats_db_name}.alldatasets as +select rp.ri_initiative ri_initiative, count(distinct rp.id) no_alldatasets from ${stats_db_name}.result_contexts rp +join ${stats_db_name}.dataset r on r.id=rp.id +where cast(r.year as int)>2003 +group by rp.ri_initiative; + +CREATE TEMPORARY TABLE ${stats_db_name}.allsoftware as +select rp.ri_initiative ri_initiative, count(distinct rp.id) no_allsoftware from ${stats_db_name}.result_contexts rp +join ${stats_db_name}.software r on r.id=rp.id +where cast(r.year as int)>2003 +group by rp.ri_initiative; + +CREATE TEMPORARY TABLE ${stats_db_name}.allpubsshare as +select pubs_oa.ri_initiative, pubs_oa.no_oapubs/allpubs.no_allpubs p from ${stats_db_name}.allpubs + join ${stats_db_name}.pubs_oa on allpubs.ri_initiative=pubs_oa.ri_initiative; + +CREATE TEMPORARY TABLE ${stats_db_name}.alldatasetssshare as +select datasets_oa.ri_initiative, datasets_oa.no_oadatasets/alldatasets.no_alldatasets d + from ${stats_db_name}.alldatasets + join ${stats_db_name}.datasets_oa on alldatasets.ri_initiative=datasets_oa.ri_initiative; + +CREATE TEMPORARY TABLE ${stats_db_name}.allsoftwaresshare as +select software_oa.ri_initiative, software_oa.no_oasoftware/allsoftware.no_allsoftware s + from ${stats_db_name}.allsoftware + join ${stats_db_name}.software_oa on allsoftware.ri_initiative=software_oa.ri_initiative; + +create table if not exists ${stats_db_name}.indi_ris_openess stored as parquet as +select allpubsshare.ri_initiative, + (p+if(isnull(s),0,s)+if(isnull(d),0,d))/(1+(case when s is null then 0 else 1 end) + +(case when d is null then 0 else 1 end)) + ris_openess FROM ${stats_db_name}.allpubsshare + left outer join (select ri_initiative,d from + ${stats_db_name}.alldatasetssshare) tmp1 + on tmp1.ri_initiative=allpubsshare.ri_initiative + left outer join (select ri_initiative,s from + ${stats_db_name}.allsoftwaresshare) tmp2 + on tmp2.ri_initiative=allpubsshare.ri_initiative; + +DROP TABLE ${stats_db_name}.result_contexts purge; +DROP TABLE ${stats_db_name}.pubs_oa purge; +DROP TABLE ${stats_db_name}.datasets_oa purge; +DROP TABLE ${stats_db_name}.software_oa purge; +DROP TABLE ${stats_db_name}.allpubs purge; +DROP TABLE ${stats_db_name}.alldatasets purge; +DROP TABLE ${stats_db_name}.allsoftware purge; +DROP TABLE ${stats_db_name}.allpubsshare purge; +DROP TABLE ${stats_db_name}.alldatasetssshare purge; +DROP TABLE ${stats_db_name}.allsoftwaresshare purge; + +--Funder Findability +create table if not exists ${stats_db_name}.indi_funder_findable stored as parquet as +with result_findable as + (select p.funder funder, count(distinct rp.id) no_result_findable from ${stats_db_name}.result_projects rp + join ${stats_db_name}.publication r on r.id=rp.id + join ${stats_db_name}.project p on p.id=rp.project + join ${stats_db_name}.result_pids rpi on rpi.id=r.id + where cast(year as int)>2003 + group by p.funder), + allresults as (select p.funder funder, count(distinct rp.id) no_allresults from ${stats_db_name}.result_projects rp + join ${stats_db_name}.result r on r.id=rp.id + join ${stats_db_name}.project p on p.id=rp.project + where cast(year as int)>2003 + group by p.funder) +select allresults.funder, result_findable.no_result_findable/allresults.no_allresults funder_findable +from allresults + join result_findable on result_findable.funder=allresults.funder; + +--RIs Findability +create table if not exists ${stats_db_name}.indi_ris_findable stored as parquet as +with result_contexts as +(select distinct rc.id, context.name ri_initiative from ${stats_db_name}.result_concepts rc +join ${stats_db_name}.concept on concept.id=rc.concept +join ${stats_db_name}.category on category.id=concept.category +join ${stats_db_name}.context on context.id=category.context), +result_findable as + (select rc.ri_initiative ri_initiative, count(distinct rc.id) no_result_findable from result_contexts rc + join ${stats_db_name}.result r on r.id=rc.id + join ${stats_db_name}.result_pids rp on rp.id=r.id + where cast(r.year as int)>2003 + group by rc.ri_initiative), +allresults as +(select rc.ri_initiative ri_initiative, count(distinct rc.id) no_allresults from result_contexts rc + join ${stats_db_name}.result r on r.id=rc.id + where cast(r.year as int)>2003 + group by rc.ri_initiative) +select allresults.ri_initiative, result_findable.no_result_findable/allresults.no_allresults ris_findable +from allresults + join result_findable on result_findable.ri_initiative=allresults.ri_initiative; + diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql index 3eeb792c7..586bee347 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql @@ -88,6 +88,7 @@ create view if not exists TARGET.doctoratestudents as select * from SOURCE.docto create view if not exists TARGET.totalresearchers as select * from SOURCE.totalresearchers; create view if not exists TARGET.totalresearchersft as select * from SOURCE.totalresearchersft; create view if not exists TARGET.hrrst as select * from SOURCE.hrrst; +create view if not exists TARGET.graduatedoctorates as select * from SOURCE.graduatedoctorates; create table TARGET.result_citations stored as parquet as select * from SOURCE.result_citations orig where exists (select 1 from TARGET.result r where r.id=orig.id); --ANALYZE TABLE TARGET.result_citations COMPUTE STATISTICS; @@ -270,3 +271,11 @@ create table TARGET.indi_pub_interdisciplinarity stored as parquet as select * f --ANALYZE TABLE TARGET.indi_pub_interdisciplinarity COMPUTE STATISTICS; create table TARGET.result_apc_affiliations stored as parquet as select * from SOURCE.result_apc_affiliations orig where exists (select 1 from TARGET.result r where r.id=orig.id); --ANALYZE TABLE TARGET.result_apc_affiliations COMPUTE STATISTICS; +create table TARGET.indi_is_project_result_after stored as parquet as select * from SOURCE.indi_is_project_result_after orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); +create table TARGET.indi_is_funder_plan_s stored as parquet as select * from SOURCE.indi_is_funder_plan_s orig where exists (select 1 from TARGET.result r where r.id=orig.id); +create view TARGET.indi_funder_fairness as select * from SOURCE.indi_funder_fairness; +create view TARGET.indi_funder_openess as select * from SOURCE.indi_funder_openess; +create view TARGET.indi_funder_findable as select * from SOURCE.indi_funder_findable; +create view TARGET.indi_ris_fairness as select * from SOURCE.indi_ris_fairness; +create view TARGET.indi_ris_openess as select * from SOURCE.indi_ris_openess; +create view TARGET.indi_ris_findable as select * from SOURCE.indi_ris_findable; \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql index a59791084..df4795e3e 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql @@ -15,6 +15,7 @@ create view if not exists TARGET.doctoratestudents as select * from SOURCE.docto create view if not exists TARGET.totalresearchers as select * from SOURCE.totalresearchers; create view if not exists TARGET.totalresearchersft as select * from SOURCE.totalresearchersft; create view if not exists TARGET.hrrst as select * from SOURCE.hrrst; +create view if not exists TARGET.graduatedoctorates as select * from SOURCE.graduatedoctorates; create table TARGET.result stored as parquet as select distinct * from ( @@ -73,7 +74,11 @@ create table TARGET.result stored as parquet as 'openorgs____::c0b262bd6eab819e4c994914f9c010e2', -- National Institute of Geophysics and Volcanology 'openorgs____::1624ff7c01bb641b91f4518539a0c28a', -- Vrije Universiteit Amsterdam 'openorgs____::4d4051b56708688235252f1d8fddb8c1', --Iscte - Instituto Universitário de Lisboa - 'openorgs____::ab4ac74c35fa5dada770cf08e5110fab' -- Universidade Católica Portuguesa + 'openorgs____::ab4ac74c35fa5dada770cf08e5110fab', -- Universidade Católica Portuguesa + 'openorgs____::4d4051b56708688235252f1d8fddb8c1', -- Iscte - Instituto Universitário de Lisboa + 'openorgs____::5d55fb216b14691cf68218daf5d78cd9', -- Munster Technological University + 'openorgs____::0fccc7640f0cb44d5cd1b06b312a06b9', -- Cardiff University + 'openorgs____::8839b55dae0c84d56fd533f52d5d483a' -- Leibniz Institute of Ecological Urban and Regional Development ) )) foo; --ANALYZE TABLE TARGET.result COMPUTE STATISTICS; @@ -92,6 +97,7 @@ create view if not exists TARGET.doctoratestudents as select * from SOURCE.docto create view if not exists TARGET.totalresearchers as select * from SOURCE.totalresearchers; create view if not exists TARGET.totalresearchersft as select * from SOURCE.totalresearchersft; create view if not exists TARGET.hrrst as select * from SOURCE.hrrst; +--create view if not exists TARGET.graduatedoctorates as select * from SOURCE.graduatedoctorates; create table TARGET.result_citations stored as parquet as select * from SOURCE.result_citations orig where exists (select 1 from TARGET.result r where r.id=orig.id); --ANALYZE TABLE TARGET.result_citations COMPUTE STATISTICS; @@ -274,3 +280,13 @@ create table TARGET.indi_pub_interdisciplinarity stored as parquet as select * f --ANALYZE TABLE TARGET.indi_pub_interdisciplinarity COMPUTE STATISTICS; create table TARGET.result_apc_affiliations stored as parquet as select * from SOURCE.result_apc_affiliations orig where exists (select 1 from TARGET.result r where r.id=orig.id); --ANALYZE TABLE TARGET.result_apc_affiliations COMPUTE STATISTICS; +create table TARGET.indi_is_project_result_after stored as parquet as select * from SOURCE.indi_is_project_result_after orig where exists (select 1 from TARGET.result r where r.id=orig.id); +create table TARGET.indi_is_funder_plan_s stored as parquet as select * from SOURCE.indi_is_funder_plan_s orig where exists (select 1 from TARGET.result r where r.id=orig.id); +create view TARGET.indi_funder_fairness as select * from SOURCE.indi_funder_fairness; +create view TARGET.indi_funder_openess as select * from SOURCE.indi_funder_openess; +create view TARGET.indi_funder_findable as select * from SOURCE.indi_funder_findable; +create view TARGET.indi_ris_fairness as select * from SOURCE.indi_ris_fairness; +create view TARGET.indi_ris_openess as select * from SOURCE.indi_ris_openess; +create view TARGET.indi_ris_findable as select * from SOURCE.indi_ris_findable; + + diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql index 1f75c3cd1..7bfba92a8 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql @@ -34,16 +34,16 @@ create table TARGET.result stored as parquet as 'openorgs____::3e8d1f8c3f6cd7f418b09f1f58b4873b', -- Aristotle University of Thessaloniki 'openorgs____::3fcef6e1c469c10f2a84b281372c9814', -- World Bank 'openorgs____::1698a2eb1885ef8adb5a4a969e745ad3', -- École des Ponts ParisTech - 'openorgs____::e15adb13c4dadd49de4d35c39b5da93a', -- Nanyang Technological University - 'openorgs____::4b34103bde246228fcd837f5f1bf4212', -- Autonomous University of Barcelona - 'openorgs____::72ec75fcfc4e0df1a76dc4c49007fceb', -- McMaster University - 'openorgs____::51c7fc556e46381734a25a6fbc3fd398', -- University of Modena and Reggio Emilia - 'openorgs____::235d7f9ad18ecd7e6dc62ea4990cb9db', -- Bilkent University - 'openorgs____::31f2fa9e05b49d4cf40a19c3fed8eb06', -- Saints Cyril and Methodius University of Skopje + 'openorgs____::e15adb13c4dadd49de4d35c39b5da93a', -- Nanyang Technological University + 'openorgs____::4b34103bde246228fcd837f5f1bf4212', -- Autonomous University of Barcelona + 'openorgs____::72ec75fcfc4e0df1a76dc4c49007fceb', -- McMaster University + 'openorgs____::51c7fc556e46381734a25a6fbc3fd398', -- University of Modena and Reggio Emilia + 'openorgs____::235d7f9ad18ecd7e6dc62ea4990cb9db', -- Bilkent University + 'openorgs____::31f2fa9e05b49d4cf40a19c3fed8eb06', -- Saints Cyril and Methodius University of Skopje 'openorgs____::db7686f30f22cbe73a4fde872ce812a6', -- University of Milan - 'openorgs____::b8b8ca674452579f3f593d9f5e557483', -- University College Cork + 'openorgs____::b8b8ca674452579f3f593d9f5e557483', -- University College Cork 'openorgs____::38d7097854736583dde879d12dacafca', -- Brown University - 'openorgs____::57784c9e047e826fefdb1ef816120d92', --Arts et Métiers ParisTech + 'openorgs____::57784c9e047e826fefdb1ef816120d92', --Arts et Métiers ParisTech 'openorgs____::2530baca8a15936ba2e3297f2bce2e7e', -- University of Cape Town 'openorgs____::d11f981828c485cd23d93f7f24f24db1', -- Technological University Dublin 'openorgs____::5e6bf8962665cdd040341171e5c631d8', -- Delft University of Technology @@ -52,10 +52,13 @@ create table TARGET.result stored as parquet as 'openorgs____::66aa9fc2fceb271423dfabcc38752dc0', -- Lund University 'openorgs____::3cff625a4370d51e08624cc586138b2f', -- IMT Atlantique 'openorgs____::c0b262bd6eab819e4c994914f9c010e2', -- National Institute of Geophysics and Volcanology - 'openorgs____::1624ff7c01bb641b91f4518539a0c28a', -- Vrije Universiteit Amsterdam + 'openorgs____::1624ff7c01bb641b91f4518539a0c28a', -- Vrije Universiteit Amsterdam 'openorgs____::4d4051b56708688235252f1d8fddb8c1', --Iscte - Instituto Universitário de Lisboa - 'openorgs____::ab4ac74c35fa5dada770cf08e5110fab' -- Universidade Católica Portuguesa - + 'openorgs____::ab4ac74c35fa5dada770cf08e5110fab', -- Universidade Católica Portuguesa + 'openorgs____::4d4051b56708688235252f1d8fddb8c1', -- Iscte - Instituto Universitário de Lisboa + 'openorgs____::5d55fb216b14691cf68218daf5d78cd9', -- Munster Technological University + 'openorgs____::0fccc7640f0cb44d5cd1b06b312a06b9', -- Cardiff University + 'openorgs____::8839b55dae0c84d56fd533f52d5d483a' -- Leibniz Institute of Ecological Urban and Regional Development ))) foo; --ANALYZE TABLE TARGET.result COMPUTE STATISTICS; \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step5.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step5.sql old mode 100644 new mode 100755 diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml index c03520e48..aa991730b 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml @@ -467,7 +467,7 @@ ${usage_stats_db_shadow_name} finalizeImpalaCluster.sh - + From 2caaaec42d24023b7195f8922bafd43bc12494dd Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Fri, 1 Sep 2023 09:32:57 +0200 Subject: [PATCH 009/148] Include SparkCleanRelation logic in SparkPropagateRelation SparkPropagateRelation includes merge relations Revised tests for SparkPropagateRelation --- .../dhp/oa/dedup/RelationAggregator.java | 57 ------- .../dhp/oa/dedup/SparkCleanRelation.scala | 78 --------- .../dhp/oa/dedup/SparkPropagateRelation.java | 138 ++++++++-------- .../oa/dedup/cleanRelation_parameters.json | 20 --- .../dedup/consistency/oozie_app/workflow.xml | 28 +--- .../dnetlib/dhp/oa/dedup/SparkDedupTest.java | 150 ++++++++---------- .../oa/dedup/SparkOpenorgsProvisionTest.java | 103 ++++++------ 7 files changed, 185 insertions(+), 389 deletions(-) delete mode 100644 dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/RelationAggregator.java delete mode 100644 dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCleanRelation.scala delete mode 100644 dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/cleanRelation_parameters.json diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/RelationAggregator.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/RelationAggregator.java deleted file mode 100644 index 96d783dbf..000000000 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/RelationAggregator.java +++ /dev/null @@ -1,57 +0,0 @@ - -package eu.dnetlib.dhp.oa.dedup; - -import java.util.Objects; - -import org.apache.spark.sql.Encoder; -import org.apache.spark.sql.Encoders; -import org.apache.spark.sql.expressions.Aggregator; - -import eu.dnetlib.dhp.schema.oaf.Relation; - -public class RelationAggregator extends Aggregator { - - private static final Relation ZERO = new Relation(); - - @Override - public Relation zero() { - return ZERO; - } - - @Override - public Relation reduce(Relation b, Relation a) { - return mergeRel(b, a); - } - - @Override - public Relation merge(Relation b, Relation a) { - return mergeRel(b, a); - } - - @Override - public Relation finish(Relation r) { - return r; - } - - private Relation mergeRel(Relation b, Relation a) { - if (Objects.equals(b, ZERO)) { - return a; - } - if (Objects.equals(a, ZERO)) { - return b; - } - - b.mergeFrom(a); - return b; - } - - @Override - public Encoder bufferEncoder() { - return Encoders.kryo(Relation.class); - } - - @Override - public Encoder outputEncoder() { - return Encoders.kryo(Relation.class); - } -} diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCleanRelation.scala b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCleanRelation.scala deleted file mode 100644 index 5d8da42c2..000000000 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCleanRelation.scala +++ /dev/null @@ -1,78 +0,0 @@ -package eu.dnetlib.dhp.oa.dedup - -import eu.dnetlib.dhp.application.ArgumentApplicationParser -import eu.dnetlib.dhp.common.HdfsSupport -import eu.dnetlib.dhp.schema.oaf.Relation -import eu.dnetlib.dhp.utils.ISLookupClientFactory -import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService -import org.apache.commons.io.IOUtils -import org.apache.spark.SparkConf -import org.apache.spark.sql._ -import org.apache.spark.sql.functions.col -import org.apache.spark.sql.types.{DataTypes, StructField, StructType} -import org.slf4j.LoggerFactory - -object SparkCleanRelation { - private val log = LoggerFactory.getLogger(classOf[SparkCleanRelation]) - - @throws[Exception] - def main(args: Array[String]): Unit = { - val parser = new ArgumentApplicationParser( - IOUtils.toString( - classOf[SparkCleanRelation].getResourceAsStream("/eu/dnetlib/dhp/oa/dedup/cleanRelation_parameters.json") - ) - ) - parser.parseArgument(args) - val conf = new SparkConf - - new SparkCleanRelation(parser, AbstractSparkAction.getSparkSession(conf)) - .run(ISLookupClientFactory.getLookUpService(parser.get("isLookUpUrl"))) - } -} - -class SparkCleanRelation(parser: ArgumentApplicationParser, spark: SparkSession) - extends AbstractSparkAction(parser, spark) { - override def run(isLookUpService: ISLookUpService): Unit = { - val graphBasePath = parser.get("graphBasePath") - val inputPath = parser.get("inputPath") - val outputPath = parser.get("outputPath") - - SparkCleanRelation.log.info("graphBasePath: '{}'", graphBasePath) - SparkCleanRelation.log.info("inputPath: '{}'", inputPath) - SparkCleanRelation.log.info("outputPath: '{}'", outputPath) - - AbstractSparkAction.removeOutputDir(spark, outputPath) - - val entities = - Seq("datasource", "project", "organization", "publication", "dataset", "software", "otherresearchproduct") - - val idsSchema = StructType.fromDDL("`id` STRING, `dataInfo` STRUCT<`deletedbyinference`:BOOLEAN,`invisible`:BOOLEAN>") - - val emptyIds = spark.createDataFrame(spark.sparkContext.emptyRDD[Row].setName("empty"), - idsSchema) - - val ids = entities - .foldLeft(emptyIds)((ds, entity) => { - val entityPath = graphBasePath + '/' + entity - if (HdfsSupport.exists(entityPath, spark.sparkContext.hadoopConfiguration)) { - ds.union(spark.read.schema(idsSchema).json(entityPath)) - } else { - ds - } - }) - .filter("dataInfo.deletedbyinference != true AND dataInfo.invisible != true") - .select("id") - .distinct() - - val relations = spark.read.schema(Encoders.bean(classOf[Relation]).schema).json(inputPath) - .filter("dataInfo.deletedbyinference != true AND dataInfo.invisible != true") - - AbstractSparkAction.save( - relations - .join(ids, col("source") === ids("id"), "leftsemi") - .join(ids, col("target") === ids("id"), "leftsemi"), - outputPath, - SaveMode.Overwrite - ) - } -} diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java index 175ebf8a6..739295c91 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java @@ -3,23 +3,19 @@ package eu.dnetlib.dhp.oa.dedup; import static org.apache.spark.sql.functions.col; -import java.util.Arrays; -import java.util.Collections; -import java.util.Iterator; -import java.util.Objects; - -import org.apache.commons.beanutils.BeanUtils; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.spark.SparkConf; -import org.apache.spark.api.java.function.FilterFunction; import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.api.java.function.ReduceFunction; import org.apache.spark.sql.*; +import org.apache.spark.sql.catalyst.encoders.RowEncoder; +import org.apache.spark.sql.types.StructType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.HdfsSupport; +import eu.dnetlib.dhp.schema.common.EntityType; import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.oaf.DataInfo; @@ -70,73 +66,63 @@ public class SparkPropagateRelation extends AbstractSparkAction { log.info("workingPath: '{}'", workingPath); log.info("graphOutputPath: '{}'", graphOutputPath); - final String outputRelationPath = DedupUtility.createEntityPath(graphOutputPath, "relation"); - removeOutputDir(spark, outputRelationPath); - Dataset mergeRels = spark - .read() - .load(DedupUtility.createMergeRelPath(workingPath, "*", "*")) - .as(REL_BEAN_ENC); + .read() + .load(DedupUtility.createMergeRelPath(workingPath, "*", "*")) + .as(REL_BEAN_ENC); // - Dataset mergedIds = mergeRels - .where(col("relClass").equalTo(ModelConstants.MERGES)) - .select(col("source").as("dedupID"), col("target").as("mergedObjectID")) - .distinct() - .cache(); + Dataset idsToMerge = mergeRels + .where(col("relClass").equalTo(ModelConstants.MERGES)) + .select(col("source").as("dedupID"), col("target").as("mergedObjectID")) + .distinct(); Dataset allRels = spark - .read() - .schema(REL_BEAN_ENC.schema()) - .json(DedupUtility.createEntityPath(graphBasePath, "relation")); + .read() + .schema(REL_BEAN_ENC.schema()) + .json(graphBasePath + "/relation"); Dataset dedupedRels = allRels - .joinWith(mergedIds, allRels.col("source").equalTo(mergedIds.col("mergedObjectID")), "left_outer") - .joinWith(mergedIds, col("_1.target").equalTo(mergedIds.col("mergedObjectID")), "left_outer") - .select("_1._1", "_1._2.dedupID", "_2.dedupID") - .as(Encoders.tuple(REL_BEAN_ENC, Encoders.STRING(), Encoders.STRING())) - .flatMap(SparkPropagateRelation::addInferredRelations, REL_KRYO_ENC); + .joinWith(idsToMerge, allRels.col("source").equalTo(idsToMerge.col("mergedObjectID")), "left_outer") + .joinWith(idsToMerge, col("_1.target").equalTo(idsToMerge.col("mergedObjectID")), "left_outer") + .select("_1._1", "_1._2.dedupID", "_2.dedupID") + .as(Encoders.tuple(REL_BEAN_ENC, Encoders.STRING(), Encoders.STRING())) + .map((MapFunction, Relation>) t -> { + Relation rel = t._1(); + String newSource = t._2(); + String newTarget = t._3(); - Dataset processedRelations = distinctRelations( - dedupedRels.union(mergeRels.map((MapFunction) r -> r, REL_KRYO_ENC))) - .filter((FilterFunction) r -> !Objects.equals(r.getSource(), r.getTarget())); + if (rel.getDataInfo() == null) { + rel.setDataInfo(new DataInfo()); + } - save(processedRelations, outputRelationPath, SaveMode.Overwrite); - } + if (newSource != null || newTarget != null) { + rel.getDataInfo().setDeletedbyinference(false); - private static Iterator addInferredRelations(Tuple3 t) throws Exception { - Relation existingRel = t._1(); - String newSource = t._2(); - String newTarget = t._3(); + if (newSource != null) + rel.setSource(newSource); - if (newSource == null && newTarget == null) { - return Collections.singleton(t._1()).iterator(); - } + if (newTarget != null) + rel.setTarget(newTarget); + } - // update existing relation - if (existingRel.getDataInfo() == null) { - existingRel.setDataInfo(new DataInfo()); - } - existingRel.getDataInfo().setDeletedbyinference(true); + return rel; + }, REL_BEAN_ENC); - // Create new relation inferred by dedupIDs - Relation inferredRel = (Relation) BeanUtils.cloneBean(existingRel); + // ids of records that are both not deletedbyinference and not invisible + Dataset ids = validIds(spark, graphBasePath); - inferredRel.setDataInfo((DataInfo) BeanUtils.cloneBean(existingRel.getDataInfo())); - inferredRel.getDataInfo().setDeletedbyinference(false); + // filter relations that point to valid records, can force them to be visible + Dataset cleanedRels = dedupedRels + .join(ids, col("source").equalTo(ids.col("id")), "leftsemi") + .join(ids, col("target").equalTo(ids.col("id")), "leftsemi") + .as(REL_BEAN_ENC) + .map((MapFunction) r -> { + r.getDataInfo().setInvisible(false); + return r; + }, REL_KRYO_ENC); - if (newSource != null) - inferredRel.setSource(newSource); - - if (newTarget != null) - inferredRel.setTarget(newTarget); - - return Arrays.asList(existingRel, inferredRel).iterator(); - } - - private Dataset distinctRelations(Dataset rels) { - return rels - .filter(getRelationFilterFunction()) + Dataset distinctRels = cleanedRels .groupByKey( (MapFunction) r -> String .join(" ", r.getSource(), r.getTarget(), r.getRelType(), r.getSubRelType(), r.getRelClass()), @@ -146,13 +132,33 @@ public class SparkPropagateRelation extends AbstractSparkAction { return b; }) .map((MapFunction, Relation>) Tuple2::_2, REL_BEAN_ENC); + + final String outputRelationPath = graphOutputPath + "/relation"; + removeOutputDir(spark, outputRelationPath); + save( + distinctRels + .union(mergeRels) + .filter("source != target AND dataInfo.deletedbyinference != true AND dataInfo.invisible != true"), + outputRelationPath, + SaveMode.Overwrite); } - private FilterFunction getRelationFilterFunction() { - return r -> StringUtils.isNotBlank(r.getSource()) || - StringUtils.isNotBlank(r.getTarget()) || - StringUtils.isNotBlank(r.getRelType()) || - StringUtils.isNotBlank(r.getSubRelType()) || - StringUtils.isNotBlank(r.getRelClass()); + static Dataset validIds(SparkSession spark, String graphBasePath) { + StructType idsSchema = StructType + .fromDDL("`id` STRING, `dataInfo` STRUCT<`deletedbyinference`:BOOLEAN,`invisible`:BOOLEAN>"); + + Dataset allIds = spark.emptyDataset(RowEncoder.apply(idsSchema)); + + for (EntityType entityType : ModelSupport.entityTypes.keySet()) { + String entityPath = graphBasePath + '/' + entityType.name(); + if (HdfsSupport.exists(entityPath, spark.sparkContext().hadoopConfiguration())) { + allIds = allIds.union(spark.read().schema(idsSchema).json(entityPath)); + } + } + + return allIds + .filter("dataInfo.deletedbyinference != true AND dataInfo.invisible != true") + .select("id") + .distinct(); } } diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/cleanRelation_parameters.json b/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/cleanRelation_parameters.json deleted file mode 100644 index 860539ad9..000000000 --- a/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/cleanRelation_parameters.json +++ /dev/null @@ -1,20 +0,0 @@ -[ - { - "paramName": "i", - "paramLongName": "graphBasePath", - "paramDescription": "the base path of raw graph", - "paramRequired": true - }, - { - "paramName": "w", - "paramLongName": "inputPath", - "paramDescription": "the path to the input relation to cleanup", - "paramRequired": true - }, - { - "paramName": "o", - "paramLongName": "outputPath", - "paramDescription": "the path of the output relation cleaned", - "paramRequired": true - } -] \ No newline at end of file diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/workflow.xml b/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/workflow.xml index b724e5d0b..0083339cf 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/workflow.xml @@ -100,35 +100,9 @@ --conf spark.sql.shuffle.partitions=15000 --graphBasePath${graphBasePath} - --graphOutputPath${workingPath}/propagaterelation/ + --graphOutputPath${graphOutputPath} --workingPath${workingPath} - - - - - - - yarn - cluster - Clean Relations - eu.dnetlib.dhp.oa.dedup.SparkCleanRelation - dhp-dedup-openaire-${projectVersion}.jar - - --executor-memory=${sparkExecutorMemory} - --conf spark.executor.memoryOverhead=${sparkExecutorMemoryOverhead} - --executor-cores=${sparkExecutorCores} - --driver-memory=${sparkDriverMemory} - --conf spark.extraListeners=${spark2ExtraListeners} - --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} - --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} - --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.sql.shuffle.partitions=15000 - - --graphBasePath${graphBasePath} - --inputPath${workingPath}/propagaterelation/relation - --outputPath${graphOutputPath}/relation - diff --git a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java index 38bd72a5e..6c4935637 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java +++ b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkDedupTest.java @@ -3,7 +3,6 @@ package eu.dnetlib.dhp.oa.dedup; import static java.nio.file.Files.createTempDirectory; -import static org.apache.spark.sql.functions.col; import static org.apache.spark.sql.functions.count; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.lenient; @@ -23,14 +22,13 @@ import java.util.stream.Collectors; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.spark.SparkConf; -import org.apache.spark.api.java.JavaPairRDD; -import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.function.FilterFunction; import org.apache.spark.api.java.function.MapFunction; -import org.apache.spark.api.java.function.PairFunction; -import org.apache.spark.sql.*; import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; import org.junit.jupiter.api.*; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -46,8 +44,6 @@ import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.oaf.*; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; -import eu.dnetlib.pace.util.MapDocumentUtil; -import scala.Tuple2; @ExtendWith(MockitoExtension.class) @TestMethodOrder(MethodOrderer.OrderAnnotation.class) @@ -62,6 +58,8 @@ public class SparkDedupTest implements Serializable { private static String testGraphBasePath; private static String testOutputBasePath; private static String testDedupGraphBasePath; + private static String testConsistencyGraphBasePath; + private static final String testActionSetId = "test-orchestrator"; private static String whitelistPath; private static List whiteList; @@ -75,6 +73,7 @@ public class SparkDedupTest implements Serializable { .get(SparkDedupTest.class.getResource("/eu/dnetlib/dhp/dedup/entities").toURI()) .toFile() .getAbsolutePath(); + testOutputBasePath = createTempDirectory(SparkDedupTest.class.getSimpleName() + "-") .toAbsolutePath() .toString(); @@ -83,6 +82,10 @@ public class SparkDedupTest implements Serializable { .toAbsolutePath() .toString(); + testConsistencyGraphBasePath = createTempDirectory(SparkDedupTest.class.getSimpleName() + "-") + .toAbsolutePath() + .toString(); + whitelistPath = Paths .get(SparkDedupTest.class.getResource("/eu/dnetlib/dhp/dedup/whitelist.simrels.txt").toURI()) .toFile() @@ -674,22 +677,45 @@ public class SparkDedupTest implements Serializable { assertEquals(mergedOrp, deletedOrp); } + @Test + @Order(6) + void copyRelationsNoOpenorgsTest() throws Exception { + + ArgumentApplicationParser parser = new ArgumentApplicationParser( + IOUtils + .toString( + SparkCopyRelationsNoOpenorgs.class + .getResourceAsStream( + "/eu/dnetlib/dhp/oa/dedup/updateEntity_parameters.json"))); + parser + .parseArgument( + new String[] { + "-i", testGraphBasePath, "-w", testOutputBasePath, "-o", testDedupGraphBasePath + }); + + new SparkCopyRelationsNoOpenorgs(parser, spark).run(isLookUpService); + + final Dataset outputRels = spark.read().text(testDedupGraphBasePath + "/relation"); + + System.out.println(outputRels.count()); + // assertEquals(2382, outputRels.count()); + } + @Test @Order(7) void propagateRelationTest() throws Exception { ArgumentApplicationParser parser = new ArgumentApplicationParser( classPathResourceAsString("/eu/dnetlib/dhp/oa/dedup/propagateRelation_parameters.json")); - String outputRelPath = testDedupGraphBasePath + "/propagaterelation"; parser .parseArgument( new String[] { - "-i", testGraphBasePath, "-w", testOutputBasePath, "-o", outputRelPath + "-i", testDedupGraphBasePath, "-w", testOutputBasePath, "-o", testConsistencyGraphBasePath }); new SparkPropagateRelation(parser, spark).run(isLookUpService); - long relations = jsc.textFile(outputRelPath + "/relation").count(); + long relations = jsc.textFile(testDedupGraphBasePath + "/relation").count(); // assertEquals(4860, relations); System.out.println("relations = " + relations); @@ -699,95 +725,52 @@ public class SparkDedupTest implements Serializable { .read() .load(DedupUtility.createMergeRelPath(testOutputBasePath, "*", "*")) .as(Encoders.bean(Relation.class)); - final JavaPairRDD mergedIds = mergeRels - .where("relClass == 'merges'") - .select(mergeRels.col("target")) - .distinct() - .toJavaRDD() - .mapToPair( - (PairFunction) r -> new Tuple2(r.getString(0), "d")); - JavaRDD toCheck = jsc - .textFile(outputRelPath + "/relation") - .mapToPair(json -> new Tuple2<>(MapDocumentUtil.getJPathString("$.source", json), json)) - .join(mergedIds) - .map(t -> t._2()._1()) - .mapToPair(json -> new Tuple2<>(MapDocumentUtil.getJPathString("$.target", json), json)) - .join(mergedIds) - .map(t -> t._2()._1()); + Dataset inputRels = spark + .read() + .json(testDedupGraphBasePath + "/relation"); - long deletedbyinference = toCheck.filter(this::isDeletedByInference).count(); - long updated = toCheck.count(); + Dataset outputRels = spark + .read() + .json(testConsistencyGraphBasePath + "/relation"); - assertEquals(updated, deletedbyinference); + assertEquals( + 0, outputRels + .filter("dataInfo.deletedbyinference == true OR dataInfo.invisible == true") + .count()); + + assertEquals( + 5, outputRels + .filter("relClass NOT IN ('merges', 'isMergedIn')") + .count()); + + assertEquals(5 + mergeRels.count(), outputRels.count()); } @Test @Order(8) - void testCleanBaseRelations() throws Exception { - ArgumentApplicationParser parser = new ArgumentApplicationParser( - classPathResourceAsString("/eu/dnetlib/dhp/oa/dedup/cleanRelation_parameters.json")); - - // append dangling relations to be cleaned up + void testCleanedPropagatedRelations() throws Exception { Dataset df_before = spark .read() .schema(Encoders.bean(Relation.class).schema()) - .json(testGraphBasePath + "/relation"); - Dataset df_input = df_before - .unionByName(df_before.drop("source").withColumn("source", functions.lit("n/a"))) - .unionByName(df_before.drop("target").withColumn("target", functions.lit("n/a"))); - df_input.write().mode(SaveMode.Overwrite).json(testOutputBasePath + "_tmp"); - - parser - .parseArgument( - new String[] { - "--graphBasePath", testGraphBasePath, - "--inputPath", testGraphBasePath + "/relation", - "--outputPath", testDedupGraphBasePath + "/relation" - }); - - new SparkCleanRelation(parser, spark).run(isLookUpService); + .json(testDedupGraphBasePath + "/relation"); Dataset df_after = spark .read() .schema(Encoders.bean(Relation.class).schema()) - .json(testDedupGraphBasePath + "/relation"); - - assertNotEquals(df_before.count(), df_input.count()); - assertNotEquals(df_input.count(), df_after.count()); - assertEquals(5, df_after.count()); - } - - @Test - @Order(9) - void testCleanDedupedRelations() throws Exception { - ArgumentApplicationParser parser = new ArgumentApplicationParser( - classPathResourceAsString("/eu/dnetlib/dhp/oa/dedup/cleanRelation_parameters.json")); - - String inputRelPath = testDedupGraphBasePath + "/propagaterelation/relation"; - - // append dangling relations to be cleaned up - Dataset df_before = spark.read().schema(Encoders.bean(Relation.class).schema()).json(inputRelPath); - - df_before.filter(col("dataInfo.deletedbyinference").notEqual(true)).show(50, false); - - parser - .parseArgument( - new String[] { - "--graphBasePath", testGraphBasePath, - "--inputPath", inputRelPath, - "--outputPath", testDedupGraphBasePath + "/relation" - }); - - new SparkCleanRelation(parser, spark).run(isLookUpService); - - Dataset df_after = spark - .read() - .schema(Encoders.bean(Relation.class).schema()) - .json(testDedupGraphBasePath + "/relation"); + .json(testConsistencyGraphBasePath + "/relation"); assertNotEquals(df_before.count(), df_after.count()); - assertEquals(0, df_after.count()); + + assertEquals( + 0, df_after + .filter("dataInfo.deletedbyinference == true OR dataInfo.invisible == true") + .count()); + + assertEquals( + 5, df_after + .filter("relClass NOT IN ('merges', 'isMergedIn')") + .count()); } @Test @@ -813,6 +796,7 @@ public class SparkDedupTest implements Serializable { public static void finalCleanUp() throws IOException { FileUtils.deleteDirectory(new File(testOutputBasePath)); FileUtils.deleteDirectory(new File(testDedupGraphBasePath)); + FileUtils.deleteDirectory(new File(testConsistencyGraphBasePath)); } public boolean isDeletedByInference(String s) { diff --git a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkOpenorgsProvisionTest.java b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkOpenorgsProvisionTest.java index a0bf6b37e..73e768cf1 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkOpenorgsProvisionTest.java +++ b/dhp-workflows/dhp-dedup-openaire/src/test/java/eu/dnetlib/dhp/oa/dedup/SparkOpenorgsProvisionTest.java @@ -3,6 +3,7 @@ package eu.dnetlib.dhp.oa.dedup; import static java.nio.file.Files.createTempDirectory; +import static org.apache.spark.sql.functions.col; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.lenient; @@ -15,10 +16,6 @@ import java.nio.file.Paths; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.spark.SparkConf; -import org.apache.spark.api.java.JavaPairRDD; -import org.apache.spark.api.java.JavaRDD; -import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.api.java.function.PairFunction; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoders; import org.apache.spark.sql.Row; @@ -33,8 +30,6 @@ import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.schema.oaf.Relation; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; -import eu.dnetlib.pace.util.MapDocumentUtil; -import scala.Tuple2; @ExtendWith(MockitoExtension.class) @TestMethodOrder(MethodOrderer.OrderAnnotation.class) @@ -44,11 +39,11 @@ public class SparkOpenorgsProvisionTest implements Serializable { ISLookUpService isLookUpService; private static SparkSession spark; - private static JavaSparkContext jsc; private static String testGraphBasePath; private static String testOutputBasePath; private static String testDedupGraphBasePath; + private static String testConsistencyGraphBasePath; private static final String testActionSetId = "test-orchestrator"; @BeforeAll @@ -64,6 +59,9 @@ public class SparkOpenorgsProvisionTest implements Serializable { testDedupGraphBasePath = createTempDirectory(SparkOpenorgsProvisionTest.class.getSimpleName() + "-") .toAbsolutePath() .toString(); + testConsistencyGraphBasePath = createTempDirectory(SparkOpenorgsProvisionTest.class.getSimpleName() + "-") + .toAbsolutePath() + .toString(); FileUtils.deleteDirectory(new File(testOutputBasePath)); FileUtils.deleteDirectory(new File(testDedupGraphBasePath)); @@ -76,8 +74,13 @@ public class SparkOpenorgsProvisionTest implements Serializable { .master("local[*]") .config(conf) .getOrCreate(); + } - jsc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + @AfterAll + public static void finalCleanUp() throws IOException { + FileUtils.deleteDirectory(new File(testOutputBasePath)); + FileUtils.deleteDirectory(new File(testDedupGraphBasePath)); + FileUtils.deleteDirectory(new File(testConsistencyGraphBasePath)); } @BeforeEach @@ -186,26 +189,21 @@ public class SparkOpenorgsProvisionTest implements Serializable { new SparkUpdateEntity(parser, spark).run(isLookUpService); - long organizations = jsc.textFile(testDedupGraphBasePath + "/organization").count(); + Dataset organizations = spark.read().json(testDedupGraphBasePath + "/organization"); - long mergedOrgs = spark + Dataset mergedOrgs = spark .read() .load(testOutputBasePath + "/" + testActionSetId + "/organization_mergerel") - .as(Encoders.bean(Relation.class)) .where("relClass=='merges'") - .javaRDD() - .map(Relation::getTarget) - .distinct() - .count(); + .select("target") + .distinct(); - assertEquals(80, organizations); + assertEquals(80, organizations.count()); - long deletedOrgs = jsc - .textFile(testDedupGraphBasePath + "/organization") - .filter(this::isDeletedByInference) - .count(); + Dataset deletedOrgs = organizations + .filter("dataInfo.deletedbyinference = TRUE"); - assertEquals(mergedOrgs, deletedOrgs); + assertEquals(mergedOrgs.count(), deletedOrgs.count()); } @Test @@ -226,10 +224,9 @@ public class SparkOpenorgsProvisionTest implements Serializable { new SparkCopyRelationsNoOpenorgs(parser, spark).run(isLookUpService); - final JavaRDD rels = jsc.textFile(testDedupGraphBasePath + "/relation"); - - assertEquals(2382, rels.count()); + final Dataset outputRels = spark.read().text(testDedupGraphBasePath + "/relation"); + assertEquals(2382, outputRels.count()); } @Test @@ -244,51 +241,41 @@ public class SparkOpenorgsProvisionTest implements Serializable { parser .parseArgument( new String[] { - "-i", testGraphBasePath, "-w", testOutputBasePath, "-o", testDedupGraphBasePath + "-i", testDedupGraphBasePath, "-w", testOutputBasePath, "-o", testConsistencyGraphBasePath }); new SparkPropagateRelation(parser, spark).run(isLookUpService); - long relations = jsc.textFile(testDedupGraphBasePath + "/relation").count(); - - assertEquals(4896, relations); - - // check deletedbyinference final Dataset mergeRels = spark .read() .load(DedupUtility.createMergeRelPath(testOutputBasePath, "*", "*")) .as(Encoders.bean(Relation.class)); - final JavaPairRDD mergedIds = mergeRels + + Dataset inputRels = spark + .read() + .json(testDedupGraphBasePath + "/relation"); + + Dataset outputRels = spark + .read() + .json(testConsistencyGraphBasePath + "/relation"); + + final Dataset mergedIds = mergeRels .where("relClass == 'merges'") - .select(mergeRels.col("target")) - .distinct() - .toJavaRDD() - .mapToPair( - (PairFunction) r -> new Tuple2(r.getString(0), "d")); + .select(col("target").as("id")) + .distinct(); - JavaRDD toCheck = jsc - .textFile(testDedupGraphBasePath + "/relation") - .mapToPair(json -> new Tuple2<>(MapDocumentUtil.getJPathString("$.source", json), json)) - .join(mergedIds) - .map(t -> t._2()._1()) - .mapToPair(json -> new Tuple2<>(MapDocumentUtil.getJPathString("$.target", json), json)) - .join(mergedIds) - .map(t -> t._2()._1()); + Dataset toUpdateRels = inputRels + .as("rel") + .join(mergedIds.as("s"), col("rel.source").equalTo(col("s.id")), "left_outer") + .join(mergedIds.as("t"), col("rel.target").equalTo(col("t.id")), "left_outer") + .filter("s.id IS NOT NULL OR t.id IS NOT NULL") + .distinct(); - long deletedbyinference = toCheck.filter(this::isDeletedByInference).count(); - long updated = toCheck.count(); + Dataset updatedRels = inputRels + .select("source", "target", "relClass") + .except(outputRels.select("source", "target", "relClass")); - assertEquals(updated, deletedbyinference); + assertEquals(toUpdateRels.count(), updatedRels.count()); + assertEquals(140, outputRels.count()); } - - @AfterAll - public static void finalCleanUp() throws IOException { - FileUtils.deleteDirectory(new File(testOutputBasePath)); - FileUtils.deleteDirectory(new File(testDedupGraphBasePath)); - } - - public boolean isDeletedByInference(String s) { - return s.contains("\"deletedbyinference\":true"); - } - } From 5f90cc11e98d0addbfb22bf8ce0a83e87a269e00 Mon Sep 17 00:00:00 2001 From: dimitrispie Date: Wed, 6 Sep 2023 14:14:38 +0300 Subject: [PATCH 010/148] Update step16-createIndicatorsTables.sql Fix indi_pub_bronze_oa --- .../oozie_app/scripts/step16-createIndicatorsTables.sql | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql index 1c80f6757..dd249d371 100755 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql @@ -696,7 +696,11 @@ left outer join (select p.id, 1 as is_bronze_oa from ${stats_db_name}.publication p join ${stats_db_name}.indi_result_has_cc_licence cc on cc.id=p.id join ${stats_db_name}.indi_pub_gold_oa ga on ga.id=p.id -where cc.has_cc_license=0 and ga.is_gold=0) tmp on tmp.id=p.id; +join ${stats_db_name}.result_instance ri on ri.id=p.id +join ${stats_db_name}.datasource d on d.id=ri.hostedby +where cc.has_cc_license=0 and ga.is_gold=0 +and (d.type='Journal' or d.type='Journal Aggregator/Publisher') +and ri.accessright='Open Access') tmp on tmp.id=p.id; CREATE TEMPORARY TABLE ${stats_db_name}.project_year_result_year as select p.id project_id, acronym, r.id result_id, r.year, p.end_year From 4786aa0e094fe848a5e7024b68c0d4e7c80ec65f Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Thu, 7 Sep 2023 11:20:35 +0200 Subject: [PATCH 011/148] added Archive ouverte UNIGE (ETHZ.UNIGENF, opendoar____::1400) to the Datacite hostedBy_map --- .../main/resources/eu/dnetlib/dhp/datacite/hostedBy_map.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/datacite/hostedBy_map.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/datacite/hostedBy_map.json index 9088d2960..d07cc33cb 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/datacite/hostedBy_map.json +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/datacite/hostedBy_map.json @@ -1,4 +1,9 @@ { + "ETHZ.UNIGENF": { + "openaire_id": "opendoar____::1400", + "datacite_name": "Uni Genf", + "official_name": "Archive ouverte UNIGE" + }, "GESIS.RKI": { "openaire_id": "re3data_____::r3d100010436", "datacite_name": "Forschungsdatenzentrum am Robert Koch Institut", From 8a6892cc638c84fb6b05db27fa9e2ed538899896 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 12 Sep 2023 14:34:28 +0200 Subject: [PATCH 012/148] [graph dedup] consistency wf should not remove the relations while dispatching the entities --- .../dhp/oa/merge/DispatchEntitiesSparkJob.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/DispatchEntitiesSparkJob.java b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/DispatchEntitiesSparkJob.java index 4d2ccb178..cf0a183d7 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/DispatchEntitiesSparkJob.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/DispatchEntitiesSparkJob.java @@ -47,17 +47,14 @@ public class DispatchEntitiesSparkJob { String outputPath = parser.get("outputPath"); log.info("outputPath: {}", outputPath); - boolean filterInvisible = Boolean.valueOf(parser.get("filterInvisible")); + boolean filterInvisible = Boolean.parseBoolean(parser.get("filterInvisible")); log.info("filterInvisible: {}", filterInvisible); SparkConf conf = new SparkConf(); runWithSparkSession( conf, isSparkSessionManaged, - spark -> { - HdfsSupport.remove(outputPath, spark.sparkContext().hadoopConfiguration()); - dispatchEntities(spark, inputPath, outputPath, filterInvisible); - }); + spark -> dispatchEntities(spark, inputPath, outputPath, filterInvisible)); } private static void dispatchEntities( @@ -72,7 +69,9 @@ public class DispatchEntitiesSparkJob { String entityType = entry.getKey(); Class clazz = entry.getValue(); + final String entityPath = outputPath + "/" + entityType; if (!entityType.equalsIgnoreCase("relation")) { + HdfsSupport.remove(entityPath, spark.sparkContext().hadoopConfiguration()); Dataset entityDF = spark .read() .schema(Encoders.bean(clazz).schema()) @@ -91,7 +90,7 @@ public class DispatchEntitiesSparkJob { .write() .mode(SaveMode.Overwrite) .option("compression", "gzip") - .json(outputPath + "/" + entityType); + .json(entityPath); } }); } From 395a4af020621633b9e33e05a6ee5ab5f089413b Mon Sep 17 00:00:00 2001 From: Serafeim Chatzopoulos Date: Tue, 12 Sep 2023 22:31:50 +0300 Subject: [PATCH 013/148] Run CC and RAM sequentieally in dhp-impact-indicators WF --- .../impact_indicators/oozie_app/workflow.xml | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/dhp-workflows/dhp-impact-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/impact_indicators/oozie_app/workflow.xml b/dhp-workflows/dhp-impact-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/impact_indicators/oozie_app/workflow.xml index 0d7d29bfe..e43e7cf14 100644 --- a/dhp-workflows/dhp-impact-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/impact_indicators/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-impact-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/impact_indicators/oozie_app/workflow.xml @@ -39,7 +39,8 @@ - ${wf:conf('resume') eq "rankings-start"} + ${wf:conf('resume') eq "cc"} + ${wf:conf('resume') eq "ram"} ${wf:conf('resume') eq "impulse"} ${wf:conf('resume') eq "pagerank"} ${wf:conf('resume') eq "attrank"} @@ -89,18 +90,11 @@ ${nameNode}${wfAppPath}/create_openaire_ranking_graph.py#create_openaire_ranking_graph.py - + - - - - - - - @@ -129,7 +123,7 @@ ${wfAppPath}/bip-ranker/CC.py#CC.py - + @@ -165,14 +159,11 @@ ${wfAppPath}/bip-ranker/TAR.py#TAR.py - + - - - From 9d44418d381b14ab7a6301f1979b2a3af5efbd2f Mon Sep 17 00:00:00 2001 From: Serafeim Chatzopoulos Date: Thu, 14 Sep 2023 18:43:25 +0300 Subject: [PATCH 014/148] Add collecting software code repository URLs --- dhp-workflows/dhp-swh/pom.xml | 104 +++++++++ .../swh/CollectSoftwareRepositoryURLs.java | 211 ++++++++++++++++++ .../dhp/swh/models/LastVisitResponse.java | 40 ++++ .../eu/dnetlib/dhp/swh/input_parameters.json | 26 +++ .../eu/dnetlib/dhp/swh/job.properties | 25 +++ .../eu/dnetlib/dhp/swh/oozie_app/workflow.xml | 101 +++++++++ dhp-workflows/pom.xml | 1 + 7 files changed, 508 insertions(+) create mode 100644 dhp-workflows/dhp-swh/pom.xml create mode 100644 dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectSoftwareRepositoryURLs.java create mode 100644 dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/models/LastVisitResponse.java create mode 100644 dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_parameters.json create mode 100644 dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties create mode 100644 dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml diff --git a/dhp-workflows/dhp-swh/pom.xml b/dhp-workflows/dhp-swh/pom.xml new file mode 100644 index 000000000..501b2aef8 --- /dev/null +++ b/dhp-workflows/dhp-swh/pom.xml @@ -0,0 +1,104 @@ + + + 4.0.0 + + eu.dnetlib.dhp + dhp-workflows + 1.2.5-SNAPSHOT + + dhp-swh + + + + org.apache.spark + spark-core_${scala.binary.version} + + + + org.apache.spark + spark-sql_${scala.binary.version} + + + + eu.dnetlib.dhp + dhp-common + ${project.version} + + + net.sf.saxon + Saxon-HE + + + + + + dom4j + dom4j + + + + xml-apis + xml-apis + + + + jaxen + jaxen + + + + org.apache.hadoop + hadoop-distcp + + + + eu.dnetlib + dnet-actionmanager-api + + + eu.dnetlib + dnet-actionmanager-common + + + eu.dnetlib + dnet-openaireplus-mapping-utils + + + saxonica + saxon + + + saxonica + saxon-dom + + + jgrapht + jgrapht + + + net.sf.ehcache + ehcache + + + org.springframework + spring-test + + + org.apache.* + * + + + apache + * + + + + + + org.apache.httpcomponents + httpclient + 4.5.13 + + + + diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectSoftwareRepositoryURLs.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectSoftwareRepositoryURLs.java new file mode 100644 index 000000000..c91f2bb8c --- /dev/null +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectSoftwareRepositoryURLs.java @@ -0,0 +1,211 @@ + +package eu.dnetlib.dhp.swh; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession; +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.io.IOUtils; +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.FlatMapFunction; +import org.apache.spark.sql.*; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.schema.common.ModelConstants; +import eu.dnetlib.dhp.schema.oaf.*; + +/** + * Creates action sets for Crossref affiliation relations inferred by BIP! + */ +public class CollectSoftwareRepositoryURLs implements Serializable { + + private static final Logger log = LoggerFactory.getLogger(CollectSoftwareRepositoryURLs.class); + // public static final String BIP_AFFILIATIONS_CLASSID = "result:organization:bipinference"; +// public static final String BIP_AFFILIATIONS_CLASSNAME = "Affiliation relation inferred by BIP!"; +// public static final String BIP_INFERENCE_PROVENANCE = "bip:affiliation:crossref"; + private static final String DEFAULT_VISIT_TYPE = "git"; + private static final int CONCURRENT_API_CALLS = 1; + + private static final String SWH_LATEST_VISIT_URL = "https://archive.softwareheritage.org/api/1/origin/%s/visit/latest/"; + + public static void main(String[] args) throws Exception { + + String jsonConfiguration = IOUtils + .toString( + CollectSoftwareRepositoryURLs.class + .getResourceAsStream("/eu/dnetlib/dhp/swh/input_parameters.json")); + + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + parser.parseArgument(args); + + final Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + final String hiveDbName = parser.get("hiveDbName"); + log.info("hiveDbName {}: ", hiveDbName); + + final String outputPath = parser.get("softwareCodeRepositoryURLs"); + log.info("softwareCodeRepositoryURLs {}: ", outputPath); + + final String hiveMetastoreUris = parser.get("hiveMetastoreUris"); + log.info("hiveMetastoreUris: {}", hiveMetastoreUris); + + SparkConf conf = new SparkConf(); + conf.set("hive.metastore.uris", hiveMetastoreUris); + + runWithSparkHiveSession( + conf, + isSparkSessionManaged, + spark -> { + doRun(spark, hiveDbName, outputPath); + }); + } + + private static void doRun(SparkSession spark, String hiveDbName, String outputPath) { + + String queryTemplate = "SELECT distinct coderepositoryurl.value " + + "FROM %s.software " + + "WHERE coderepositoryurl.value IS NOT NULL"; + String query = String.format(queryTemplate, hiveDbName); + + log.info("Hive query to fetch software code URLs: {}", query); + + Dataset df = spark.sql(query); + + // write distinct repository URLs + df + .write() + .mode(SaveMode.Overwrite) +// .option("compression", "gzip") + .csv(outputPath); + } + + private static Dataset readSoftware(SparkSession spark, String inputPath) { + return spark + .read() + .json(inputPath) + .select( + new Column("codeRepositoryUrl.value").as("codeRepositoryUrl"), + new Column("dataInfo.deletedbyinference"), + new Column("dataInfo.invisible")); + } + + private static Dataset filterSoftware(Dataset softwareDF, Integer limit) { + + Dataset df = softwareDF + .where(softwareDF.col("codeRepositoryUrl").isNotNull()) + .where("deletedbyinference = false") + .where("invisible = false") + .drop("deletedbyinference") + .drop("invisible"); + +// TODO remove when done + df = df.limit(limit); + + return df; + } + + public static Dataset makeParallelRequests(SparkSession spark, Dataset softwareDF) { + // TODO replace with coalesce ? + Dataset df = softwareDF.repartition(CONCURRENT_API_CALLS); + + log.info("Number of partitions: {}", df.rdd().getNumPartitions()); + + ObjectMapper objectMapper = new ObjectMapper(); + + List collectedRows = df + .javaRDD() + // max parallelism should be equal to the number of partitions here + .mapPartitions((FlatMapFunction, Row>) partition -> { + List resultRows = new ArrayList<>(); + while (partition.hasNext()) { + Row row = partition.next(); + String url = String.format(SWH_LATEST_VISIT_URL, row.getString(0)); + +// String snapshotId = null; +// String type = null; +// String date = null; + + String responseBody = makeAPICall(url); + TimeUnit.SECONDS.sleep(1); +// Thread.sleep(500); +// if (responseBody != null) { +// LastVisitResponse visitResponse = objectMapper.readValue(responseBody, LastVisitResponse.class); +// snapshotId = visitResponse.getSnapshot(); +// type = visitResponse.getType(); +// date = visitResponse.getDate(); +// } +// resultRows.add(RowFactory.create(url, snapshotId, type, date)); + + resultRows.add(RowFactory.create(url, responseBody)); + } + return resultRows.iterator(); + + }) + .collect(); + + StructType resultSchema = new StructType() + .add("codeRepositoryUrl", DataTypes.StringType) + .add("response", DataTypes.StringType); + +// .add("snapshotId", DataTypes.StringType) +// .add("type", DataTypes.StringType) +// .add("date", DataTypes.StringType); + + // create a DataFrame from the collected rows + return spark.createDataFrame(collectedRows, resultSchema); + } + + private static String makeAPICall(String url) throws IOException { + System.out.println(java.time.LocalDateTime.now()); + + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpGet httpGet = new HttpGet(url); + httpGet + .setHeader( + "Authorization", + "Bearer eyJhbGciOiJIUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJhMTMxYTQ1My1hM2IyLTQwMTUtODQ2Ny05MzAyZjk3MTFkOGEifQ.eyJpYXQiOjE2OTQ2MzYwMjAsImp0aSI6IjkwZjdkNTNjLTQ5YTktNGFiMy1hY2E0LTcwMTViMjEyZTNjNiIsImlzcyI6Imh0dHBzOi8vYXV0aC5zb2Z0d2FyZWhlcml0YWdlLm9yZy9hdXRoL3JlYWxtcy9Tb2Z0d2FyZUhlcml0YWdlIiwiYXVkIjoiaHR0cHM6Ly9hdXRoLnNvZnR3YXJlaGVyaXRhZ2Uub3JnL2F1dGgvcmVhbG1zL1NvZnR3YXJlSGVyaXRhZ2UiLCJzdWIiOiIzMTY5OWZkNC0xNmE0LTQxOWItYTdhMi00NjI5MDY4ZjI3OWEiLCJ0eXAiOiJPZmZsaW5lIiwiYXpwIjoic3doLXdlYiIsInNlc3Npb25fc3RhdGUiOiIzMjYzMzEwMS00ZDRkLTQwMjItODU2NC1iMzNlMTJiNTE3ZDkiLCJzY29wZSI6Im9wZW5pZCBvZmZsaW5lX2FjY2VzcyBwcm9maWxlIGVtYWlsIn0.XHj1VIZu1dZ4Ej32-oU84mFmaox9cLNjXosNxwZM0Xs"); + try (CloseableHttpResponse response = httpClient.execute(httpGet)) { + int statusCode = response.getStatusLine().getStatusCode(); +// if (statusCode != 200) +// return null; + Header[] headers = response.getHeaders("X-RateLimit-Remaining"); + for (Header header : headers) { + System.out + .println( + "Key : " + header.getName() + + " ,Value : " + header.getValue()); + } + HttpEntity entity = response.getEntity(); + if (entity != null) { + return EntityUtils.toString(entity); + } + } + } + return null; + } +} diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/models/LastVisitResponse.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/models/LastVisitResponse.java new file mode 100644 index 000000000..435397590 --- /dev/null +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/models/LastVisitResponse.java @@ -0,0 +1,40 @@ + +package eu.dnetlib.dhp.swh.models; + +import com.cloudera.com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class LastVisitResponse { + + private String type; + + private String date; + + @JsonProperty("snapshot") + private String snapshotId; + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getDate() { + return date; + } + + public void setDate(String date) { + this.date = date; + } + + public String getSnapshot() { + return snapshotId; + } + + public void setSnapshot(String snapshotId) { + this.snapshotId = snapshotId; + } +} diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_parameters.json b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_parameters.json new file mode 100644 index 000000000..dd5432b93 --- /dev/null +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_parameters.json @@ -0,0 +1,26 @@ +[ + { + "paramName": "issm", + "paramLongName": "isSparkSessionManaged", + "paramDescription": "when true will stop SparkSession after job execution", + "paramRequired": false + }, + { + "paramName": "ip", + "paramLongName": "softwareCodeRepositoryURLs", + "paramDescription": "the URL where to store software repository URLs", + "paramRequired": true + }, + { + "paramName": "db", + "paramLongName": "hiveDbName", + "paramDescription": "the target hive database name", + "paramRequired": true + }, + { + "paramName": "hmu", + "paramLongName": "hiveMetastoreUris", + "paramDescription": "the hive metastore uris", + "paramRequired": true + } +] \ No newline at end of file diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties new file mode 100644 index 000000000..a63343aed --- /dev/null +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties @@ -0,0 +1,25 @@ +# hive +hiveDbName=openaire_prod_20230914 +hiveMetastoreUris=thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083 + +# oozie +oozie.action.sharelib.for.spark=spark2 +oozie.use.system.libpath=true +oozie.wf.application.path=${oozieTopWfApplicationPath} +oozie.wf.application.path=${oozieTopWfApplicationPath} +oozieActionShareLibForSpark2=spark2 + +# spark +spark2EventLogDir=/user/spark/spark2ApplicationHistory +spark2ExtraListeners=com.cloudera.spark.lineage.NavigatorAppListener +spark2SqlQueryExecutionListeners=com.cloudera.spark.lineage.NavigatorQueryListener +spark2YarnHistoryServerAddress=http://iis-cdh5-test-gw.ocean.icm.edu.pl:18089 +sparkSqlWarehouseDir=/user/hive/warehouse + +# misc +wfAppPath=${oozieTopWfApplicationPath} +resourceManager=http://iis-cdh5-test-m2.ocean.icm.edu.pl:8088/cluster + +# custom params +softwareCodeRepositoryURLs=${workingDir}/code_repo_urls.csv +resume=collect-software-repository-urls diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml new file mode 100644 index 000000000..9832e5f26 --- /dev/null +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml @@ -0,0 +1,101 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${jobTracker} + ${nameNode} + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + ${wf:conf('startFrom') eq 'collect-software-repository-urls'} + + + + + + + yarn + cluster + Collect software repository URLs + eu.dnetlib.dhp.swh.CollectSoftwareRepositoryURLs + dhp-swh-${projectVersion}.jar + + --executor-memory=${sparkExecutorMemory} + --executor-cores=${sparkExecutorCores} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} + + + --softwareCodeRepositoryURLs${softwareCodeRepositoryURLs} + --hiveDbName${hiveDbName} + --hiveMetastoreUris${hiveMetastoreUris} + + + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/pom.xml b/dhp-workflows/pom.xml index d054ba39b..64f5f2d26 100644 --- a/dhp-workflows/pom.xml +++ b/dhp-workflows/pom.xml @@ -39,6 +39,7 @@ dhp-broker-events dhp-doiboost dhp-impact-indicators + dhp-swh From 9ef971a1464e5d307c407316cda69eb97d6ecb9a Mon Sep 17 00:00:00 2001 From: dimitrispie Date: Tue, 19 Sep 2023 14:25:42 +0300 Subject: [PATCH 015/148] Update step16-createIndicatorsTables.sql Fix int year for: indi_org_openess_year indi_org_fairness_year indi_org_findable_year --- .../scripts/step16-createIndicatorsTables.sql | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql index dd249d371..ae95727a6 100755 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql @@ -444,9 +444,9 @@ CREATE TEMPORARY TABLE ${stats_db_name}.allresults as group by ro.organization, year; create table if not exists ${stats_db_name}.indi_org_fairness_year stored as parquet as - select allresults.year, allresults.organization, result_fair.no_result_fair/allresults.no_allresults org_fairness + select cast(allresults.year as int) year, allresults.organization, result_fair.no_result_fair/allresults.no_allresults org_fairness from ${stats_db_name}.allresults - join ${stats_db_name}.result_fair on result_fair.organization=allresults.organization and result_fair.year=allresults.year; + join ${stats_db_name}.result_fair on result_fair.organization=allresults.organization and cast(result_fair.year as int)=cast(allresults.year as int); DROP table ${stats_db_name}.result_fair purge; DROP table ${stats_db_name}.allresults purge; @@ -465,9 +465,9 @@ CREATE TEMPORARY TABLE ${stats_db_name}.allresults as group by ro.organization, year; create table if not exists ${stats_db_name}.indi_org_findable_year stored as parquet as -select allresults.year, allresults.organization, result_with_pid.no_result_with_pid/allresults.no_allresults org_findable +select cast(allresults.year as int) year, allresults.organization, result_with_pid.no_result_with_pid/allresults.no_allresults org_findable from ${stats_db_name}.allresults - join ${stats_db_name}.result_with_pid on result_with_pid.organization=allresults.organization and result_with_pid.year=allresults.year; + join ${stats_db_name}.result_with_pid on result_with_pid.organization=allresults.organization and cast(result_with_pid.year as int)=cast(allresults.year as int); DROP table ${stats_db_name}.result_with_pid purge; DROP table ${stats_db_name}.allresults purge; @@ -626,16 +626,16 @@ select allsoftware.year, software_oa.organization, software_oa.no_oasoftware/all create table if not exists ${stats_db_name}.indi_org_openess_year stored as parquet as -select cast(allpubsshare.year as int), allpubsshare.organization, +select cast(allpubsshare.year as int) year, allpubsshare.organization, (p+if(isnull(s),0,s)+if(isnull(d),0,d))/(1+(case when s is null then 0 else 1 end) +(case when d is null then 0 else 1 end)) org_openess FROM ${stats_db_name}.allpubsshare - left outer join (select year, organization,d from + left outer join (select cast(year as int), organization,d from ${stats_db_name}.alldatasetssshare) tmp1 on tmp1.organization=allpubsshare.organization and tmp1.year=allpubsshare.year - left outer join (select year, organization,s from + left outer join (select cast(year as int), organization,s from ${stats_db_name}.allsoftwaresshare) tmp2 - on tmp2.organization=allpubsshare.organization and tmp2.year=allpubsshare.year; + on tmp2.organization=allpubsshare.organization and cast(tmp2.year as int)=cast(allpubsshare.year as int); DROP TABLE ${stats_db_name}.pubs_oa purge; DROP TABLE ${stats_db_name}.datasets_oa purge; From 76476cdfb63c2c0570f0669e50970d43e9aecb16 Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Wed, 20 Sep 2023 10:33:14 +0200 Subject: [PATCH 016/148] Added maven repo for dependencies that are not in maven central --- .../eu/dnetlib/pace/util/DiffPatchMatch.java | 17 +++++++++++++++++ pom.xml | 10 ++++++++++ 2 files changed, 27 insertions(+) diff --git a/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java b/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java index 84d49bd5c..cfd9acd70 100644 --- a/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java +++ b/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java @@ -1,6 +1,23 @@ package eu.dnetlib.pace.util; +/* + * Diff Match and Patch + * Copyright 2018 The diff-match-patch Authors. + * https://github.com/google/diff-match-patch + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /* * Diff Match and Patch * Copyright 2018 The diff-match-patch Authors. diff --git a/pom.xml b/pom.xml index c6b65e27a..9cd82a343 100644 --- a/pom.xml +++ b/pom.xml @@ -112,6 +112,16 @@ https://maven.d4science.org/nexus/content/repositories/dnet-deps default + + maven-restlet + Restlet repository + https://maven.restlet.talend.com + + + conjars + conjars + https://conjars.wensel.net/repo/ + From cc7204a08904a8b23ac8bd30be5f829ff93e7cc0 Mon Sep 17 00:00:00 2001 From: Alessia Bardi Date: Tue, 19 Sep 2023 13:38:25 +0200 Subject: [PATCH 017/148] tests for d4science catalog --- .../dnetlib/dhp/oa/graph/raw/MappersTest.java | 22 +++++ .../dhp/oa/graph/raw/d4science-1-training.xml | 93 +++++++++++++++++++ .../dhp/oa/graph/raw/d4science-2-dataset.xml | 72 ++++++++++++++ .../oa/provision/XmlRecordFactoryTest.java | 52 ++++++++++- .../oa/provision/d4science-1-training.json | 1 + .../dhp/oa/provision/d4science-2-dataset.json | 1 + 6 files changed, 237 insertions(+), 4 deletions(-) create mode 100644 dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/d4science-1-training.xml create mode 100644 dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/d4science-2-dataset.xml create mode 100644 dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/d4science-1-training.json create mode 100644 dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/d4science-2-dataset.json diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java index 894ed33f7..b506d3a62 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java @@ -1067,6 +1067,28 @@ class MappersTest { System.out.println("***************"); } + @Test + public void testD4ScienceTraining() throws IOException { + final String xml = IOUtils + .toString(Objects.requireNonNull(getClass().getResourceAsStream("d4science-1-training.xml"))); + final List list = new OdfToOafMapper(vocs, false, true).processMdRecord(xml); + final OtherResearchProduct trainingMaterial = (OtherResearchProduct) list.get(0); + System.out.println("***************"); + System.out.println(new ObjectMapper().writeValueAsString(trainingMaterial)); + System.out.println("***************"); + } + + @Test + public void testD4ScienceDataset() throws IOException { + final String xml = IOUtils + .toString(Objects.requireNonNull(getClass().getResourceAsStream("d4science-2-dataset.xml"))); + final List list = new OdfToOafMapper(vocs, false, true).processMdRecord(xml); + final Dataset trainingMaterial = (Dataset) list.get(0); + System.out.println("***************"); + System.out.println(new ObjectMapper().writeValueAsString(trainingMaterial)); + System.out.println("***************"); + } + @Test void testNotWellFormed() throws IOException { final String xml = IOUtils diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/d4science-1-training.xml b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/d4science-1-training.xml new file mode 100644 index 000000000..91f9f9118 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/d4science-1-training.xml @@ -0,0 +1,93 @@ + + + + alessia_____::104c2d4ba8878c16fa824dce5b1bea57 + 12d8f77e-d66f-46f5-8d88-af7db23bc4c9 + 2023-09-08T10:12:35.864+02:00 + alessia_____ + 2023-09-08T11:31:45.692+02:00 + + + + http://data.d4science.org/ctlg/ResourceCatalogue/visual_analytics_for_data_scientists + + + + BRAGHIERI MARCO + + + + Visual Analytics for Data Scientists + + SoBigData++ + + + + + TrainingMaterial + + Participants to this module shall + - Learn the principles and rules underlying the design of visual data + representations and human-computer interactions + - Understand, adapt and apply representative visual analytics methods and systems for diverse types + of data and problems + - Analyse and evaluate the structure and properties + of data to select or devise appropriate methods for data exploration + - Combine visualization, interactive techniques, and computational + processing to develop practical data analysis for problem solving + + (This teaching material on Visual Analytics for Data Scientists is part of a MSc module at City University London). + + The author did not intend to violate any copyright on figures or content. In case you are the legal owner of any copyrighted content, please contact info@sobigdata.eu and we will immediately remove it + + + Visual analytics + + + Slides + Other + PDF + PDF + PDF + PDF + PDF + PDF + PDF + PDF + PDF + PDF + ZIP + + + OPEN + 0010 + + + + other-open + corda__h2020::871042 + + + + + https%3A%2F%2Fapi.d4science.org%2Fcatalogue%2Fitems + + + + + + + false + false + 0.9 + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/d4science-2-dataset.xml b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/d4science-2-dataset.xml new file mode 100644 index 000000000..48ceb6c13 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/d4science-2-dataset.xml @@ -0,0 +1,72 @@ + + + + alessia_____::028879484548f4e1c630e1c503e35231 + 4fed018e-c2ff-4afa-b7b5-1ca1beebf850 + 2023-09-08T12:14:27.615+02:00 + alessia_____ + 2023-09-08T12:14:51.7+02:00 + + + + http://data.d4science.org/ctlg/ResourceCatalogue/city-to-city_migration + + + + + + Pappalardo, Luca + + 0000-0002-1547-6007 + + + + City-to-city migration + + SoBigData++ + + + 2018-02-15 + + Dataset + + Census data recording the migration of people between metropolitan areas in + the US + + + Human Mobility data + + + + OPEN + 0021 + 2018-02-15 + + + AFL-3.0 + corda__h2020::871042 + + + + + https%3A%2F%2Fapi.d4science.org%2Fcatalogue%2Fitems + + + + + + + false + false + 0.9 + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java index 761539780..88bffd0e7 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java +++ b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java @@ -24,10 +24,7 @@ import eu.dnetlib.dhp.oa.provision.model.RelatedEntity; import eu.dnetlib.dhp.oa.provision.model.RelatedEntityWrapper; import eu.dnetlib.dhp.oa.provision.utils.ContextMapper; import eu.dnetlib.dhp.oa.provision.utils.XmlRecordFactory; -import eu.dnetlib.dhp.schema.oaf.Datasource; -import eu.dnetlib.dhp.schema.oaf.Project; -import eu.dnetlib.dhp.schema.oaf.Publication; -import eu.dnetlib.dhp.schema.oaf.Relation; +import eu.dnetlib.dhp.schema.oaf.*; public class XmlRecordFactoryTest { @@ -196,4 +193,51 @@ public class XmlRecordFactoryTest { assertEquals("dnet:pid_types", ((Element) pids.get(0)).attribute("schemeid").getValue()); assertEquals("dnet:pid_types", ((Element) pids.get(0)).attribute("schemename").getValue()); } + + @Test + public void testD4ScienceTraining() throws DocumentException, IOException { + final ContextMapper contextMapper = new ContextMapper(); + + final XmlRecordFactory xmlRecordFactory = new XmlRecordFactory(contextMapper, false, + XmlConverterJob.schemaLocation); + + final OtherResearchProduct p = OBJECT_MAPPER + .readValue( + IOUtils.toString(getClass().getResourceAsStream("d4science-1-training.json")), + OtherResearchProduct.class); + + final String xml = xmlRecordFactory.build(new JoinedEntity<>(p)); + + assertNotNull(xml); + + final Document doc = new SAXReader().read(new StringReader(xml)); + + assertNotNull(doc); + System.out.println(doc.asXML()); + + } + + @Test + public void testD4ScienceDataset() throws DocumentException, IOException { + final ContextMapper contextMapper = new ContextMapper(); + + final XmlRecordFactory xmlRecordFactory = new XmlRecordFactory(contextMapper, false, + XmlConverterJob.schemaLocation); + + final OtherResearchProduct p = OBJECT_MAPPER + .readValue( + IOUtils.toString(getClass().getResourceAsStream("d4science-2-dataset.json")), + OtherResearchProduct.class); + + final String xml = xmlRecordFactory.build(new JoinedEntity<>(p)); + + assertNotNull(xml); + + final Document doc = new SAXReader().read(new StringReader(xml)); + + assertNotNull(doc); + System.out.println(doc.asXML()); + + } + } diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/d4science-1-training.json b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/d4science-1-training.json new file mode 100644 index 000000000..3ce397f10 --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/d4science-1-training.json @@ -0,0 +1 @@ +{"collectedfrom":[{"key":"10|alessia_____::6332e88a4c7dba6f7743d3a7a0c6ea2c","value":"Alessia","dataInfo":null}],"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1694165542374,"id":"50|alessia_____::104c2d4ba8878c16fa824dce5b1bea57","originalId":["12d8f77e-d66f-46f5-8d88-af7db23bc4c9","50|alessia_____::104c2d4ba8878c16fa824dce5b1bea57"],"pid":[],"dateofcollection":"2023-09-08T10:12:35.864+02:00","dateoftransformation":"2023-09-08T11:31:45.692+02:00","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2023-09-08T10:12:35.864+02:00","altered":true,"baseURL":"https%3A%2F%2Fapi.d4science.org%2Fcatalogue%2Fitems","identifier":"","datestamp":"","metadataNamespace":""}},"measures":null,"processingchargeamount":null,"processingchargecurrency":null,"author":[{"fullname":"BRAGHIERI MARCO","name":"","surname":"","rank":1,"pid":[],"affiliation":[]}],"resulttype":{"classid":"other","classname":"other","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"","classname":"","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Visual analytics","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Visual Analytics for Data Scientists","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[{"value":"","qualifier":{"classid":"Issued","classname":"Issued","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"description":[{"value":"Participants to this module shall\n - Learn the principles and rules underlying the design of visual data\n representations and human-computer interactions\n - Understand, adapt and apply representative visual analytics methods and systems for diverse types\n of data and problems\n - Analyse and evaluate the structure and properties\n of data to select or devise appropriate methods for data exploration\n - Combine visualization, interactive techniques, and computational\n processing to develop practical data analysis for problem solving\n\n (This teaching material on Visual Analytics for Data Scientists is part of a MSc module at City University London).\n\n The author did not intend to violate any copyright on figures or content. In case you are the legal owner of any copyrighted content, please contact info@sobigdata.eu and we will immediately remove it","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":null,"publisher":{"value":"SoBigData++","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"embargoenddate":null,"source":[],"fulltext":[],"format":[{"value":"Slides","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Other","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"PDF","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"ZIP","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"contributor":[],"resourcetype":{"classid":"TrainingMaterial","classname":"TrainingMaterial","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"coverage":[],"bestaccessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"context":[],"externalReference":[],"instance":[{"license":{"value":"other-open","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes","openAccessRoute":null},"instancetype":{"classid":"0010","classname":"Lecture","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"10|alessia_____::6332e88a4c7dba6f7743d3a7a0c6ea2c","value":"Alessia","dataInfo":null},"url":["http://data.d4science.org/ctlg/ResourceCatalogue/visual_analytics_for_data_scientists"],"distributionlocation":null,"collectedfrom":{"key":"10|alessia_____::6332e88a4c7dba6f7743d3a7a0c6ea2c","value":"Alessia","dataInfo":null},"pid":[],"alternateIdentifier":[],"dateofacceptance":null,"processingchargeamount":null,"processingchargecurrency":null,"refereed":{"classid":"","classname":"","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"measures":null,"fulltext":null}],"eoscifguidelines":[],"contactperson":[],"contactgroup":[],"tool":[]} diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/d4science-2-dataset.json b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/d4science-2-dataset.json new file mode 100644 index 000000000..ea8465e36 --- /dev/null +++ b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/d4science-2-dataset.json @@ -0,0 +1 @@ +{"collectedfrom":[{"key":"10|alessia_____::6332e88a4c7dba6f7743d3a7a0c6ea2c","value":"Alessia","dataInfo":null}],"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1694507584675,"id":"50|alessia_____::028879484548f4e1c630e1c503e35231","originalId":["4fed018e-c2ff-4afa-b7b5-1ca1beebf850","50|alessia_____::028879484548f4e1c630e1c503e35231"],"pid":[],"dateofcollection":"2023-09-08T12:14:27.615+02:00","dateoftransformation":"2023-09-08T12:14:51.7+02:00","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2023-09-08T12:14:27.615+02:00","altered":true,"baseURL":"https%3A%2F%2Fapi.d4science.org%2Fcatalogue%2Fitems","identifier":"","datestamp":"","metadataNamespace":""}},"measures":null,"processingchargeamount":null,"processingchargecurrency":null,"author":[{"fullname":"Pappalardo, Luca","name":"Luca","surname":"Pappalardo","rank":1,"pid":[{"value":"0000-0002-1547-6007","qualifier":{"classid":"orcid_pending","classname":"Open Researcher and Contributor ID","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"affiliation":[]}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"","classname":"","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Human Mobility data","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"City-to-city migration","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[{"value":"2018-02-15","qualifier":{"classid":"Issued","classname":"Issued","schemeid":"dnet:dataCite_date","schemename":"dnet:dataCite_date"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"description":[{"value":"Census data recording the migration of people between metropolitan areas in\n the US","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-02-15","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":{"value":"SoBigData++","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"coverage":[],"bestaccessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"context":[],"externalReference":[],"instance":[{"license":{"value":"AFL-3.0","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes","openAccessRoute":null},"instancetype":{"classid":"0021","classname":"Dataset","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"10|alessia_____::6332e88a4c7dba6f7743d3a7a0c6ea2c","value":"Alessia","dataInfo":null},"url":["http://data.d4science.org/ctlg/ResourceCatalogue/city-to-city_migration"],"distributionlocation":null,"collectedfrom":{"key":"10|alessia_____::6332e88a4c7dba6f7743d3a7a0c6ea2c","value":"Alessia","dataInfo":null},"pid":[],"alternateIdentifier":[],"dateofacceptance":{"value":"2018-02-15","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":{"classid":"","classname":"","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"measures":null,"fulltext":null}],"eoscifguidelines":[],"storagedate":{"value":"2018-02-15","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} From 0935d7757cfc9c4efce6500e3e8f02792d56c2ad Mon Sep 17 00:00:00 2001 From: Alessia Bardi Date: Tue, 19 Sep 2023 14:47:01 +0200 Subject: [PATCH 018/148] Use v5 of the UNIBI Gold ISSN list in test --- .../eu/dnetlib/dhp/oa/graph/hostedbymap/DownloadCsvTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/hostedbymap/DownloadCsvTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/hostedbymap/DownloadCsvTest.java index 48f1e0c06..9bd32968a 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/hostedbymap/DownloadCsvTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/hostedbymap/DownloadCsvTest.java @@ -49,7 +49,7 @@ public class DownloadCsvTest { @Test void getUnibiFileTest() throws CollectorException, IOException, ClassNotFoundException { - String fileURL = "https://pub.uni-bielefeld.de/download/2944717/2944718/issn_gold_oa_version_4.csv"; + String fileURL = "https://pub.uni-bielefeld.de/download/2944717/2944718/issn_gold_oa_version_5.csv"; final String outputFile = workingDir + "/unibi_gold.json"; new DownloadCSV() From ed9c81a0b7a12ac2b337843ef42f63ca62b1f063 Mon Sep 17 00:00:00 2001 From: Serafeim Chatzopoulos Date: Wed, 27 Sep 2023 19:00:54 +0300 Subject: [PATCH 019/148] Add steps to collect last visit data && archive not found repository URLs --- .../java/eu/dnetlib/dhp/common/Constants.java | 1 + .../common/collection/HttpClientParams.java | 37 +++- .../dhp/common/collection/HttpConnector2.java | 7 + dhp-workflows/dhp-swh/pom.xml | 6 + .../dhp/swh/ArchiveRepositoryURLs.java | 137 +++++++++++++++ .../swh/CollectLastVisitRepositoryData.java | 120 +++++++++++++ .../swh/CollectSoftwareRepositoryURLs.java | 158 ++---------------- ...tVisitResponse.java => LastVisitData.java} | 10 +- .../dnetlib/dhp/swh/utils/SWHConnection.java | 138 +++++++++++++++ .../dnetlib/dhp/swh/utils/SWHConstants.java | 13 ++ .../eu/dnetlib/dhp/swh/utils/SWHUtils.java | 94 +++++++++++ .../swh/input_archive_repository_urls.json | 26 +++ ...ut_collect_last_visit_repository_data.json | 38 +++++ ...put_collect_software_repository_urls.json} | 2 +- .../eu/dnetlib/dhp/swh/job.properties | 22 +-- .../dhp/swh/oozie_app/config-default.xml | 50 ++++++ .../eu/dnetlib/dhp/swh/oozie_app/workflow.xml | 109 ++++++------ .../dhp/swh/ArchiveRepositoryURLsTest.java | 35 ++++ .../eu/dnetlib/dhp/swh/SWHConnectionTest.java | 57 +++++++ .../dhp/swh/lastVisitDataToArchive.csv | 6 + 20 files changed, 848 insertions(+), 218 deletions(-) create mode 100644 dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/ArchiveRepositoryURLs.java create mode 100644 dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectLastVisitRepositoryData.java rename dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/models/{LastVisitResponse.java => LastVisitData.java} (81%) create mode 100644 dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConnection.java create mode 100644 dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConstants.java create mode 100644 dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHUtils.java create mode 100644 dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_archive_repository_urls.json create mode 100644 dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_collect_last_visit_repository_data.json rename dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/{input_parameters.json => input_collect_software_repository_urls.json} (96%) create mode 100644 dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/config-default.xml create mode 100644 dhp-workflows/dhp-swh/src/test/java/eu/dnetlib/dhp/swh/ArchiveRepositoryURLsTest.java create mode 100644 dhp-workflows/dhp-swh/src/test/java/eu/dnetlib/dhp/swh/SWHConnectionTest.java create mode 100644 dhp-workflows/dhp-swh/src/test/resources/eu/dnetlib/dhp/swh/lastVisitDataToArchive.csv diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/Constants.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/Constants.java index 4f2c6341e..0477d6399 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/Constants.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/Constants.java @@ -51,6 +51,7 @@ public class Constants { public static final String RETRY_DELAY = "retryDelay"; public static final String CONNECT_TIMEOUT = "connectTimeOut"; public static final String READ_TIMEOUT = "readTimeOut"; + public static final String REQUEST_METHOD = "requestMethod"; public static final String FROM_DATE_OVERRIDE = "fromDateOverride"; public static final String UNTIL_DATE_OVERRIDE = "untilDateOverride"; diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/collection/HttpClientParams.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/collection/HttpClientParams.java index 6fcec00dd..55f9ceb8b 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/collection/HttpClientParams.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/collection/HttpClientParams.java @@ -1,6 +1,9 @@ package eu.dnetlib.dhp.common.collection; +import java.util.HashMap; +import java.util.Map; + /** * Bundles the http connection parameters driving the client behaviour. */ @@ -13,6 +16,8 @@ public class HttpClientParams { public static int _connectTimeOut = 10; // seconds public static int _readTimeOut = 30; // seconds + public static String _requestMethod = "GET"; + /** * Maximum number of allowed retires before failing */ @@ -38,17 +43,30 @@ public class HttpClientParams { */ private int readTimeOut; + /** + * Custom http headers + */ + private Map headers; + + /** + * Request method (i.e., GET, POST etc) + */ + private String requestMethod; + + public HttpClientParams() { - this(_maxNumberOfRetry, _requestDelay, _retryDelay, _connectTimeOut, _readTimeOut); + this(_maxNumberOfRetry, _requestDelay, _retryDelay, _connectTimeOut, _readTimeOut, new HashMap<>(), _requestMethod); } public HttpClientParams(int maxNumberOfRetry, int requestDelay, int retryDelay, int connectTimeOut, - int readTimeOut) { + int readTimeOut, Map headers, String requestMethod) { this.maxNumberOfRetry = maxNumberOfRetry; this.requestDelay = requestDelay; this.retryDelay = retryDelay; this.connectTimeOut = connectTimeOut; this.readTimeOut = readTimeOut; + this.headers = headers; + this.requestMethod = requestMethod; } public int getMaxNumberOfRetry() { @@ -91,4 +109,19 @@ public class HttpClientParams { this.readTimeOut = readTimeOut; } + public Map getHeaders() { + return headers; + } + + public void setHeaders(Map headers) { + this.headers = headers; + } + + public String getRequestMethod() { + return requestMethod; + } + + public void setRequestMethod(String requestMethod) { + this.requestMethod = requestMethod; + } } diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/collection/HttpConnector2.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/collection/HttpConnector2.java index dd46ab1f4..905457bcd 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/collection/HttpConnector2.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/collection/HttpConnector2.java @@ -107,7 +107,14 @@ public class HttpConnector2 { urlConn.setReadTimeout(getClientParams().getReadTimeOut() * 1000); urlConn.setConnectTimeout(getClientParams().getConnectTimeOut() * 1000); urlConn.addRequestProperty(HttpHeaders.USER_AGENT, userAgent); + urlConn.setRequestMethod(getClientParams().getRequestMethod()); + // if provided, add custom headers + if (!getClientParams().getHeaders().isEmpty()) { + for (Map.Entry headerEntry : getClientParams().getHeaders().entrySet()) { + urlConn.addRequestProperty(headerEntry.getKey(), headerEntry.getValue()); + } + } if (log.isDebugEnabled()) { logHeaderFields(urlConn); } diff --git a/dhp-workflows/dhp-swh/pom.xml b/dhp-workflows/dhp-swh/pom.xml index 501b2aef8..80fff4587 100644 --- a/dhp-workflows/dhp-swh/pom.xml +++ b/dhp-workflows/dhp-swh/pom.xml @@ -99,6 +99,12 @@ httpclient 4.5.13 + + org.datanucleus + datanucleus-core + 3.2.10 + compile + diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/ArchiveRepositoryURLs.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/ArchiveRepositoryURLs.java new file mode 100644 index 000000000..7b3b74d9e --- /dev/null +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/ArchiveRepositoryURLs.java @@ -0,0 +1,137 @@ + +package eu.dnetlib.dhp.swh; + +import com.fasterxml.jackson.databind.ObjectMapper; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.collection.CollectorException; +import eu.dnetlib.dhp.common.collection.HttpClientParams; +import eu.dnetlib.dhp.swh.models.LastVisitData; +import eu.dnetlib.dhp.swh.utils.SWHConnection; +import eu.dnetlib.dhp.swh.utils.SWHConstants; +import eu.dnetlib.dhp.swh.utils.SWHUtils; +import org.apache.commons.cli.ParseException; +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URL; +import java.util.Date; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +import static eu.dnetlib.dhp.common.Constants.REQUEST_METHOD; +import static eu.dnetlib.dhp.utils.DHPUtils.getHadoopConfiguration; + +/** + * Sends archive requests to the SWH API for those software repository URLs that are missing from them + * + * @author Serafeim Chatzopoulos + */ +public class ArchiveRepositoryURLs { + + private static final Logger log = LoggerFactory.getLogger(ArchiveRepositoryURLs.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static SWHConnection swhConnection = null; + + public static void main(final String[] args) throws IOException, ParseException { + final ArgumentApplicationParser argumentParser = new ArgumentApplicationParser( + IOUtils + .toString( + CollectLastVisitRepositoryData.class + .getResourceAsStream( + "/eu/dnetlib/dhp/swh/input_archive_repository_urls.json"))); + argumentParser.parseArgument(args); + + final String hdfsuri = argumentParser.get("namenode"); + log.info("hdfsURI: {}", hdfsuri); + + final String inputPath = argumentParser.get("lastVisitsPath"); + log.info("inputPath: {}", inputPath); + + final String outputPath = argumentParser.get("archiveRequestsPath"); + log.info("outputPath: {}", outputPath); + + final Integer archiveThresholdInDays = Integer.parseInt(argumentParser.get("archiveThresholdInDays")); + log.info("archiveThresholdInDays: {}", archiveThresholdInDays); + + final HttpClientParams clientParams = SWHUtils.getClientParams(argumentParser); + + swhConnection = new SWHConnection(clientParams); + + final FileSystem fs = FileSystem.get(getHadoopConfiguration(hdfsuri)); + + archive(fs, inputPath, outputPath, archiveThresholdInDays); + + } + + private static void archive(FileSystem fs, String inputPath, String outputPath, Integer archiveThresholdInDays) throws IOException { + + SequenceFile.Reader fr = SWHUtils.getSequenceFileReader(fs, inputPath); + SequenceFile.Writer fw = SWHUtils.getSequenceFileWriter(fs, outputPath); + + // Create key and value objects to hold data + Text repoUrl = new Text(); + Text lastVisitData = new Text(); + + // Read key-value pairs from the SequenceFile and handle appropriately + while (fr.next(repoUrl, lastVisitData)) { + + String response = handleRecord(repoUrl.toString(), lastVisitData.toString(), archiveThresholdInDays); + + // response is equal to null when no need for request + if (response != null) { + SWHUtils.appendToSequenceFile(fw, repoUrl.toString(), response); + } + + } + + // Close readers + fw.close(); + fr.close(); + } + + public static String handleRecord(String repoUrl, String lastVisitData, Integer archiveThresholdInDays) throws IOException { + System.out.println("Key: " + repoUrl + ", Value: " + lastVisitData); + + LastVisitData lastVisit = OBJECT_MAPPER.readValue(lastVisitData, LastVisitData.class); + + // perform an archive request when no repoUrl was not found in previous step + if (lastVisit.getSnapshot() != null) { + + // OR last visit was before (now() - archiveThresholdInDays) + long diffInMillies = Math.abs((new Date()).getTime() - lastVisit.getDate().getTime()); + long diffInDays = TimeUnit.DAYS.convert(diffInMillies, TimeUnit.MILLISECONDS); + + if (archiveThresholdInDays >= diffInDays) { + return null; + } + } + + // if last visit data are available, re-use version control type, else use the default one (i.e., git) + String visitType = Optional + .ofNullable(lastVisit.getType()) + .orElse(SWHConstants.DEFAULT_VISIT_TYPE); + + URL url = new URL(String.format(SWHConstants.SWH_ARCHIVE_URL, visitType, repoUrl.trim())); + System.out.println(url.toString()); + + String response; + try { + response = swhConnection.call(url.toString()); + } catch (CollectorException e) { + log.info("Error in request: {}", url); + response = "{}"; + } + + return response; + + } + + + +} diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectLastVisitRepositoryData.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectLastVisitRepositoryData.java new file mode 100644 index 000000000..c4b6412b5 --- /dev/null +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectLastVisitRepositoryData.java @@ -0,0 +1,120 @@ + +package eu.dnetlib.dhp.swh; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.collection.CollectorException; +import eu.dnetlib.dhp.common.collection.HttpClientParams; +import eu.dnetlib.dhp.swh.utils.SWHConnection; +import eu.dnetlib.dhp.swh.utils.SWHConstants; +import eu.dnetlib.dhp.swh.utils.SWHUtils; +import org.apache.commons.cli.ParseException; +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.charset.StandardCharsets; + +import static eu.dnetlib.dhp.utils.DHPUtils.getHadoopConfiguration; + +/** + * Given a file with software repository URLs, this class + * collects last visit data from the Software Heritage API. + * + * @author Serafeim Chatzopoulos + */ +public class CollectLastVisitRepositoryData { + + private static final Logger log = LoggerFactory.getLogger(CollectLastVisitRepositoryData.class); + private static SWHConnection swhConnection = null; + + public static void main(final String[] args) + throws IOException, ParseException, InterruptedException, URISyntaxException, CollectorException { + final ArgumentApplicationParser argumentParser = new ArgumentApplicationParser( + IOUtils + .toString( + CollectLastVisitRepositoryData.class + .getResourceAsStream( + "/eu/dnetlib/dhp/swh/input_collect_last_visit_repository_data.json"))); + argumentParser.parseArgument(args); + + log.info("Java Xmx: {}m", Runtime.getRuntime().maxMemory() / (1024 * 1024)); + + final String hdfsuri = argumentParser.get("namenode"); + log.info("hdfsURI: {}", hdfsuri); + + final String inputPath = argumentParser.get("softwareCodeRepositoryURLs"); + log.info("inputPath: {}", inputPath); + + final String outputPath = argumentParser.get("lastVisitsPath"); + log.info("outputPath: {}", outputPath); + + final HttpClientParams clientParams = SWHUtils.getClientParams(argumentParser); + + swhConnection = new SWHConnection(clientParams); + + final FileSystem fs = FileSystem.get(getHadoopConfiguration(hdfsuri)); + + collect(fs, inputPath, outputPath); + + fs.close(); + } + + private static void collect(FileSystem fs, String inputPath, String outputPath) + throws IOException { + + SequenceFile.Writer fw = SWHUtils.getSequenceFileWriter(fs, outputPath); + + // Specify the HDFS directory path you want to read + Path directoryPath = new Path(inputPath); + + // List all files in the directory + FileStatus[] partStatuses = fs.listStatus(directoryPath); + + for (FileStatus partStatus : partStatuses) { + + // Check if it's a file (not a directory) + if (partStatus.isFile()) { + handleFile(fs, partStatus.getPath(), fw); + } + + } + + fw.close(); + } + + private static void handleFile(FileSystem fs, Path partInputPath, SequenceFile.Writer fw) + throws IOException { + + BufferedReader br = SWHUtils.getFileReader(fs, partInputPath); + + String repoUrl; + while ((repoUrl = br.readLine()) != null) { + + URL url = new URL(String.format(SWHConstants.SWH_LATEST_VISIT_URL, repoUrl.trim())); + + String response; + try { + response = swhConnection.call(url.toString()); + } catch (CollectorException e) { + log.info("Error in request: {}", url); + response = "{}"; + } + + SWHUtils.appendToSequenceFile(fw, repoUrl, response); + } + + br.close(); + } + +} diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectSoftwareRepositoryURLs.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectSoftwareRepositoryURLs.java index c91f2bb8c..f93280b5e 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectSoftwareRepositoryURLs.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectSoftwareRepositoryURLs.java @@ -1,60 +1,37 @@ package eu.dnetlib.dhp.swh; -import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession; -import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; - -import java.io.IOException; -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.TimeUnit; - +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.schema.oaf.Result; import org.apache.commons.io.IOUtils; -import org.apache.http.Header; -import org.apache.http.HttpEntity; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.util.EntityUtils; import org.apache.spark.SparkConf; -import org.apache.spark.api.java.function.FlatMapFunction; -import org.apache.spark.sql.*; import org.apache.spark.sql.Dataset; -import org.apache.spark.sql.types.DataTypes; -import org.apache.spark.sql.types.StructType; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SaveMode; +import org.apache.spark.sql.SparkSession; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.Serializable; +import java.util.Optional; -import eu.dnetlib.dhp.application.ArgumentApplicationParser; -import eu.dnetlib.dhp.schema.common.ModelConstants; -import eu.dnetlib.dhp.schema.oaf.*; +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession; /** - * Creates action sets for Crossref affiliation relations inferred by BIP! + * Collects unique software repository URLs in the Graph using Hive + * + * @author Serafeim Chatzopoulos */ public class CollectSoftwareRepositoryURLs implements Serializable { private static final Logger log = LoggerFactory.getLogger(CollectSoftwareRepositoryURLs.class); - // public static final String BIP_AFFILIATIONS_CLASSID = "result:organization:bipinference"; -// public static final String BIP_AFFILIATIONS_CLASSNAME = "Affiliation relation inferred by BIP!"; -// public static final String BIP_INFERENCE_PROVENANCE = "bip:affiliation:crossref"; - private static final String DEFAULT_VISIT_TYPE = "git"; - private static final int CONCURRENT_API_CALLS = 1; - - private static final String SWH_LATEST_VISIT_URL = "https://archive.softwareheritage.org/api/1/origin/%s/visit/latest/"; public static void main(String[] args) throws Exception { String jsonConfiguration = IOUtils .toString( CollectSoftwareRepositoryURLs.class - .getResourceAsStream("/eu/dnetlib/dhp/swh/input_parameters.json")); + .getResourceAsStream("/eu/dnetlib/dhp/swh/input_collect_software_repository_urls.json")); final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); parser.parseArgument(args); @@ -89,7 +66,10 @@ public class CollectSoftwareRepositoryURLs implements Serializable { String queryTemplate = "SELECT distinct coderepositoryurl.value " + "FROM %s.software " + - "WHERE coderepositoryurl.value IS NOT NULL"; + "WHERE coderepositoryurl.value IS NOT NULL " + + "AND datainfo.deletedbyinference = FALSE " + + "AND datainfo.invisible = FALSE " + + "LIMIT 1000"; // TODO remove String query = String.format(queryTemplate, hiveDbName); log.info("Hive query to fetch software code URLs: {}", query); @@ -100,112 +80,6 @@ public class CollectSoftwareRepositoryURLs implements Serializable { df .write() .mode(SaveMode.Overwrite) -// .option("compression", "gzip") .csv(outputPath); } - - private static Dataset readSoftware(SparkSession spark, String inputPath) { - return spark - .read() - .json(inputPath) - .select( - new Column("codeRepositoryUrl.value").as("codeRepositoryUrl"), - new Column("dataInfo.deletedbyinference"), - new Column("dataInfo.invisible")); - } - - private static Dataset filterSoftware(Dataset softwareDF, Integer limit) { - - Dataset df = softwareDF - .where(softwareDF.col("codeRepositoryUrl").isNotNull()) - .where("deletedbyinference = false") - .where("invisible = false") - .drop("deletedbyinference") - .drop("invisible"); - -// TODO remove when done - df = df.limit(limit); - - return df; - } - - public static Dataset makeParallelRequests(SparkSession spark, Dataset softwareDF) { - // TODO replace with coalesce ? - Dataset df = softwareDF.repartition(CONCURRENT_API_CALLS); - - log.info("Number of partitions: {}", df.rdd().getNumPartitions()); - - ObjectMapper objectMapper = new ObjectMapper(); - - List collectedRows = df - .javaRDD() - // max parallelism should be equal to the number of partitions here - .mapPartitions((FlatMapFunction, Row>) partition -> { - List resultRows = new ArrayList<>(); - while (partition.hasNext()) { - Row row = partition.next(); - String url = String.format(SWH_LATEST_VISIT_URL, row.getString(0)); - -// String snapshotId = null; -// String type = null; -// String date = null; - - String responseBody = makeAPICall(url); - TimeUnit.SECONDS.sleep(1); -// Thread.sleep(500); -// if (responseBody != null) { -// LastVisitResponse visitResponse = objectMapper.readValue(responseBody, LastVisitResponse.class); -// snapshotId = visitResponse.getSnapshot(); -// type = visitResponse.getType(); -// date = visitResponse.getDate(); -// } -// resultRows.add(RowFactory.create(url, snapshotId, type, date)); - - resultRows.add(RowFactory.create(url, responseBody)); - } - return resultRows.iterator(); - - }) - .collect(); - - StructType resultSchema = new StructType() - .add("codeRepositoryUrl", DataTypes.StringType) - .add("response", DataTypes.StringType); - -// .add("snapshotId", DataTypes.StringType) -// .add("type", DataTypes.StringType) -// .add("date", DataTypes.StringType); - - // create a DataFrame from the collected rows - return spark.createDataFrame(collectedRows, resultSchema); - } - - private static String makeAPICall(String url) throws IOException { - System.out.println(java.time.LocalDateTime.now()); - - try (CloseableHttpClient httpClient = HttpClients.createDefault()) { - HttpGet httpGet = new HttpGet(url); - httpGet - .setHeader( - "Authorization", - "Bearer eyJhbGciOiJIUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJhMTMxYTQ1My1hM2IyLTQwMTUtODQ2Ny05MzAyZjk3MTFkOGEifQ.eyJpYXQiOjE2OTQ2MzYwMjAsImp0aSI6IjkwZjdkNTNjLTQ5YTktNGFiMy1hY2E0LTcwMTViMjEyZTNjNiIsImlzcyI6Imh0dHBzOi8vYXV0aC5zb2Z0d2FyZWhlcml0YWdlLm9yZy9hdXRoL3JlYWxtcy9Tb2Z0d2FyZUhlcml0YWdlIiwiYXVkIjoiaHR0cHM6Ly9hdXRoLnNvZnR3YXJlaGVyaXRhZ2Uub3JnL2F1dGgvcmVhbG1zL1NvZnR3YXJlSGVyaXRhZ2UiLCJzdWIiOiIzMTY5OWZkNC0xNmE0LTQxOWItYTdhMi00NjI5MDY4ZjI3OWEiLCJ0eXAiOiJPZmZsaW5lIiwiYXpwIjoic3doLXdlYiIsInNlc3Npb25fc3RhdGUiOiIzMjYzMzEwMS00ZDRkLTQwMjItODU2NC1iMzNlMTJiNTE3ZDkiLCJzY29wZSI6Im9wZW5pZCBvZmZsaW5lX2FjY2VzcyBwcm9maWxlIGVtYWlsIn0.XHj1VIZu1dZ4Ej32-oU84mFmaox9cLNjXosNxwZM0Xs"); - try (CloseableHttpResponse response = httpClient.execute(httpGet)) { - int statusCode = response.getStatusLine().getStatusCode(); -// if (statusCode != 200) -// return null; - Header[] headers = response.getHeaders("X-RateLimit-Remaining"); - for (Header header : headers) { - System.out - .println( - "Key : " + header.getName() - + " ,Value : " + header.getValue()); - } - HttpEntity entity = response.getEntity(); - if (entity != null) { - return EntityUtils.toString(entity); - } - } - } - return null; - } } diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/models/LastVisitResponse.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/models/LastVisitData.java similarity index 81% rename from dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/models/LastVisitResponse.java rename to dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/models/LastVisitData.java index 435397590..b8cd6de6e 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/models/LastVisitResponse.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/models/LastVisitData.java @@ -4,12 +4,14 @@ package eu.dnetlib.dhp.swh.models; import com.cloudera.com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import java.util.Date; + @JsonIgnoreProperties(ignoreUnknown = true) -public class LastVisitResponse { +public class LastVisitData { private String type; - private String date; + private Date date; @JsonProperty("snapshot") private String snapshotId; @@ -22,11 +24,11 @@ public class LastVisitResponse { this.type = type; } - public String getDate() { + public Date getDate() { return date; } - public void setDate(String date) { + public void setDate(Date date) { this.date = date; } diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConnection.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConnection.java new file mode 100644 index 000000000..46d512dcb --- /dev/null +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConnection.java @@ -0,0 +1,138 @@ + +package eu.dnetlib.dhp.swh.utils; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang3.math.NumberUtils; +import org.apache.http.Header; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpStatus; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import eu.dnetlib.dhp.common.Constants; +import eu.dnetlib.dhp.common.collection.CollectorException; +import eu.dnetlib.dhp.common.collection.HttpClientParams; +import eu.dnetlib.dhp.common.collection.HttpConnector2; + +public class SWHConnection { + + private static final Logger log = LoggerFactory.getLogger(SWHConnection.class); + + CloseableHttpClient httpClient; + + HttpClientParams clientParams; + + HttpConnector2 conn; + + public SWHConnection(HttpClientParams clientParams) { + +// // force http client to NOT transform double quotes (//) to single quote (/) +// RequestConfig requestConfig = RequestConfig.custom().setNormalizeUri(false).build(); +// +// // Create an HttpClient instance +// httpClient = HttpClientBuilder +// .create() +// .setDefaultRequestConfig(requestConfig) +// .build(); +// +// this.clientParams = clientParams; + // set custom headers + Map headers = new HashMap() { + { + put(HttpHeaders.ACCEPT, "application/json"); + put(HttpHeaders.AUTHORIZATION, String.format("Bearer %s", SWHConstants.ACCESS_TOKEN)); + } + }; + + clientParams.setHeaders(headers); + + // create http connector + conn = new HttpConnector2(clientParams); + + } + + public String call(String url) throws CollectorException { + return conn.getInputSource(url); + } + + public String getLib(String url) throws IOException, CollectorException { + + // delay between requests + if (this.clientParams.getRequestDelay() > 0) { + log.info("Request delay: {}", this.clientParams.getRequestDelay()); + this.backOff(this.clientParams.getRequestDelay()); + } + + // Create an HttpGet request with the URL + HttpGet httpGet = new HttpGet(url); + httpGet.setHeader("Accept", "application/json"); + httpGet.setHeader("Authorization", String.format("Bearer %s", SWHConstants.ACCESS_TOKEN)); + + // Execute the request and get the response + try (CloseableHttpResponse response = httpClient.execute(httpGet)) { + + System.out.println(url); + + int responseCode = response.getStatusLine().getStatusCode(); + if (responseCode != HttpStatus.SC_OK) { + + } + + System.out.println(responseCode); + + List
httpHeaders = Arrays.asList(response.getAllHeaders()); + for (Header header : httpHeaders) { + System.out.println(header.getName() + ":\t" + header.getValue()); + } + + String rateRemaining = this.getRateRemaining(response); + + // back off when rate remaining limit is approaching + if (rateRemaining != null && (Integer.parseInt(rateRemaining) < 2)) { + int retryAfter = this.getRetryAfter(response); + + log.info("Rate Limit: {} - Backing off: {}", rateRemaining, retryAfter); + this.backOff(retryAfter); + } + + return EntityUtils.toString(response.getEntity()); + } + } + + private String getRateRemaining(CloseableHttpResponse response) { + Header header = response.getFirstHeader(Constants.HTTPHEADER_IETF_DRAFT_RATELIMIT_REMAINING); + if (header != null) { + return header.getValue(); + } + return null; + } + + private int getRetryAfter(CloseableHttpResponse response) { + Header header = response.getFirstHeader(HttpHeaders.RETRY_AFTER); + if (header != null) { + String retryAfter = header.getValue(); + if (NumberUtils.isCreatable(retryAfter)) { + return Integer.parseInt(retryAfter) + 10; + } + } + return 1000; + } + + private void backOff(int sleepTimeMs) throws CollectorException { + try { + Thread.sleep(sleepTimeMs); + } catch (InterruptedException e) { + throw new CollectorException(e); + } + } + +} diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConstants.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConstants.java new file mode 100644 index 000000000..1299bc805 --- /dev/null +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConstants.java @@ -0,0 +1,13 @@ + +package eu.dnetlib.dhp.swh.utils; + +public class SWHConstants { + public static final String SWH_LATEST_VISIT_URL = "https://archive.softwareheritage.org/api/1/origin/%s/visit/latest/"; + + public static final String SWH_ARCHIVE_URL = "https://archive.softwareheritage.org/api/1/origin/save/%s/url/%s/"; + + public static final String ACCESS_TOKEN = ""; + + public static final String DEFAULT_VISIT_TYPE = "git"; + +} diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHUtils.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHUtils.java new file mode 100644 index 000000000..8200e7b34 --- /dev/null +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHUtils.java @@ -0,0 +1,94 @@ + +package eu.dnetlib.dhp.swh.utils; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.collection.HttpClientParams; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.Optional; + +import static eu.dnetlib.dhp.common.Constants.*; + +public class SWHUtils { + + private static final Logger log = LoggerFactory.getLogger(SWHUtils.class); + + public static HttpClientParams getClientParams(ArgumentApplicationParser argumentParser) { + + final HttpClientParams clientParams = new HttpClientParams(); + clientParams + .setMaxNumberOfRetry( + Optional + .ofNullable(argumentParser.get(MAX_NUMBER_OF_RETRY)) + .map(Integer::parseInt) + .orElse(HttpClientParams._maxNumberOfRetry)); + log.info("maxNumberOfRetry is {}", clientParams.getMaxNumberOfRetry()); + + clientParams + .setRequestDelay( + Optional + .ofNullable(argumentParser.get(REQUEST_DELAY)) + .map(Integer::parseInt) + .orElse(HttpClientParams._requestDelay)); + log.info("requestDelay is {}", clientParams.getRequestDelay()); + + clientParams + .setRetryDelay( + Optional + .ofNullable(argumentParser.get(RETRY_DELAY)) + .map(Integer::parseInt) + .orElse(HttpClientParams._retryDelay)); + log.info("retryDelay is {}", clientParams.getRetryDelay()); + + clientParams + .setRequestMethod( + Optional + .ofNullable(argumentParser.get(REQUEST_METHOD)) + .orElse(HttpClientParams._requestMethod)); + log.info("requestMethod is {}", clientParams.getRequestMethod()); + + return clientParams; + } + + public static BufferedReader getFileReader(FileSystem fs, Path inputPath) throws IOException { + FSDataInputStream inputStream = fs.open(inputPath); + return new BufferedReader( + new InputStreamReader(inputStream, StandardCharsets.UTF_8)); + } + + public static SequenceFile.Writer getSequenceFileWriter(FileSystem fs, String outputPath) throws IOException { + return SequenceFile + .createWriter( + fs.getConf(), + SequenceFile.Writer.file(new Path(outputPath)), + SequenceFile.Writer.keyClass(Text.class), + SequenceFile.Writer.valueClass(Text.class)); + } + + public static SequenceFile.Reader getSequenceFileReader(FileSystem fs, String inputPath) throws IOException { + Path filePath = new Path(inputPath); + SequenceFile.Reader.Option fileOption = SequenceFile.Reader.file(filePath); + + return new SequenceFile.Reader(fs.getConf(), fileOption); + } + + public static void appendToSequenceFile(SequenceFile.Writer fw, String keyStr, String valueStr) throws IOException { + Text key = new Text(); + key.set(keyStr); + + Text value = new Text(); + value.set(valueStr); + + fw.append(key, value); + } +} diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_archive_repository_urls.json b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_archive_repository_urls.json new file mode 100644 index 000000000..5ec481305 --- /dev/null +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_archive_repository_urls.json @@ -0,0 +1,26 @@ +[ + { + "paramName": "n", + "paramLongName": "namenode", + "paramDescription": "the Name Node URI", + "paramRequired": true + }, + { + "paramName": "lv", + "paramLongName": "lastVisitsPath", + "paramDescription": "the URL where to store last visits data", + "paramRequired": true + }, + { + "paramName": "rqd", + "paramLongName": "requestDelay", + "paramDescription": "the delay (ms) between requests", + "paramRequired": false + }, + { + "paramName": "atid", + "paramLongName": "archiveThresholdInDays", + "paramDescription": "the thershold (in days) required to issue an archive request", + "paramRequired": false + } +] \ No newline at end of file diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_collect_last_visit_repository_data.json b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_collect_last_visit_repository_data.json new file mode 100644 index 000000000..6c59123be --- /dev/null +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_collect_last_visit_repository_data.json @@ -0,0 +1,38 @@ +[ + { + "paramName": "n", + "paramLongName": "namenode", + "paramDescription": "the Name Node URI", + "paramRequired": true + }, + { + "paramName": "scr", + "paramLongName": "softwareCodeRepositoryURLs", + "paramDescription": "the URL from where to read software repository URLs", + "paramRequired": true + }, + { + "paramName": "lv", + "paramLongName": "lastVisitsPath", + "paramDescription": "the URL where to store last visits data", + "paramRequired": true + }, + { + "paramName": "mnr", + "paramLongName": "maxNumberOfRetry", + "paramDescription": "the maximum number of admitted connection retries", + "paramRequired": false + }, + { + "paramName": "rqd", + "paramLongName": "requestDelay", + "paramDescription": "the delay (ms) between requests", + "paramRequired": false + }, + { + "paramName": "rtd", + "paramLongName": "retryDelay", + "paramDescription": "the delay (ms) between retries", + "paramRequired": false + } +] \ No newline at end of file diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_parameters.json b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_collect_software_repository_urls.json similarity index 96% rename from dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_parameters.json rename to dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_collect_software_repository_urls.json index dd5432b93..6e98c7673 100644 --- a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_parameters.json +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_collect_software_repository_urls.json @@ -6,7 +6,7 @@ "paramRequired": false }, { - "paramName": "ip", + "paramName": "scr", "paramLongName": "softwareCodeRepositoryURLs", "paramDescription": "the URL where to store software repository URLs", "paramRequired": true diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties index a63343aed..e2c2af852 100644 --- a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties @@ -1,25 +1,11 @@ # hive hiveDbName=openaire_prod_20230914 -hiveMetastoreUris=thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083 -# oozie -oozie.action.sharelib.for.spark=spark2 -oozie.use.system.libpath=true -oozie.wf.application.path=${oozieTopWfApplicationPath} -oozie.wf.application.path=${oozieTopWfApplicationPath} -oozieActionShareLibForSpark2=spark2 - -# spark -spark2EventLogDir=/user/spark/spark2ApplicationHistory -spark2ExtraListeners=com.cloudera.spark.lineage.NavigatorAppListener -spark2SqlQueryExecutionListeners=com.cloudera.spark.lineage.NavigatorQueryListener -spark2YarnHistoryServerAddress=http://iis-cdh5-test-gw.ocean.icm.edu.pl:18089 sparkSqlWarehouseDir=/user/hive/warehouse -# misc -wfAppPath=${oozieTopWfApplicationPath} -resourceManager=http://iis-cdh5-test-m2.ocean.icm.edu.pl:8088/cluster +# input/output files +softwareCodeRepositoryURLs=${workingDir}/1_code_repo_urls.csv +lastVisitsPath=${workingDir}/2_last_visits.seq +archiveRequestsPath=${workingDir}/3_archive_requests.seq -# custom params -softwareCodeRepositoryURLs=${workingDir}/code_repo_urls.csv resume=collect-software-repository-urls diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/config-default.xml b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/config-default.xml new file mode 100644 index 000000000..7873d595e --- /dev/null +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/config-default.xml @@ -0,0 +1,50 @@ + + + jobTracker + yarnRM + + + nameNode + hdfs://nameservice1 + + + oozie.use.system.libpath + true + + + oozie.action.sharelib.for.spark + spark2 + + + hiveMetastoreUris + thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083 + + + spark2YarnHistoryServerAddress + http://iis-cdh5-test-gw.ocean.icm.edu.pl:18089 + + + spark2EventLogDir + /user/spark/spark2ApplicationHistory + + + spark2ExtraListeners + "com.cloudera.spark.lineage.NavigatorAppListener" + + + spark2SqlQueryExecutionListeners + "com.cloudera.spark.lineage.NavigatorQueryListener" + + + oozieActionShareLibForSpark2 + spark2 + + + resourceManager + http://iis-cdh5-test-m2.ocean.icm.edu.pl:8088/cluster + + + oozie.launcher.mapreduce.user.classpath.first + true + + \ No newline at end of file diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml index 9832e5f26..5062d562b 100644 --- a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml @@ -1,59 +1,31 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + hiveDbName + The name of the Hive DB to be used + + + softwareCodeRepositoryURLs + The path in the HDSF to save the software repository URLs + + + resume + Variable that indicates the step to start from + + + ${jobTracker} ${nameNode} + + + oozie.action.sharelib.for.spark + ${oozieActionShareLibForSpark2} + + @@ -90,8 +62,43 @@ --softwareCodeRepositoryURLs${softwareCodeRepositoryURLs} --hiveDbName${hiveDbName} --hiveMetastoreUris${hiveMetastoreUris} - + + + + + + + eu.dnetlib.dhp.swh.CollectLastVisitRepositoryData + + --namenode${nameNode} + --softwareCodeRepositoryURLs${softwareCodeRepositoryURLs} + --lastVisitsPath${lastVisitsPath} + + --maxNumberOfRetry2 + --requestDelay0 + --retryDelay1 + --requestMethodGET + + + + + + + + + eu.dnetlib.dhp.swh.ArchiveRepositoryURLs + + --namenode${nameNode} + --lastVisitsPath${lastVisitsPath} + --archiveThresholdInDays365 + + --maxNumberOfRetry2 + --requestDelay0 + --retryDelay1 + --requestMethodPOST + + diff --git a/dhp-workflows/dhp-swh/src/test/java/eu/dnetlib/dhp/swh/ArchiveRepositoryURLsTest.java b/dhp-workflows/dhp-swh/src/test/java/eu/dnetlib/dhp/swh/ArchiveRepositoryURLsTest.java new file mode 100644 index 000000000..06e40ae14 --- /dev/null +++ b/dhp-workflows/dhp-swh/src/test/java/eu/dnetlib/dhp/swh/ArchiveRepositoryURLsTest.java @@ -0,0 +1,35 @@ +package eu.dnetlib.dhp.swh; + +import eu.dnetlib.dhp.swh.utils.SWHUtils; +import org.apache.hadoop.fs.FileSystem; +import org.junit.jupiter.api.Test; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.util.Arrays; + +public class ArchiveRepositoryURLsTest { + + @Test + void testArchive() throws IOException { + String inputPath = getClass() + .getResource("/eu/dnetlib/dhp/swh/lastVisitDataToArchive.csv") + .getPath(); + + File file = new File(inputPath); + FileReader fr = new FileReader(file); + BufferedReader br = new BufferedReader(fr); //creates a buffering character input stream + + String line; + while((line = br.readLine()) != null) { + String[] tokens = line.split("\t"); + + String response = ArchiveRepositoryURLs.handleRecord(tokens[0], tokens[1], 365); + System.out.println(tokens[0] + "\t" + response); + System.out.println(); + } + fr.close(); + } +} diff --git a/dhp-workflows/dhp-swh/src/test/java/eu/dnetlib/dhp/swh/SWHConnectionTest.java b/dhp-workflows/dhp-swh/src/test/java/eu/dnetlib/dhp/swh/SWHConnectionTest.java new file mode 100644 index 000000000..d69f6ff1b --- /dev/null +++ b/dhp-workflows/dhp-swh/src/test/java/eu/dnetlib/dhp/swh/SWHConnectionTest.java @@ -0,0 +1,57 @@ + +package eu.dnetlib.dhp.swh; + +import eu.dnetlib.dhp.common.collection.CollectorException; +import eu.dnetlib.dhp.common.collection.HttpClientParams; +import eu.dnetlib.dhp.swh.utils.SWHConnection; +import eu.dnetlib.dhp.swh.utils.SWHConstants; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; + +//import org.apache.hadoop.hdfs.MiniDFSCluster; + +public class SWHConnectionTest { + private static final Logger log = LoggerFactory.getLogger(SWHConnectionTest.class); + + @Test + void testGetCall() throws IOException { + + HttpClientParams clientParams = new HttpClientParams(); + clientParams.setRequestMethod("GET"); + + SWHConnection swhConnection = new SWHConnection(clientParams); + + String repoUrl = "https://github.com/stanford-futuredata/FAST"; + URL url = new URL(String.format(SWHConstants.SWH_LATEST_VISIT_URL, repoUrl)); + String response = null; + try { + response = swhConnection.call(url.toString()); + } catch (CollectorException e) { + System.out.println("Error in request: " + url); + } + System.out.println(response); + } + + @Test + void testPostCall() throws MalformedURLException { + HttpClientParams clientParams = new HttpClientParams(); + clientParams.setRequestMethod("POST"); + + SWHConnection swhConnection = new SWHConnection(clientParams); + + String repoUrl = "https://github.com/stanford-futuredata/FAST"; + URL url = new URL(String.format(SWHConstants.SWH_ARCHIVE_URL, SWHConstants.DEFAULT_VISIT_TYPE, repoUrl)); + String response = null; + try { + response = swhConnection.call(url.toString()); + } catch (CollectorException e) { + System.out.println("Error in request: " + url); + } + System.out.println(response); + } +} diff --git a/dhp-workflows/dhp-swh/src/test/resources/eu/dnetlib/dhp/swh/lastVisitDataToArchive.csv b/dhp-workflows/dhp-swh/src/test/resources/eu/dnetlib/dhp/swh/lastVisitDataToArchive.csv new file mode 100644 index 000000000..6477dd62a --- /dev/null +++ b/dhp-workflows/dhp-swh/src/test/resources/eu/dnetlib/dhp/swh/lastVisitDataToArchive.csv @@ -0,0 +1,6 @@ +https://github.com/bioinsilico/BIPSPI {"origin":"https://github.com/bioinsilico/BIPSPI","visit":1,"date":"2020-03-18T14:50:21.541822+00:00","status":"full","snapshot":"c6c69d2cd73ce89811448da5f031611df6f63bdb","type":"git","metadata":{},"origin_url":"https://archive.softwareheritage.org/api/1/origin/https://github.com/bioinsilico/BIPSPI/get/","snapshot_url":"https://archive.softwareheritage.org/api/1/snapshot/c6c69d2cd73ce89811448da5f031611df6f63bdb/"} +https://github.com/mloop/kdiff-type1-error-rate/blob/master/analysis/simulation.R {} +https://github.com/schwanbeck/YSMR {"origin":"https://github.com/schwanbeck/YSMR","visit":6,"date":"2023-08-02T15:25:02.650676+00:00","status":"full","snapshot":"a9d1c5f0bca2def198b89f65bc9f7da3be8439ed","type":"git","metadata":{},"origin_url":"https://archive.softwareheritage.org/api/1/origin/https://github.com/schwanbeck/YSMR/get/","snapshot_url":"https://archive.softwareheritage.org/api/1/snapshot/a9d1c5f0bca2def198b89f65bc9f7da3be8439ed/"} +https://github.com/lvclark/TASSELGBS_combine {"origin":"https://github.com/lvclark/TASSELGBS_combine","visit":1,"date":"2020-04-12T20:44:09.405589+00:00","status":"full","snapshot":"ffa6fefd3f5becefbea9fe0e6d5d93859c95c071","type":"git","metadata":{},"origin_url":"https://archive.softwareheritage.org/api/1/origin/https://github.com/lvclark/TASSELGBS_combine/get/","snapshot_url":"https://archive.softwareheritage.org/api/1/snapshot/ffa6fefd3f5becefbea9fe0e6d5d93859c95c071/"} +https://github.com/PRIDE-Toolsuite/inspector-example-files {"origin":"https://github.com/PRIDE-Toolsuite/inspector-example-files","visit":12,"date":"2021-01-25T08:54:13.394674+00:00","status":"full","snapshot":"0b56eb0ad07cf778df6dabefc4b73636e0ae8b37","type":"git","metadata":{},"origin_url":"https://archive.softwareheritage.org/api/1/origin/https://github.com/PRIDE-Toolsuite/inspector-example-files/get/","snapshot_url":"https://archive.softwareheritage.org/api/1/snapshot/0b56eb0ad07cf778df6dabefc4b73636e0ae8b37/"} +https://bitbucket.org/matwey/chelyabinsk {"origin":"https://bitbucket.org/matwey/chelyabinsk","visit":6,"date":"2021-09-24T19:32:43.322909+00:00","status":"full","snapshot":"215913858c3ee0e61e1aaea18241c5ee006da1b0","type":"hg","metadata":{},"origin_url":"https://archive.softwareheritage.org/api/1/origin/https://bitbucket.org/matwey/chelyabinsk/get/","snapshot_url":"https://archive.softwareheritage.org/api/1/snapshot/215913858c3ee0e61e1aaea18241c5ee006da1b0/"} \ No newline at end of file From ab0d70691cf4c21b886142b9a5d2b7327d6445d5 Mon Sep 17 00:00:00 2001 From: Serafeim Chatzopoulos Date: Thu, 28 Sep 2023 20:56:18 +0300 Subject: [PATCH 020/148] Add step for archiving repoUrls to SWH --- .../common/collection/HttpClientParams.java | 4 +- .../dhp/swh/ArchiveRepositoryURLs.java | 99 +++++++++++++------ .../swh/CollectLastVisitRepositoryData.java | 31 +++--- .../swh/CollectSoftwareRepositoryURLs.java | 15 +-- .../dnetlib/dhp/swh/models/LastVisitData.java | 22 +++-- .../dnetlib/dhp/swh/utils/SWHConstants.java | 4 +- .../eu/dnetlib/dhp/swh/utils/SWHUtils.java | 39 ++++---- .../swh/input_archive_repository_urls.json | 24 +++++ ...ut_collect_last_visit_repository_data.json | 6 ++ .../eu/dnetlib/dhp/swh/job.properties | 4 + .../eu/dnetlib/dhp/swh/oozie_app/workflow.xml | 35 +++++-- .../dhp/swh/ArchiveRepositoryURLsTest.java | 45 +++++---- .../eu/dnetlib/dhp/swh/SWHConnectionTest.java | 19 ++-- .../dhp/swh/lastVisitDataToArchive.csv | 1 + 14 files changed, 230 insertions(+), 118 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/collection/HttpClientParams.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/collection/HttpClientParams.java index 55f9ceb8b..d26d9c0e9 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/collection/HttpClientParams.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/collection/HttpClientParams.java @@ -53,9 +53,9 @@ public class HttpClientParams { */ private String requestMethod; - public HttpClientParams() { - this(_maxNumberOfRetry, _requestDelay, _retryDelay, _connectTimeOut, _readTimeOut, new HashMap<>(), _requestMethod); + this(_maxNumberOfRetry, _requestDelay, _retryDelay, _connectTimeOut, _readTimeOut, new HashMap<>(), + _requestMethod); } public HttpClientParams(int maxNumberOfRetry, int requestDelay, int retryDelay, int connectTimeOut, diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/ArchiveRepositoryURLs.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/ArchiveRepositoryURLs.java index 7b3b74d9e..38db27baf 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/ArchiveRepositoryURLs.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/ArchiveRepositoryURLs.java @@ -1,14 +1,16 @@ package eu.dnetlib.dhp.swh; -import com.fasterxml.jackson.databind.ObjectMapper; -import eu.dnetlib.dhp.application.ArgumentApplicationParser; -import eu.dnetlib.dhp.common.collection.CollectorException; -import eu.dnetlib.dhp.common.collection.HttpClientParams; -import eu.dnetlib.dhp.swh.models.LastVisitData; -import eu.dnetlib.dhp.swh.utils.SWHConnection; -import eu.dnetlib.dhp.swh.utils.SWHConstants; -import eu.dnetlib.dhp.swh.utils.SWHUtils; +import static eu.dnetlib.dhp.common.Constants.REQUEST_METHOD; +import static eu.dnetlib.dhp.utils.DHPUtils.getHadoopConfiguration; + +import java.io.IOException; +import java.net.URL; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + import org.apache.commons.cli.ParseException; import org.apache.commons.io.IOUtils; import org.apache.hadoop.fs.FileSystem; @@ -17,14 +19,17 @@ import org.apache.hadoop.io.Text; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.net.URL; -import java.util.Date; -import java.util.Optional; -import java.util.concurrent.TimeUnit; +import com.fasterxml.jackson.databind.ObjectMapper; -import static eu.dnetlib.dhp.common.Constants.REQUEST_METHOD; -import static eu.dnetlib.dhp.utils.DHPUtils.getHadoopConfiguration; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.collection.CollectorException; +import eu.dnetlib.dhp.common.collection.HttpClientParams; +import eu.dnetlib.dhp.schema.common.ModelSupport; +import eu.dnetlib.dhp.schema.oaf.utils.GraphCleaningFunctions; +import eu.dnetlib.dhp.swh.models.LastVisitData; +import eu.dnetlib.dhp.swh.utils.SWHConnection; +import eu.dnetlib.dhp.swh.utils.SWHConstants; +import eu.dnetlib.dhp.swh.utils.SWHUtils; /** * Sends archive requests to the SWH API for those software repository URLs that are missing from them @@ -69,7 +74,8 @@ public class ArchiveRepositoryURLs { } - private static void archive(FileSystem fs, String inputPath, String outputPath, Integer archiveThresholdInDays) throws IOException { + private static void archive(FileSystem fs, String inputPath, String outputPath, Integer archiveThresholdInDays) + throws IOException { SequenceFile.Reader fr = SWHUtils.getSequenceFileReader(fs, inputPath); SequenceFile.Writer fw = SWHUtils.getSequenceFileWriter(fs, outputPath); @@ -81,7 +87,13 @@ public class ArchiveRepositoryURLs { // Read key-value pairs from the SequenceFile and handle appropriately while (fr.next(repoUrl, lastVisitData)) { - String response = handleRecord(repoUrl.toString(), lastVisitData.toString(), archiveThresholdInDays); + String response = null; + try { + response = handleRecord(repoUrl.toString(), lastVisitData.toString(), archiveThresholdInDays); + } catch (java.text.ParseException e) { + log.error("Could not handle record with repo Url: {}", repoUrl.toString()); + throw new RuntimeException(e); + } // response is equal to null when no need for request if (response != null) { @@ -95,43 +107,68 @@ public class ArchiveRepositoryURLs { fr.close(); } - public static String handleRecord(String repoUrl, String lastVisitData, Integer archiveThresholdInDays) throws IOException { - System.out.println("Key: " + repoUrl + ", Value: " + lastVisitData); + public static String handleRecord(String repoUrl, String lastVisitData, Integer archiveThresholdInDays) + throws IOException, java.text.ParseException { + + log.info("{ Key: {}, Value: {} }", repoUrl, lastVisitData); LastVisitData lastVisit = OBJECT_MAPPER.readValue(lastVisitData, LastVisitData.class); - // perform an archive request when no repoUrl was not found in previous step + // a previous attempt for archival has been made, and repository URL was not found + // avoid performing the same archive request again + if (lastVisit.getType() != null && + lastVisit.getType().equals(SWHConstants.VISIT_STATUS_NOT_FOUND)) { + + log.info("Avoid request -- previous archive request returned NOT_FOUND"); + return null; + } + + // if we have last visit data if (lastVisit.getSnapshot() != null) { - // OR last visit was before (now() - archiveThresholdInDays) - long diffInMillies = Math.abs((new Date()).getTime() - lastVisit.getDate().getTime()); - long diffInDays = TimeUnit.DAYS.convert(diffInMillies, TimeUnit.MILLISECONDS); + String cleanDate = GraphCleaningFunctions.cleanDate(lastVisit.getDate()); - if (archiveThresholdInDays >= diffInDays) { - return null; + // and the last visit date can be parsed + if (cleanDate != null) { + + SimpleDateFormat formatter = new SimpleDateFormat(ModelSupport.DATE_FORMAT); + Date lastVisitDate = formatter.parse(cleanDate); + + // OR last visit time < (now() - archiveThresholdInDays) + long diffInMillies = Math.abs((new Date()).getTime() - lastVisitDate.getTime()); + long diffInDays = TimeUnit.DAYS.convert(diffInMillies, TimeUnit.MILLISECONDS); + log.info("Date diff from now (in days): {}", diffInDays); + + // do not perform a request, if the last visit date is no older than $archiveThresholdInDays + if (archiveThresholdInDays >= diffInDays) { + log.info("Avoid request -- no older than {} days", archiveThresholdInDays); + return null; + } } } - // if last visit data are available, re-use version control type, else use the default one (i.e., git) + // ELSE perform an archive request + log.info("Perform archive request for: {}", repoUrl); + + // if last visit data are available, re-use version control type, + // else use the default one (i.e., git) String visitType = Optional .ofNullable(lastVisit.getType()) .orElse(SWHConstants.DEFAULT_VISIT_TYPE); URL url = new URL(String.format(SWHConstants.SWH_ARCHIVE_URL, visitType, repoUrl.trim())); - System.out.println(url.toString()); + + log.info("Sending archive request: {}", url); String response; try { response = swhConnection.call(url.toString()); } catch (CollectorException e) { - log.info("Error in request: {}", url); + log.error("Error in request: {}", url); response = "{}"; } return response; - } - - } diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectLastVisitRepositoryData.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectLastVisitRepositoryData.java index c4b6412b5..9386b6876 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectLastVisitRepositoryData.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectLastVisitRepositoryData.java @@ -1,12 +1,15 @@ package eu.dnetlib.dhp.swh; -import eu.dnetlib.dhp.application.ArgumentApplicationParser; -import eu.dnetlib.dhp.common.collection.CollectorException; -import eu.dnetlib.dhp.common.collection.HttpClientParams; -import eu.dnetlib.dhp.swh.utils.SWHConnection; -import eu.dnetlib.dhp.swh.utils.SWHConstants; -import eu.dnetlib.dhp.swh.utils.SWHUtils; +import static eu.dnetlib.dhp.utils.DHPUtils.getHadoopConfiguration; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.charset.StandardCharsets; + import org.apache.commons.cli.ParseException; import org.apache.commons.io.IOUtils; import org.apache.hadoop.fs.FSDataInputStream; @@ -18,14 +21,12 @@ import org.apache.hadoop.io.Text; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.URISyntaxException; -import java.net.URL; -import java.nio.charset.StandardCharsets; - -import static eu.dnetlib.dhp.utils.DHPUtils.getHadoopConfiguration; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.collection.CollectorException; +import eu.dnetlib.dhp.common.collection.HttpClientParams; +import eu.dnetlib.dhp.swh.utils.SWHConnection; +import eu.dnetlib.dhp.swh.utils.SWHConstants; +import eu.dnetlib.dhp.swh.utils.SWHUtils; /** * Given a file with software repository URLs, this class @@ -107,7 +108,7 @@ public class CollectLastVisitRepositoryData { try { response = swhConnection.call(url.toString()); } catch (CollectorException e) { - log.info("Error in request: {}", url); + log.error("Error in request: {}", url); response = "{}"; } diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectSoftwareRepositoryURLs.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectSoftwareRepositoryURLs.java index f93280b5e..c1a0fafa5 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectSoftwareRepositoryURLs.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectSoftwareRepositoryURLs.java @@ -1,8 +1,11 @@ package eu.dnetlib.dhp.swh; -import eu.dnetlib.dhp.application.ArgumentApplicationParser; -import eu.dnetlib.dhp.schema.oaf.Result; +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession; + +import java.io.Serializable; +import java.util.Optional; + import org.apache.commons.io.IOUtils; import org.apache.spark.SparkConf; import org.apache.spark.sql.Dataset; @@ -12,10 +15,8 @@ import org.apache.spark.sql.SparkSession; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.Serializable; -import java.util.Optional; - -import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.schema.oaf.Result; /** * Collects unique software repository URLs in the Graph using Hive @@ -69,7 +70,7 @@ public class CollectSoftwareRepositoryURLs implements Serializable { "WHERE coderepositoryurl.value IS NOT NULL " + "AND datainfo.deletedbyinference = FALSE " + "AND datainfo.invisible = FALSE " + - "LIMIT 1000"; // TODO remove + "LIMIT 1000"; String query = String.format(queryTemplate, hiveDbName); log.info("Hive query to fetch software code URLs: {}", query); diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/models/LastVisitData.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/models/LastVisitData.java index b8cd6de6e..eaff5ce02 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/models/LastVisitData.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/models/LastVisitData.java @@ -1,21 +1,23 @@ package eu.dnetlib.dhp.swh.models; +import java.util.Date; + +import com.cloudera.com.fasterxml.jackson.annotation.JsonFormat; import com.cloudera.com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import java.util.Date; - @JsonIgnoreProperties(ignoreUnknown = true) public class LastVisitData { private String type; - - private Date date; + private String date; @JsonProperty("snapshot") private String snapshotId; + private String status; + public String getType() { return type; } @@ -24,11 +26,11 @@ public class LastVisitData { this.type = type; } - public Date getDate() { + public String getDate() { return date; } - public void setDate(Date date) { + public void setDate(String date) { this.date = date; } @@ -39,4 +41,12 @@ public class LastVisitData { public void setSnapshot(String snapshotId) { this.snapshotId = snapshotId; } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } } diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConstants.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConstants.java index 1299bc805..f58705188 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConstants.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConstants.java @@ -6,8 +6,10 @@ public class SWHConstants { public static final String SWH_ARCHIVE_URL = "https://archive.softwareheritage.org/api/1/origin/save/%s/url/%s/"; - public static final String ACCESS_TOKEN = ""; + public static final String ACCESS_TOKEN = "eyJhbGciOiJIUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJhMTMxYTQ1My1hM2IyLTQwMTUtODQ2Ny05MzAyZjk3MTFkOGEifQ.eyJpYXQiOjE2OTQ2MzYwMjAsImp0aSI6IjkwZjdkNTNjLTQ5YTktNGFiMy1hY2E0LTcwMTViMjEyZTNjNiIsImlzcyI6Imh0dHBzOi8vYXV0aC5zb2Z0d2FyZWhlcml0YWdlLm9yZy9hdXRoL3JlYWxtcy9Tb2Z0d2FyZUhlcml0YWdlIiwiYXVkIjoiaHR0cHM6Ly9hdXRoLnNvZnR3YXJlaGVyaXRhZ2Uub3JnL2F1dGgvcmVhbG1zL1NvZnR3YXJlSGVyaXRhZ2UiLCJzdWIiOiIzMTY5OWZkNC0xNmE0LTQxOWItYTdhMi00NjI5MDY4ZjI3OWEiLCJ0eXAiOiJPZmZsaW5lIiwiYXpwIjoic3doLXdlYiIsInNlc3Npb25fc3RhdGUiOiIzMjYzMzEwMS00ZDRkLTQwMjItODU2NC1iMzNlMTJiNTE3ZDkiLCJzY29wZSI6Im9wZW5pZCBvZmZsaW5lX2FjY2VzcyBwcm9maWxlIGVtYWlsIn0.XHj1VIZu1dZ4Ej32-oU84mFmaox9cLNjXosNxwZM0Xs"; public static final String DEFAULT_VISIT_TYPE = "git"; + public static final String VISIT_STATUS_NOT_FOUND = "not_found"; + } diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHUtils.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHUtils.java index 8200e7b34..405ce51e4 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHUtils.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHUtils.java @@ -1,8 +1,14 @@ package eu.dnetlib.dhp.swh.utils; -import eu.dnetlib.dhp.application.ArgumentApplicationParser; -import eu.dnetlib.dhp.common.collection.HttpClientParams; +import static eu.dnetlib.dhp.common.Constants.*; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.Optional; + import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -11,13 +17,8 @@ import org.apache.hadoop.io.Text; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.Optional; - -import static eu.dnetlib.dhp.common.Constants.*; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.collection.HttpClientParams; public class SWHUtils { @@ -51,10 +52,10 @@ public class SWHUtils { log.info("retryDelay is {}", clientParams.getRetryDelay()); clientParams - .setRequestMethod( - Optional - .ofNullable(argumentParser.get(REQUEST_METHOD)) - .orElse(HttpClientParams._requestMethod)); + .setRequestMethod( + Optional + .ofNullable(argumentParser.get(REQUEST_METHOD)) + .orElse(HttpClientParams._requestMethod)); log.info("requestMethod is {}", clientParams.getRequestMethod()); return clientParams; @@ -63,16 +64,16 @@ public class SWHUtils { public static BufferedReader getFileReader(FileSystem fs, Path inputPath) throws IOException { FSDataInputStream inputStream = fs.open(inputPath); return new BufferedReader( - new InputStreamReader(inputStream, StandardCharsets.UTF_8)); + new InputStreamReader(inputStream, StandardCharsets.UTF_8)); } public static SequenceFile.Writer getSequenceFileWriter(FileSystem fs, String outputPath) throws IOException { return SequenceFile - .createWriter( - fs.getConf(), - SequenceFile.Writer.file(new Path(outputPath)), - SequenceFile.Writer.keyClass(Text.class), - SequenceFile.Writer.valueClass(Text.class)); + .createWriter( + fs.getConf(), + SequenceFile.Writer.file(new Path(outputPath)), + SequenceFile.Writer.keyClass(Text.class), + SequenceFile.Writer.valueClass(Text.class)); } public static SequenceFile.Reader getSequenceFileReader(FileSystem fs, String inputPath) throws IOException { diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_archive_repository_urls.json b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_archive_repository_urls.json index 5ec481305..ce80d6f4a 100644 --- a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_archive_repository_urls.json +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_archive_repository_urls.json @@ -11,12 +11,36 @@ "paramDescription": "the URL where to store last visits data", "paramRequired": true }, + { + "paramName": "arp", + "paramLongName": "archiveRequestsPath", + "paramDescription": "the URL where to store the responses of the archive requests", + "paramRequired": true + }, + { + "paramName": "mnr", + "paramLongName": "maxNumberOfRetry", + "paramDescription": "the maximum number of admitted connection retries", + "paramRequired": false + }, { "paramName": "rqd", "paramLongName": "requestDelay", "paramDescription": "the delay (ms) between requests", "paramRequired": false }, + { + "paramName": "rtd", + "paramLongName": "retryDelay", + "paramDescription": "the delay (ms) between retries", + "paramRequired": false + }, + { + "paramName": "rm", + "paramLongName": "requestMethod", + "paramDescription": "the method of the requests to perform", + "paramRequired": false + }, { "paramName": "atid", "paramLongName": "archiveThresholdInDays", diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_collect_last_visit_repository_data.json b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_collect_last_visit_repository_data.json index 6c59123be..8bf41f0ae 100644 --- a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_collect_last_visit_repository_data.json +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_collect_last_visit_repository_data.json @@ -34,5 +34,11 @@ "paramLongName": "retryDelay", "paramDescription": "the delay (ms) between retries", "paramRequired": false + }, + { + "paramName": "rm", + "paramLongName": "requestMethod", + "paramDescription": "the method of the requests to perform", + "paramRequired": false } ] \ No newline at end of file diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties index e2c2af852..4cc1c1e25 100644 --- a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties @@ -8,4 +8,8 @@ softwareCodeRepositoryURLs=${workingDir}/1_code_repo_urls.csv lastVisitsPath=${workingDir}/2_last_visits.seq archiveRequestsPath=${workingDir}/3_archive_requests.seq +maxNumberOfRetry=2 +retryDelay=1 +requestDelay=100 + resume=collect-software-repository-urls diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml index 5062d562b..b89165fa2 100644 --- a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml @@ -8,7 +8,27 @@ softwareCodeRepositoryURLs - The path in the HDSF to save the software repository URLs + The path in the HDFS to save the software repository URLs + + + lastVisitsPath + The path in the HDFS to save the responses of the last visit requests + + + archiveRequestsPath + The path in the HDFS to save the responses of the archive requests + + + maxNumberOfRetry + Max number of retries for failed API calls + + + retryDelay + Retry delay for failed requests (in sec) + + + requestDelay + Delay between API requests (in ms) resume @@ -75,9 +95,9 @@ --softwareCodeRepositoryURLs${softwareCodeRepositoryURLs} --lastVisitsPath${lastVisitsPath} - --maxNumberOfRetry2 - --requestDelay0 - --retryDelay1 + --maxNumberOfRetry${maxNumberOfRetry} + --requestDelay${requestDelay} + --retryDelay${retryDelay} --requestMethodGET @@ -91,11 +111,12 @@ --namenode${nameNode} --lastVisitsPath${lastVisitsPath} + --archiveRequestsPath${archiveRequestsPath} --archiveThresholdInDays365 - --maxNumberOfRetry2 - --requestDelay0 - --retryDelay1 + --maxNumberOfRetry${maxNumberOfRetry} + --requestDelay${requestDelay} + --retryDelay${retryDelay} --requestMethodPOST diff --git a/dhp-workflows/dhp-swh/src/test/java/eu/dnetlib/dhp/swh/ArchiveRepositoryURLsTest.java b/dhp-workflows/dhp-swh/src/test/java/eu/dnetlib/dhp/swh/ArchiveRepositoryURLsTest.java index 06e40ae14..e069e9655 100644 --- a/dhp-workflows/dhp-swh/src/test/java/eu/dnetlib/dhp/swh/ArchiveRepositoryURLsTest.java +++ b/dhp-workflows/dhp-swh/src/test/java/eu/dnetlib/dhp/swh/ArchiveRepositoryURLsTest.java @@ -1,35 +1,38 @@ -package eu.dnetlib.dhp.swh; -import eu.dnetlib.dhp.swh.utils.SWHUtils; -import org.apache.hadoop.fs.FileSystem; -import org.junit.jupiter.api.Test; +package eu.dnetlib.dhp.swh; import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; +import java.text.ParseException; import java.util.Arrays; +import org.apache.hadoop.fs.FileSystem; +import org.junit.jupiter.api.Test; + +import eu.dnetlib.dhp.swh.utils.SWHUtils; + public class ArchiveRepositoryURLsTest { - @Test - void testArchive() throws IOException { - String inputPath = getClass() - .getResource("/eu/dnetlib/dhp/swh/lastVisitDataToArchive.csv") - .getPath(); + @Test + void testArchive() throws IOException, ParseException { + String inputPath = getClass() + .getResource("/eu/dnetlib/dhp/swh/lastVisitDataToArchive.csv") + .getPath(); - File file = new File(inputPath); - FileReader fr = new FileReader(file); - BufferedReader br = new BufferedReader(fr); //creates a buffering character input stream + File file = new File(inputPath); + FileReader fr = new FileReader(file); + BufferedReader br = new BufferedReader(fr); // creates a buffering character input stream - String line; - while((line = br.readLine()) != null) { - String[] tokens = line.split("\t"); + String line; + while ((line = br.readLine()) != null) { + String[] tokens = line.split("\t"); - String response = ArchiveRepositoryURLs.handleRecord(tokens[0], tokens[1], 365); - System.out.println(tokens[0] + "\t" + response); - System.out.println(); - } - fr.close(); - } + String response = ArchiveRepositoryURLs.handleRecord(tokens[0], tokens[1], 365); + System.out.println(tokens[0] + "\t" + response); + System.out.println(); + } + fr.close(); + } } diff --git a/dhp-workflows/dhp-swh/src/test/java/eu/dnetlib/dhp/swh/SWHConnectionTest.java b/dhp-workflows/dhp-swh/src/test/java/eu/dnetlib/dhp/swh/SWHConnectionTest.java index d69f6ff1b..28210f1b3 100644 --- a/dhp-workflows/dhp-swh/src/test/java/eu/dnetlib/dhp/swh/SWHConnectionTest.java +++ b/dhp-workflows/dhp-swh/src/test/java/eu/dnetlib/dhp/swh/SWHConnectionTest.java @@ -1,17 +1,18 @@ package eu.dnetlib.dhp.swh; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; + +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import eu.dnetlib.dhp.common.collection.CollectorException; import eu.dnetlib.dhp.common.collection.HttpClientParams; import eu.dnetlib.dhp.swh.utils.SWHConnection; import eu.dnetlib.dhp.swh.utils.SWHConstants; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; //import org.apache.hadoop.hdfs.MiniDFSCluster; @@ -24,7 +25,7 @@ public class SWHConnectionTest { HttpClientParams clientParams = new HttpClientParams(); clientParams.setRequestMethod("GET"); - SWHConnection swhConnection = new SWHConnection(clientParams); + SWHConnection swhConnection = new SWHConnection(clientParams); String repoUrl = "https://github.com/stanford-futuredata/FAST"; URL url = new URL(String.format(SWHConstants.SWH_LATEST_VISIT_URL, repoUrl)); @@ -42,7 +43,7 @@ public class SWHConnectionTest { HttpClientParams clientParams = new HttpClientParams(); clientParams.setRequestMethod("POST"); - SWHConnection swhConnection = new SWHConnection(clientParams); + SWHConnection swhConnection = new SWHConnection(clientParams); String repoUrl = "https://github.com/stanford-futuredata/FAST"; URL url = new URL(String.format(SWHConstants.SWH_ARCHIVE_URL, SWHConstants.DEFAULT_VISIT_TYPE, repoUrl)); diff --git a/dhp-workflows/dhp-swh/src/test/resources/eu/dnetlib/dhp/swh/lastVisitDataToArchive.csv b/dhp-workflows/dhp-swh/src/test/resources/eu/dnetlib/dhp/swh/lastVisitDataToArchive.csv index 6477dd62a..568ccf482 100644 --- a/dhp-workflows/dhp-swh/src/test/resources/eu/dnetlib/dhp/swh/lastVisitDataToArchive.csv +++ b/dhp-workflows/dhp-swh/src/test/resources/eu/dnetlib/dhp/swh/lastVisitDataToArchive.csv @@ -1,3 +1,4 @@ +https://bitbucket.org/samskillman/yt-stokes {"origin":"https://bitbucket.org/samskillman/yt-stokes","visit":43,"date":"2021-09-13T21:59:27.125171+00:00","status":"failed","snapshot":null,"type":"hg","metadata":{},"origin_url":"https://archive.softwareheritage.org/api/1/origin/https://bitbucket.org/samskillman/yt-stokes/get/","snapshot_url":null} https://github.com/bioinsilico/BIPSPI {"origin":"https://github.com/bioinsilico/BIPSPI","visit":1,"date":"2020-03-18T14:50:21.541822+00:00","status":"full","snapshot":"c6c69d2cd73ce89811448da5f031611df6f63bdb","type":"git","metadata":{},"origin_url":"https://archive.softwareheritage.org/api/1/origin/https://github.com/bioinsilico/BIPSPI/get/","snapshot_url":"https://archive.softwareheritage.org/api/1/snapshot/c6c69d2cd73ce89811448da5f031611df6f63bdb/"} https://github.com/mloop/kdiff-type1-error-rate/blob/master/analysis/simulation.R {} https://github.com/schwanbeck/YSMR {"origin":"https://github.com/schwanbeck/YSMR","visit":6,"date":"2023-08-02T15:25:02.650676+00:00","status":"full","snapshot":"a9d1c5f0bca2def198b89f65bc9f7da3be8439ed","type":"git","metadata":{},"origin_url":"https://archive.softwareheritage.org/api/1/origin/https://github.com/schwanbeck/YSMR/get/","snapshot_url":"https://archive.softwareheritage.org/api/1/snapshot/a9d1c5f0bca2def198b89f65bc9f7da3be8439ed/"} From e84f5b5e6457d8e47fea99a860cd27f41c67882e Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Mon, 2 Oct 2023 09:25:16 +0200 Subject: [PATCH 021/148] extended existing codo to accomodate import of POCI from open citation --- .../CreateActionSetSparkJob.java | 29 +++++++++++++------ .../opencitations/GetOpenCitationsRefs.java | 16 ++++++---- .../actionmanager/opencitations/ReadCOCI.java | 18 +++++++++--- .../opencitations/as_parameters.json | 5 ++++ .../opencitations/input_parameters.json | 6 ++++ .../input_readcoci_parameters.json | 7 ++++- .../opencitations/oozie_app/workflow.xml | 9 ++++-- 7 files changed, 67 insertions(+), 23 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java index 4c658e52f..dafd82120 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java @@ -26,7 +26,6 @@ import eu.dnetlib.dhp.actionmanager.opencitations.model.COCI; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.schema.action.AtomicAction; import eu.dnetlib.dhp.schema.common.ModelConstants; -import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.oaf.*; import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions; import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; @@ -35,7 +34,9 @@ import scala.Tuple2; public class CreateActionSetSparkJob implements Serializable { public static final String OPENCITATIONS_CLASSID = "sysimport:crosswalk:opencitations"; public static final String OPENCITATIONS_CLASSNAME = "Imported from OpenCitations"; - private static final String ID_PREFIX = "50|doi_________::"; + private static final String DOI_PREFIX = "50|doi_________::"; + + private static final String PMID_PREFIX = "50|pmid________::"; private static final String TRUST = "0.91"; private static final Logger log = LoggerFactory.getLogger(CreateActionSetSparkJob.class); @@ -67,6 +68,9 @@ public class CreateActionSetSparkJob implements Serializable { final String outputPath = parser.get("outputPath"); log.info("outputPath {}", outputPath); + final String prefix = parser.get("prefix"); + log.info("prefix {}", prefix); + final boolean shouldDuplicateRels = Optional .ofNullable(parser.get("shouldDuplicateRels")) .map(Boolean::valueOf) @@ -77,13 +81,13 @@ public class CreateActionSetSparkJob implements Serializable { conf, isSparkSessionManaged, spark -> { - extractContent(spark, inputPath, outputPath, shouldDuplicateRels); + extractContent(spark, inputPath, outputPath, shouldDuplicateRels, prefix); }); } private static void extractContent(SparkSession spark, String inputPath, String outputPath, - boolean shouldDuplicateRels) { + boolean shouldDuplicateRels, String prefix) { spark .read() .textFile(inputPath + "/*") @@ -91,7 +95,8 @@ public class CreateActionSetSparkJob implements Serializable { (MapFunction) value -> OBJECT_MAPPER.readValue(value, COCI.class), Encoders.bean(COCI.class)) .flatMap( - (FlatMapFunction) value -> createRelation(value, shouldDuplicateRels).iterator(), + (FlatMapFunction) value -> createRelation(value, shouldDuplicateRels, prefix) + .iterator(), Encoders.bean(Relation.class)) .filter((FilterFunction) value -> value != null) .toJavaRDD() @@ -103,13 +108,19 @@ public class CreateActionSetSparkJob implements Serializable { } - private static List createRelation(COCI value, boolean duplicate) { + private static List createRelation(COCI value, boolean duplicate, String p) { List relationList = new ArrayList<>(); + String prefix; + if (p.equals("COCI")) { + prefix = DOI_PREFIX; + } else { + prefix = PMID_PREFIX; + } - String citing = ID_PREFIX + String citing = prefix + IdentifierFactory.md5(CleaningFunctions.normalizePidValue("doi", value.getCiting())); - final String cited = ID_PREFIX + final String cited = prefix + IdentifierFactory.md5(CleaningFunctions.normalizePidValue("doi", value.getCited())); if (!citing.equals(cited)) { @@ -120,7 +131,7 @@ public class CreateActionSetSparkJob implements Serializable { cited, ModelConstants.CITES)); if (duplicate && value.getCiting().endsWith(".refs")) { - citing = ID_PREFIX + IdentifierFactory + citing = prefix + IdentifierFactory .md5( CleaningFunctions .normalizePidValue( diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/GetOpenCitationsRefs.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/GetOpenCitationsRefs.java index 3530c9980..60dc998ef 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/GetOpenCitationsRefs.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/GetOpenCitationsRefs.java @@ -45,6 +45,9 @@ public class GetOpenCitationsRefs implements Serializable { final String hdfsNameNode = parser.get("hdfsNameNode"); log.info("hdfsNameNode {}", hdfsNameNode); + final String prefix = parser.get("prefix"); + log.info("prefix {}", prefix); + Configuration conf = new Configuration(); conf.set("fs.defaultFS", hdfsNameNode); @@ -53,30 +56,31 @@ public class GetOpenCitationsRefs implements Serializable { GetOpenCitationsRefs ocr = new GetOpenCitationsRefs(); for (String file : inputFile) { - ocr.doExtract(workingPath + "/Original/" + file, workingPath, fileSystem); + ocr.doExtract(workingPath + "/Original/" + file, workingPath, fileSystem, prefix); } } - private void doExtract(String inputFile, String workingPath, FileSystem fileSystem) + private void doExtract(String inputFile, String workingPath, FileSystem fileSystem, String prefix) throws IOException { final Path path = new Path(inputFile); FSDataInputStream oc_zip = fileSystem.open(path); - int count = 1; + // int count = 1; try (ZipInputStream zis = new ZipInputStream(oc_zip)) { ZipEntry entry = null; while ((entry = zis.getNextEntry()) != null) { if (!entry.isDirectory()) { String fileName = entry.getName(); - fileName = fileName.substring(0, fileName.indexOf("T")) + "_" + count; - count++; + // fileName = fileName.substring(0, fileName.indexOf("T")) + "_" + count; + fileName = fileName.substring(0, fileName.lastIndexOf(".")); + // count++; try ( FSDataOutputStream out = fileSystem - .create(new Path(workingPath + "/COCI/" + fileName + ".gz")); + .create(new Path(workingPath + "/" + prefix + "/" + fileName + ".gz")); GZIPOutputStream gzipOs = new GZIPOutputStream(new BufferedOutputStream(out))) { IOUtils.copy(zis, gzipOs); diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/ReadCOCI.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/ReadCOCI.java index 4293ca187..3d384de9d 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/ReadCOCI.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/ReadCOCI.java @@ -49,6 +49,9 @@ public class ReadCOCI implements Serializable { final String workingPath = parser.get("workingPath"); log.info("workingPath {}", workingPath); + final String format = parser.get("format"); + log.info("format {}", format); + SparkConf sconf = new SparkConf(); final String delimiter = Optional @@ -64,13 +67,14 @@ public class ReadCOCI implements Serializable { workingPath, inputFile, outputPath, - delimiter); + delimiter, + format); }); } private static void doRead(SparkSession spark, String workingPath, String[] inputFiles, String outputPath, - String delimiter) throws IOException { + String delimiter, String format) throws IOException { for (String inputFile : inputFiles) { String p_string = workingPath + "/" + inputFile + ".gz"; @@ -87,9 +91,15 @@ public class ReadCOCI implements Serializable { cociData.map((MapFunction) row -> { COCI coci = new COCI(); + if (format.equals("COCI")) { + coci.setCiting(row.getString(1)); + coci.setCited(row.getString(2)); + } else { + coci.setCiting(String.valueOf(row.getInt(1))); + coci.setCited(String.valueOf(row.getInt(2))); + } coci.setOci(row.getString(0)); - coci.setCiting(row.getString(1)); - coci.setCited(row.getString(2)); + return coci; }, Encoders.bean(COCI.class)) .write() diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/as_parameters.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/as_parameters.json index 308e02026..e25d1f4b8 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/as_parameters.json +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/as_parameters.json @@ -21,5 +21,10 @@ "paramLongName": "shouldDuplicateRels", "paramDescription": "the hdfs name node", "paramRequired": false +},{ + "paramName": "p", + "paramLongName": "prefix", + "paramDescription": "the hdfs name node", + "paramRequired": true } ] diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/input_parameters.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/input_parameters.json index 4910ad11d..96db7eeb7 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/input_parameters.json +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/input_parameters.json @@ -16,5 +16,11 @@ "paramLongName": "hdfsNameNode", "paramDescription": "the hdfs name node", "paramRequired": true + }, + { + "paramName": "p", + "paramLongName": "prefix", + "paramDescription": "COCI or POCI", + "paramRequired": true } ] diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/input_readcoci_parameters.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/input_readcoci_parameters.json index b57cb5d9a..fa840089d 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/input_readcoci_parameters.json +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/input_readcoci_parameters.json @@ -30,7 +30,12 @@ "paramLongName": "inputFile", "paramDescription": "the hdfs name node", "paramRequired": true - } + }, { + "paramName": "f", + "paramLongName": "format", + "paramDescription": "the hdfs name node", + "paramRequired": true +} ] diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/oozie_app/workflow.xml index 0f01039f7..4807da903 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/oozie_app/workflow.xml @@ -60,6 +60,7 @@ --hdfsNameNode${nameNode} --inputFile${inputFile} --workingPath${workingPath} + --prefix${prefix} @@ -82,10 +83,11 @@ --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} - --workingPath${workingPath}/COCI - --outputPath${workingPath}/COCI_JSON/ + --workingPath${workingPath}/${prefix} + --outputPath${workingPath}/${prefix}_JSON/ --delimiter${delimiter} --inputFile${inputFileCoci} + --format${prefix} @@ -108,8 +110,9 @@ --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} - --inputPath${workingPath}/COCI_JSON + --inputPath${workingPath}/${prefix}_JSON --outputPath${outputPath} + --prefix${prefix} From e239b81740bccfb90211464b5d422113eac2b783 Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Wed, 20 Sep 2023 15:42:00 +0200 Subject: [PATCH 022/148] Fix defect #8997: GenerateEventsJob is generating huge amounts of logs because broker entity similarity calculation consistently failed --- .../eu/dnetlib/pace/model/SparkModel.scala | 4 ++-- .../dhp/broker/oa/util/TrustUtils.java | 22 +++++++++---------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/dhp-pace-core/src/main/java/eu/dnetlib/pace/model/SparkModel.scala b/dhp-pace-core/src/main/java/eu/dnetlib/pace/model/SparkModel.scala index 95325ace0..3ba36aa22 100644 --- a/dhp-pace-core/src/main/java/eu/dnetlib/pace/model/SparkModel.scala +++ b/dhp-pace-core/src/main/java/eu/dnetlib/pace/model/SparkModel.scala @@ -78,10 +78,10 @@ case class SparkModel(conf: DedupConfig) { uv case Type.List | Type.JSON => - MapDocumentUtil.truncateList( + Seq(MapDocumentUtil.truncateList( MapDocumentUtil.getJPathList(fdef.getPath, documentContext, fdef.getType), fdef.getSize - ).toArray + )) case Type.StringConcat => val jpaths = CONCAT_REGEX.split(fdef.getPath) diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/TrustUtils.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/TrustUtils.java index a6fa2b1a1..6f197a8ce 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/TrustUtils.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/TrustUtils.java @@ -1,18 +1,18 @@ package eu.dnetlib.dhp.broker.oa.util; -import java.io.IOException; - -import org.apache.spark.sql.Row; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.fasterxml.jackson.databind.ObjectMapper; - import eu.dnetlib.broker.objects.OaBrokerMainEntity; import eu.dnetlib.pace.config.DedupConfig; import eu.dnetlib.pace.model.SparkDeduper; import eu.dnetlib.pace.tree.support.TreeProcessor; +import org.apache.commons.io.IOUtils; +import org.apache.spark.sql.Row; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; public class TrustUtils { @@ -27,10 +27,8 @@ public class TrustUtils { static { mapper = new ObjectMapper(); try { - dedupConfig = mapper - .readValue( - DedupConfig.class.getResourceAsStream("/eu/dnetlib/dhp/broker/oa/dedupConfig/dedupConfig.json"), - DedupConfig.class); + dedupConfig = DedupConfig.load(IOUtils.toString(DedupConfig.class.getResourceAsStream("/eu/dnetlib/dhp/broker/oa/dedupConfig/dedupConfig.json"), StandardCharsets.UTF_8)); + deduper = new SparkDeduper(dedupConfig); } catch (final IOException e) { log.error("Error loading dedupConfig, e"); @@ -57,7 +55,7 @@ public class TrustUtils { return TrustUtils.rescale(score, threshold); } catch (final Exception e) { log.error("Error computing score between results", e); - return BrokerConstants.MIN_TRUST; + throw new RuntimeException(e); } } From 7f244d9a7ad7e723d9c80503476e590835031c4d Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Wed, 20 Sep 2023 15:53:21 +0200 Subject: [PATCH 023/148] code formatting --- .../dhp/broker/oa/util/TrustUtils.java | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/TrustUtils.java b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/TrustUtils.java index 6f197a8ce..67468c6f9 100644 --- a/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/TrustUtils.java +++ b/dhp-workflows/dhp-broker-events/src/main/java/eu/dnetlib/dhp/broker/oa/util/TrustUtils.java @@ -1,18 +1,20 @@ package eu.dnetlib.dhp.broker.oa.util; -import com.fasterxml.jackson.databind.ObjectMapper; -import eu.dnetlib.broker.objects.OaBrokerMainEntity; -import eu.dnetlib.pace.config.DedupConfig; -import eu.dnetlib.pace.model.SparkDeduper; -import eu.dnetlib.pace.tree.support.TreeProcessor; +import java.io.IOException; +import java.nio.charset.StandardCharsets; + import org.apache.commons.io.IOUtils; import org.apache.spark.sql.Row; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.nio.charset.StandardCharsets; +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.broker.objects.OaBrokerMainEntity; +import eu.dnetlib.pace.config.DedupConfig; +import eu.dnetlib.pace.model.SparkDeduper; +import eu.dnetlib.pace.tree.support.TreeProcessor; public class TrustUtils { @@ -27,7 +29,13 @@ public class TrustUtils { static { mapper = new ObjectMapper(); try { - dedupConfig = DedupConfig.load(IOUtils.toString(DedupConfig.class.getResourceAsStream("/eu/dnetlib/dhp/broker/oa/dedupConfig/dedupConfig.json"), StandardCharsets.UTF_8)); + dedupConfig = DedupConfig + .load( + IOUtils + .toString( + DedupConfig.class + .getResourceAsStream("/eu/dnetlib/dhp/broker/oa/dedupConfig/dedupConfig.json"), + StandardCharsets.UTF_8)); deduper = new SparkDeduper(dedupConfig); } catch (final IOException e) { From 3c47920c7839b2004a2e8f9fbb4dbff8873d1e6d Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Wed, 20 Sep 2023 16:14:01 +0200 Subject: [PATCH 024/148] Use asScala to convert java List to Scala Sequence --- .../src/main/java/eu/dnetlib/pace/model/SparkModel.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dhp-pace-core/src/main/java/eu/dnetlib/pace/model/SparkModel.scala b/dhp-pace-core/src/main/java/eu/dnetlib/pace/model/SparkModel.scala index 3ba36aa22..aa997c6e9 100644 --- a/dhp-pace-core/src/main/java/eu/dnetlib/pace/model/SparkModel.scala +++ b/dhp-pace-core/src/main/java/eu/dnetlib/pace/model/SparkModel.scala @@ -78,10 +78,10 @@ case class SparkModel(conf: DedupConfig) { uv case Type.List | Type.JSON => - Seq(MapDocumentUtil.truncateList( + MapDocumentUtil.truncateList( MapDocumentUtil.getJPathList(fdef.getPath, documentContext, fdef.getType), fdef.getSize - )) + ).asScala case Type.StringConcat => val jpaths = CONCAT_REGEX.split(fdef.getPath) From c412dc162bc7028c1e94eb3205960e24e748347e Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Wed, 27 Sep 2023 22:30:47 +0200 Subject: [PATCH 025/148] Fix bug in conversion from dedup json model to Spark Dataset of Rows: list of strings contained the json escaped representation of the value instead of the plain value, this caused instanceTypeMatch failures because of the leading and trailing double quotes --- .../src/main/java/eu/dnetlib/pace/util/MapDocumentUtil.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/MapDocumentUtil.java b/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/MapDocumentUtil.java index a59b6248b..28244cb3b 100644 --- a/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/MapDocumentUtil.java +++ b/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/MapDocumentUtil.java @@ -117,6 +117,11 @@ public class MapDocumentUtil { return result; } + if (type == Type.List && jresult instanceof List) { + ((List) jresult).forEach(x -> result.add(x.toString())); + return result; + } + if (jresult instanceof JSONArray) { ((JSONArray) jresult).forEach(it -> { try { From 9898470b0e2c5b05ae73c5b2ae404f6f52cba2f4 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Mon, 2 Oct 2023 12:54:16 +0200 Subject: [PATCH 026/148] Addressing comments in https://code-repo.d4science.org/D-Net/dnet-hadoop/pulls/340\#issuecomment-10592 --- .../CreateActionSetSparkJob.java | 119 ++++++++---------- .../opencitations/oozie_app/workflow.xml | 2 +- 2 files changed, 54 insertions(+), 67 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java index dafd82120..e3a9833b3 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java @@ -12,6 +12,7 @@ import org.apache.commons.io.IOUtils; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.SequenceFileOutputFormat; import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.function.FilterFunction; import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.api.java.function.MapFunction; @@ -26,9 +27,12 @@ import eu.dnetlib.dhp.actionmanager.opencitations.model.COCI; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.schema.action.AtomicAction; import eu.dnetlib.dhp.schema.common.ModelConstants; +import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.oaf.*; import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions; import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; +import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils; +import eu.dnetlib.dhp.utils.DHPUtils; import scala.Tuple2; public class CreateActionSetSparkJob implements Serializable { @@ -68,9 +72,6 @@ public class CreateActionSetSparkJob implements Serializable { final String outputPath = parser.get("outputPath"); log.info("outputPath {}", outputPath); - final String prefix = parser.get("prefix"); - log.info("prefix {}", prefix); - final boolean shouldDuplicateRels = Optional .ofNullable(parser.get("shouldDuplicateRels")) .map(Boolean::valueOf) @@ -81,47 +82,62 @@ public class CreateActionSetSparkJob implements Serializable { conf, isSparkSessionManaged, spark -> { - extractContent(spark, inputPath, outputPath, shouldDuplicateRels, prefix); + extractContent(spark, inputPath, outputPath, shouldDuplicateRels); }); } private static void extractContent(SparkSession spark, String inputPath, String outputPath, + boolean shouldDuplicateRels) { + + getTextTextJavaPairRDD(spark, inputPath, shouldDuplicateRels, "COCI") + .union(getTextTextJavaPairRDD(spark, inputPath, shouldDuplicateRels, "POCI")) + .saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class); + + } + + private static JavaPairRDD getTextTextJavaPairRDD(SparkSession spark, String inputPath, boolean shouldDuplicateRels, String prefix) { - spark + return spark .read() - .textFile(inputPath + "/*") + .textFile(inputPath + "/" + prefix + "/" + prefix + "_JSON/*") .map( (MapFunction) value -> OBJECT_MAPPER.readValue(value, COCI.class), Encoders.bean(COCI.class)) .flatMap( - (FlatMapFunction) value -> createRelation(value, shouldDuplicateRels, prefix) - .iterator(), + (FlatMapFunction) value -> createRelation( + value, shouldDuplicateRels, prefix) + .iterator(), Encoders.bean(Relation.class)) .filter((FilterFunction) value -> value != null) .toJavaRDD() .map(p -> new AtomicAction(p.getClass(), p)) .mapToPair( aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()), - new Text(OBJECT_MAPPER.writeValueAsString(aa)))) - .saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class); - + new Text(OBJECT_MAPPER.writeValueAsString(aa)))); } private static List createRelation(COCI value, boolean duplicate, String p) { List relationList = new ArrayList<>(); String prefix; + String citing; + String cited; if (p.equals("COCI")) { prefix = DOI_PREFIX; + citing = prefix + + IdentifierFactory.md5(CleaningFunctions.normalizePidValue("doi", value.getCiting())); + cited = prefix + + IdentifierFactory.md5(CleaningFunctions.normalizePidValue("doi", value.getCited())); + } else { prefix = PMID_PREFIX; - } + citing = prefix + + IdentifierFactory.md5(CleaningFunctions.normalizePidValue("pmid", value.getCiting())); + cited = prefix + + IdentifierFactory.md5(CleaningFunctions.normalizePidValue("pmid", value.getCited())); - String citing = prefix - + IdentifierFactory.md5(CleaningFunctions.normalizePidValue("doi", value.getCiting())); - final String cited = prefix - + IdentifierFactory.md5(CleaningFunctions.normalizePidValue("doi", value.getCited())); + } if (!citing.equals(cited)) { relationList @@ -143,59 +159,30 @@ public class CreateActionSetSparkJob implements Serializable { return relationList; } - private static Collection getRelations(String citing, String cited) { - - return Arrays - .asList( - getRelation(citing, cited, ModelConstants.CITES), - getRelation(cited, citing, ModelConstants.IS_CITED_BY)); - } - public static Relation getRelation( String source, String target, String relclass) { - Relation r = new Relation(); - r.setCollectedfrom(getCollectedFrom()); - r.setSource(source); - r.setTarget(target); - r.setRelClass(relclass); - r.setRelType(ModelConstants.RESULT_RESULT); - r.setSubRelType(ModelConstants.CITATION); - r - .setDataInfo( - getDataInfo()); - return r; + + return OafMapperUtils + .getRelation( + source, + target, + ModelConstants.RESULT_RESULT, + ModelConstants.CITATION, + relclass, + Arrays + .asList( + OafMapperUtils.keyValue(ModelConstants.OPENOCITATIONS_ID, ModelConstants.OPENOCITATIONS_NAME)), + OafMapperUtils + .dataInfo( + false, null, false, false, + OafMapperUtils + .qualifier( + OPENCITATIONS_CLASSID, OPENCITATIONS_CLASSNAME, + ModelConstants.DNET_PROVENANCE_ACTIONS, ModelConstants.DNET_PROVENANCE_ACTIONS), + TRUST), + null); + } - - public static List getCollectedFrom() { - KeyValue kv = new KeyValue(); - kv.setKey(ModelConstants.OPENOCITATIONS_ID); - kv.setValue(ModelConstants.OPENOCITATIONS_NAME); - - return Arrays.asList(kv); - } - - public static DataInfo getDataInfo() { - DataInfo di = new DataInfo(); - di.setInferred(false); - di.setDeletedbyinference(false); - di.setTrust(TRUST); - - di - .setProvenanceaction( - getQualifier(OPENCITATIONS_CLASSID, OPENCITATIONS_CLASSNAME, ModelConstants.DNET_PROVENANCE_ACTIONS)); - return di; - } - - public static Qualifier getQualifier(String class_id, String class_name, - String qualifierSchema) { - Qualifier pa = new Qualifier(); - pa.setClassid(class_id); - pa.setClassname(class_name); - pa.setSchemeid(qualifierSchema); - pa.setSchemename(qualifierSchema); - return pa; - } - } diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/oozie_app/workflow.xml index 4807da903..bd1932dd5 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/oozie_app/workflow.xml @@ -110,7 +110,7 @@ --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} - --inputPath${workingPath}/${prefix}_JSON + --inputPath${workingPath} --outputPath${outputPath} --prefix${prefix} From d7fccdc64b53a80c5f93be8cf4826c0d01f1e3ff Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Mon, 2 Oct 2023 14:10:57 +0200 Subject: [PATCH 027/148] fixed paths in wf to match the req of the pathname --- .../actionmanager/opencitations/oozie_app/workflow.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/oozie_app/workflow.xml index bd1932dd5..d87dfa2ba 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/oozie_app/workflow.xml @@ -46,7 +46,7 @@ download.sh ${filelist} - ${workingPath}/Original + ${workingPath}/${prefix}/Original HADOOP_USER_NAME=${wf:user()} download.sh @@ -59,7 +59,7 @@ eu.dnetlib.dhp.actionmanager.opencitations.GetOpenCitationsRefs --hdfsNameNode${nameNode} --inputFile${inputFile} - --workingPath${workingPath} + --workingPath${workingPath}/${prefix} --prefix${prefix} @@ -83,8 +83,8 @@ --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} - --workingPath${workingPath}/${prefix} - --outputPath${workingPath}/${prefix}_JSON/ + --workingPath${workingPath}/${prefix}/${prefix} + --outputPath${workingPath}/${prefix}/${prefix}_JSON/ --delimiter${delimiter} --inputFile${inputFileCoci} --format${prefix} From c9a5ad6a0266de84a0be5efc493b686586705218 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Mon, 2 Oct 2023 16:28:42 +0200 Subject: [PATCH 028/148] extending the coverage of the peer non-unknown refereed instances --- .../oaf/utils/GraphCleaningFunctions.java | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java index 8afa41f95..3c3e8052e 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java @@ -36,6 +36,19 @@ public class GraphCleaningFunctions extends CleaningFunctions { public static final int TITLE_FILTER_RESIDUAL_LENGTH = 5; private static final String NAME_CLEANING_REGEX = "[\\r\\n\\t\\s]+"; + private static final HashSet PEER_REVIEWED_TYPES = new HashSet<>(); + + static { + PEER_REVIEWED_TYPES.add("Article"); + PEER_REVIEWED_TYPES.add("Part of book or chapter of book"); + PEER_REVIEWED_TYPES.add("Book"); + PEER_REVIEWED_TYPES.add("Doctoral thesis"); + PEER_REVIEWED_TYPES.add("Master thesis"); + PEER_REVIEWED_TYPES.add("Data Paper"); + PEER_REVIEWED_TYPES.add("Thesis"); + PEER_REVIEWED_TYPES.add("Bachelor thesis"); + PEER_REVIEWED_TYPES.add("Conference object"); + } public static T cleanContext(T value, String contextId, String verifyParam) { if (ModelSupport.isSubClass(value, Result.class)) { @@ -493,6 +506,28 @@ public class GraphCleaningFunctions extends CleaningFunctions { if (Objects.isNull(i.getRefereed()) || StringUtils.isBlank(i.getRefereed().getClassid())) { i.setRefereed(qualifier("0000", "Unknown", ModelConstants.DNET_REVIEW_LEVELS)); } + + // from the script from Dimitris + if ("0000".equals(i.getRefereed().getClassid())) { + final boolean isFromCrossref = ModelConstants.CROSSREF_ID + .equals(i.getCollectedfrom().getKey()); + final boolean hasDoi = i + .getPid() + .stream() + .anyMatch(pid -> PidType.doi.toString().equals(pid.getQualifier().getClassid())); + final boolean isPeerReviewedType = PEER_REVIEWED_TYPES + .contains(i.getInstancetype().getClassname()); + final boolean noOtherLitType = r + .getInstance() + .stream() + .noneMatch(ii -> "Other literature type".equals(ii.getInstancetype().getClassname())); + if (isFromCrossref && hasDoi && isPeerReviewedType && noOtherLitType) { + i.setRefereed(qualifier("0001", "peerReviewed", ModelConstants.DNET_REVIEW_LEVELS)); + } else { + i.setRefereed(qualifier("0002", "nonPeerReviewed", ModelConstants.DNET_REVIEW_LEVELS)); + } + } + if (Objects.nonNull(i.getDateofacceptance())) { Optional date = cleanDateField(i.getDateofacceptance()); if (date.isPresent()) { From 839a8524e70790440bb570c5093f0ab43e0df323 Mon Sep 17 00:00:00 2001 From: Serafeim Chatzopoulos Date: Mon, 2 Oct 2023 23:50:38 +0300 Subject: [PATCH 029/148] Add action for creating actionsets --- .../dhp/swh/ArchiveRepositoryURLs.java | 5 +- .../swh/CollectLastVisitRepositoryData.java | 2 +- .../swh/CollectSoftwareRepositoryURLs.java | 9 +- .../dnetlib/dhp/swh/PrepareSWHActionsets.java | 177 ++++++++++++++++++ .../dnetlib/dhp/swh/models/LastVisitData.java | 25 ++- .../dnetlib/dhp/swh/utils/SWHConnection.java | 100 ---------- .../dnetlib/dhp/swh/utils/SWHConstants.java | 4 + .../dhp/swh/input_prepare_swh_actionsets.json | 26 +++ .../eu/dnetlib/dhp/swh/job.properties | 2 + .../eu/dnetlib/dhp/swh/oozie_app/workflow.xml | 27 +++ .../dhp/swh/PrepareSWHActionsetsTest.java | 97 ++++++++++ .../eu/dnetlib/dhp/swh/last_visits_data.seq | Bin 0 -> 253188 bytes .../eu/dnetlib/dhp/swh/software.json.gz | Bin 0 -> 16127 bytes 13 files changed, 362 insertions(+), 112 deletions(-) create mode 100644 dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/PrepareSWHActionsets.java create mode 100644 dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_prepare_swh_actionsets.json create mode 100644 dhp-workflows/dhp-swh/src/test/java/eu/dnetlib/dhp/swh/PrepareSWHActionsetsTest.java create mode 100644 dhp-workflows/dhp-swh/src/test/resources/eu/dnetlib/dhp/swh/last_visits_data.seq create mode 100644 dhp-workflows/dhp-swh/src/test/resources/eu/dnetlib/dhp/swh/software.json.gz diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/ArchiveRepositoryURLs.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/ArchiveRepositoryURLs.java index 38db27baf..f02861953 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/ArchiveRepositoryURLs.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/ArchiveRepositoryURLs.java @@ -1,7 +1,6 @@ package eu.dnetlib.dhp.swh; -import static eu.dnetlib.dhp.common.Constants.REQUEST_METHOD; import static eu.dnetlib.dhp.utils.DHPUtils.getHadoopConfiguration; import java.io.IOException; @@ -116,8 +115,8 @@ public class ArchiveRepositoryURLs { // a previous attempt for archival has been made, and repository URL was not found // avoid performing the same archive request again - if (lastVisit.getType() != null && - lastVisit.getType().equals(SWHConstants.VISIT_STATUS_NOT_FOUND)) { + if (lastVisit.getStatus() != null && + lastVisit.getStatus().equals(SWHConstants.VISIT_STATUS_NOT_FOUND)) { log.info("Avoid request -- previous archive request returned NOT_FOUND"); return null; diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectLastVisitRepositoryData.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectLastVisitRepositoryData.java index 9386b6876..296a4cce1 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectLastVisitRepositoryData.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectLastVisitRepositoryData.java @@ -40,7 +40,7 @@ public class CollectLastVisitRepositoryData { private static SWHConnection swhConnection = null; public static void main(final String[] args) - throws IOException, ParseException, InterruptedException, URISyntaxException, CollectorException { + throws IOException, ParseException { final ArgumentApplicationParser argumentParser = new ArgumentApplicationParser( IOUtils .toString( diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectSoftwareRepositoryURLs.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectSoftwareRepositoryURLs.java index c1a0fafa5..6232fa322 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectSoftwareRepositoryURLs.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectSoftwareRepositoryURLs.java @@ -3,7 +3,6 @@ package eu.dnetlib.dhp.swh; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession; -import java.io.Serializable; import java.util.Optional; import org.apache.commons.io.IOUtils; @@ -23,7 +22,7 @@ import eu.dnetlib.dhp.schema.oaf.Result; * * @author Serafeim Chatzopoulos */ -public class CollectSoftwareRepositoryURLs implements Serializable { +public class CollectSoftwareRepositoryURLs { private static final Logger log = LoggerFactory.getLogger(CollectSoftwareRepositoryURLs.class); @@ -44,10 +43,10 @@ public class CollectSoftwareRepositoryURLs implements Serializable { log.info("isSparkSessionManaged: {}", isSparkSessionManaged); final String hiveDbName = parser.get("hiveDbName"); - log.info("hiveDbName {}: ", hiveDbName); + log.info("hiveDbName: {}", hiveDbName); final String outputPath = parser.get("softwareCodeRepositoryURLs"); - log.info("softwareCodeRepositoryURLs {}: ", outputPath); + log.info("softwareCodeRepositoryURLs: {}", outputPath); final String hiveMetastoreUris = parser.get("hiveMetastoreUris"); log.info("hiveMetastoreUris: {}", hiveMetastoreUris); @@ -70,7 +69,7 @@ public class CollectSoftwareRepositoryURLs implements Serializable { "WHERE coderepositoryurl.value IS NOT NULL " + "AND datainfo.deletedbyinference = FALSE " + "AND datainfo.invisible = FALSE " + - "LIMIT 1000"; + "LIMIT 5000"; String query = String.format(queryTemplate, hiveDbName); log.info("Hive query to fetch software code URLs: {}", query); diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/PrepareSWHActionsets.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/PrepareSWHActionsets.java new file mode 100644 index 000000000..c0ab11bc4 --- /dev/null +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/PrepareSWHActionsets.java @@ -0,0 +1,177 @@ + +package eu.dnetlib.dhp.swh; + +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; +import static org.apache.spark.sql.functions.col; + +import java.io.Serializable; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.SequenceFileOutputFormat; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.sql.*; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.schema.action.AtomicAction; +import eu.dnetlib.dhp.schema.common.ModelConstants; +import eu.dnetlib.dhp.schema.oaf.*; +import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils; +import eu.dnetlib.dhp.swh.models.LastVisitData; +import eu.dnetlib.dhp.swh.utils.SWHConstants; +import scala.Tuple2; + +/** + * Creates action sets for Software Heritage data + * + * @author Serafeim Chatzopoulos + */ +public class PrepareSWHActionsets { + + private static final Logger log = LoggerFactory.getLogger(PrepareSWHActionsets.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + public static void main(String[] args) throws Exception { + + String jsonConfiguration = IOUtils + .toString( + PrepareSWHActionsets.class + .getResourceAsStream( + "/eu/dnetlib/dhp/swh/input_prepare_swh_actionsets.json")); + + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + parser.parseArgument(args); + + final Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + final String inputPath = parser.get("lastVisitsPath"); + log.info("inputPath: {}", inputPath); + + final String softwareInputPath = parser.get("softwareInputPath"); + log.info("softwareInputPath: {}", softwareInputPath); + + final String outputPath = parser.get("actionsetsPath"); + log.info("outputPath: {}", outputPath); + + SparkConf conf = new SparkConf(); + + runWithSparkSession( + conf, + isSparkSessionManaged, + spark -> { + JavaPairRDD softwareRDD = prepareActionsets(spark, inputPath, softwareInputPath); + softwareRDD + .saveAsHadoopFile( + outputPath, Text.class, Text.class, SequenceFileOutputFormat.class); +// , GzipCodec.class); + }); + } + + private static Dataset loadSWHData(SparkSession spark, String inputPath) { + + JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + + // read from file and transform to tuples + // Note: snapshot id is the SWH id for us + JavaRDD swhRDD = sc + .sequenceFile(inputPath, Text.class, Text.class) + .map(t -> t._2().toString()) + .map(t -> OBJECT_MAPPER.readValue(t, LastVisitData.class)) + .filter(t -> t.getOrigin() != null && t.getSnapshot() != null) // response from SWH API is empty if repo URL + // was not found + .map(item -> RowFactory.create(item.getOrigin(), item.getSnapshot())); + + // convert RDD to 2-column DF + List fields = Arrays + .asList( + DataTypes.createStructField("repoUrl", DataTypes.StringType, true), + DataTypes.createStructField("swhId", DataTypes.StringType, true)); + StructType schema = DataTypes.createStructType(fields); + + return spark.createDataFrame(swhRDD, schema); + } + + private static Dataset loadGraphSoftwareData(SparkSession spark, String softwareInputPath) { + return spark + .read() + .textFile(softwareInputPath) + .map( + (MapFunction) t -> OBJECT_MAPPER.readValue(t, Software.class), + Encoders.bean(Software.class)) + .filter(t -> t.getCodeRepositoryUrl() != null) + .select(col("id"), col("codeRepositoryUrl.value").as("repoUrl")); + } + + private static JavaPairRDD prepareActionsets(SparkSession spark, String inputPath, + String softwareInputPath) { + + Dataset swhDF = loadSWHData(spark, inputPath); +// swhDF.show(false); + + Dataset graphSoftwareDF = loadGraphSoftwareData(spark, softwareInputPath); +// graphSoftwareDF.show(5); + + Dataset joinedDF = graphSoftwareDF.join(swhDF, "repoUrl").select("id", "swhid"); +// joinedDF.show(false); + + return joinedDF.map((MapFunction) row -> { + + Software s = new Software(); + + // set openaire id + s.setId(row.getString(row.fieldIndex("id"))); + + // set swh id + Qualifier qualifier = OafMapperUtils + .qualifier( + SWHConstants.SWHID, + SWHConstants.SWHID_CLASSNAME, + ModelConstants.DNET_PID_TYPES, + ModelConstants.DNET_PID_TYPES); + + DataInfo dataInfo = OafMapperUtils + .dataInfo( + false, + null, + false, + false, + ModelConstants.PROVENANCE_ACTION_SET_QUALIFIER, + ""); + + s + .setPid( + Arrays + .asList( + OafMapperUtils + .structuredProperty( + row.getString(row.fieldIndex("swhid")), + qualifier, + dataInfo))); + return s; + }, Encoders.bean(Software.class)) + .toJavaRDD() + .map(p -> new AtomicAction(Software.class, p)) + .mapToPair( + aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()), + new Text(OBJECT_MAPPER.writeValueAsString(aa)))); + } +} diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/models/LastVisitData.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/models/LastVisitData.java index eaff5ce02..5e705716c 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/models/LastVisitData.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/models/LastVisitData.java @@ -1,15 +1,15 @@ package eu.dnetlib.dhp.swh.models; -import java.util.Date; +import java.io.Serializable; -import com.cloudera.com.fasterxml.jackson.annotation.JsonFormat; import com.cloudera.com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @JsonIgnoreProperties(ignoreUnknown = true) -public class LastVisitData { +public class LastVisitData implements Serializable { + private String origin; private String type; private String date; @@ -49,4 +49,23 @@ public class LastVisitData { public void setStatus(String status) { this.status = status; } + + public String getOrigin() { + return origin; + } + + public void setOrigin(String origin) { + this.origin = origin; + } + + @Override + public String toString() { + return "LastVisitData{" + + "origin='" + origin + '\'' + + ", type='" + type + '\'' + + ", date='" + date + '\'' + + ", snapshotId='" + snapshotId + '\'' + + ", status='" + status + '\'' + + '}'; + } } diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConnection.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConnection.java index 46d512dcb..9c145fc19 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConnection.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConnection.java @@ -1,50 +1,21 @@ package eu.dnetlib.dhp.swh.utils; -import java.io.IOException; -import java.util.Arrays; import java.util.HashMap; -import java.util.List; import java.util.Map; -import org.apache.commons.lang3.math.NumberUtils; -import org.apache.http.Header; import org.apache.http.HttpHeaders; -import org.apache.http.HttpStatus; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.util.EntityUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import eu.dnetlib.dhp.common.Constants; import eu.dnetlib.dhp.common.collection.CollectorException; import eu.dnetlib.dhp.common.collection.HttpClientParams; import eu.dnetlib.dhp.common.collection.HttpConnector2; public class SWHConnection { - private static final Logger log = LoggerFactory.getLogger(SWHConnection.class); - - CloseableHttpClient httpClient; - - HttpClientParams clientParams; - HttpConnector2 conn; public SWHConnection(HttpClientParams clientParams) { -// // force http client to NOT transform double quotes (//) to single quote (/) -// RequestConfig requestConfig = RequestConfig.custom().setNormalizeUri(false).build(); -// -// // Create an HttpClient instance -// httpClient = HttpClientBuilder -// .create() -// .setDefaultRequestConfig(requestConfig) -// .build(); -// -// this.clientParams = clientParams; // set custom headers Map headers = new HashMap() { { @@ -64,75 +35,4 @@ public class SWHConnection { return conn.getInputSource(url); } - public String getLib(String url) throws IOException, CollectorException { - - // delay between requests - if (this.clientParams.getRequestDelay() > 0) { - log.info("Request delay: {}", this.clientParams.getRequestDelay()); - this.backOff(this.clientParams.getRequestDelay()); - } - - // Create an HttpGet request with the URL - HttpGet httpGet = new HttpGet(url); - httpGet.setHeader("Accept", "application/json"); - httpGet.setHeader("Authorization", String.format("Bearer %s", SWHConstants.ACCESS_TOKEN)); - - // Execute the request and get the response - try (CloseableHttpResponse response = httpClient.execute(httpGet)) { - - System.out.println(url); - - int responseCode = response.getStatusLine().getStatusCode(); - if (responseCode != HttpStatus.SC_OK) { - - } - - System.out.println(responseCode); - - List
httpHeaders = Arrays.asList(response.getAllHeaders()); - for (Header header : httpHeaders) { - System.out.println(header.getName() + ":\t" + header.getValue()); - } - - String rateRemaining = this.getRateRemaining(response); - - // back off when rate remaining limit is approaching - if (rateRemaining != null && (Integer.parseInt(rateRemaining) < 2)) { - int retryAfter = this.getRetryAfter(response); - - log.info("Rate Limit: {} - Backing off: {}", rateRemaining, retryAfter); - this.backOff(retryAfter); - } - - return EntityUtils.toString(response.getEntity()); - } - } - - private String getRateRemaining(CloseableHttpResponse response) { - Header header = response.getFirstHeader(Constants.HTTPHEADER_IETF_DRAFT_RATELIMIT_REMAINING); - if (header != null) { - return header.getValue(); - } - return null; - } - - private int getRetryAfter(CloseableHttpResponse response) { - Header header = response.getFirstHeader(HttpHeaders.RETRY_AFTER); - if (header != null) { - String retryAfter = header.getValue(); - if (NumberUtils.isCreatable(retryAfter)) { - return Integer.parseInt(retryAfter) + 10; - } - } - return 1000; - } - - private void backOff(int sleepTimeMs) throws CollectorException { - try { - Thread.sleep(sleepTimeMs); - } catch (InterruptedException e) { - throw new CollectorException(e); - } - } - } diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConstants.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConstants.java index f58705188..08400f28b 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConstants.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConstants.java @@ -12,4 +12,8 @@ public class SWHConstants { public static final String VISIT_STATUS_NOT_FOUND = "not_found"; + public static final String SWHID = "swhid"; + + public static final String SWHID_CLASSNAME = "Software Heritage Identifier"; + } diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_prepare_swh_actionsets.json b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_prepare_swh_actionsets.json new file mode 100644 index 000000000..07ab0b1f4 --- /dev/null +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_prepare_swh_actionsets.json @@ -0,0 +1,26 @@ +[ + { + "paramName": "issm", + "paramLongName": "isSparkSessionManaged", + "paramDescription": "when true will stop SparkSession after job execution", + "paramRequired": false + }, + { + "paramName": "lv", + "paramLongName": "lastVisitsPath", + "paramDescription": "the URL where to store last visits data", + "paramRequired": true + }, + { + "paramName": "ap", + "paramLongName": "actionsetsPath", + "paramDescription": "the URL path where to store actionsets", + "paramRequired": true + }, + { + "paramName": "sip", + "paramLongName": "softwareInputPath", + "paramDescription": "the URL path of the software in the graph", + "paramRequired": true + } +] \ No newline at end of file diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties index 4cc1c1e25..651bae337 100644 --- a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties @@ -7,6 +7,8 @@ sparkSqlWarehouseDir=/user/hive/warehouse softwareCodeRepositoryURLs=${workingDir}/1_code_repo_urls.csv lastVisitsPath=${workingDir}/2_last_visits.seq archiveRequestsPath=${workingDir}/3_archive_requests.seq +actionsetsPath=${workingDir}/4_actionsets +graphPath=/tmp/prod_provision/graph/18_graph_blacklisted maxNumberOfRetry=2 retryDelay=1 diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml index b89165fa2..7aa667a4a 100644 --- a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml @@ -57,6 +57,7 @@ ${wf:conf('startFrom') eq 'collect-software-repository-urls'} + ${wf:conf('startFrom') eq 'create-swh-actionsets'} @@ -120,6 +121,32 @@ --requestMethodPOST + + + + + + + yarn + cluster + Create actionsets for SWH data + eu.dnetlib.dhp.swh.PrepareSWHActionsets + dhp-swh-${projectVersion}.jar + + --executor-memory=${sparkExecutorMemory} + --executor-cores=${sparkExecutorCores} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} + + + --lastVisitsPath${lastVisitsPath} + --actionsetsPath${actionsetsPath} + --softwareInputPath${graphPath}/software + diff --git a/dhp-workflows/dhp-swh/src/test/java/eu/dnetlib/dhp/swh/PrepareSWHActionsetsTest.java b/dhp-workflows/dhp-swh/src/test/java/eu/dnetlib/dhp/swh/PrepareSWHActionsetsTest.java new file mode 100644 index 000000000..ffcb7aaee --- /dev/null +++ b/dhp-workflows/dhp-swh/src/test/java/eu/dnetlib/dhp/swh/PrepareSWHActionsetsTest.java @@ -0,0 +1,97 @@ + +package eu.dnetlib.dhp.swh; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.io.Text; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.schema.action.AtomicAction; +import eu.dnetlib.dhp.schema.common.ModelConstants; +import eu.dnetlib.dhp.schema.oaf.Relation; +import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions; +import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; + +public class PrepareSWHActionsetsTest { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static SparkSession spark; + + private static Path workingDir; + + private static final Logger log = LoggerFactory + .getLogger(PrepareSWHActionsetsTest.class); + + @BeforeAll + public static void beforeAll() throws IOException { + workingDir = Files.createTempDirectory(PrepareSWHActionsetsTest.class.getSimpleName()); + + log.info("Using work dir {}", workingDir); + + SparkConf conf = new SparkConf(); + conf.setAppName(PrepareSWHActionsetsTest.class.getSimpleName()); + + conf.setMaster("local[*]"); + conf.set("spark.driver.host", "localhost"); + conf.set("hive.metastore.local", "true"); + conf.set("spark.ui.enabled", "false"); + conf.set("spark.sql.warehouse.dir", workingDir.toString()); + conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString()); + + spark = SparkSession + .builder() + .appName(PrepareSWHActionsetsTest.class.getSimpleName()) + .config(conf) + .getOrCreate(); + } + + @AfterAll + public static void afterAll() throws IOException { + FileUtils.deleteDirectory(workingDir.toFile()); + spark.stop(); + } + + @Test + void testRun() throws Exception { + + String lastVisitsPath = getClass() + .getResource("/eu/dnetlib/dhp/swh/last_visits_data.seq") + .getPath(); + + String outputPath = workingDir.toString() + "/actionSet"; + + String softwareInputPath = getClass() + .getResource("/eu/dnetlib/dhp/swh/software.json.gz") + .getPath(); + + PrepareSWHActionsets + .main( + new String[] { + "-isSparkSessionManaged", Boolean.FALSE.toString(), + "-lastVisitsPath", lastVisitsPath, + "-softwareInputPath", softwareInputPath, + "-actionsetsPath", outputPath + }); + + } +} diff --git a/dhp-workflows/dhp-swh/src/test/resources/eu/dnetlib/dhp/swh/last_visits_data.seq b/dhp-workflows/dhp-swh/src/test/resources/eu/dnetlib/dhp/swh/last_visits_data.seq new file mode 100644 index 0000000000000000000000000000000000000000..683fc0e693c3174f4d863d95a31361c6651565ca GIT binary patch literal 253188 zcma&O2RPOL|2}RMAtNI)lO*Xl_DIOimYEgDad0?;bB;|aNw#EmciL-=zuS)=+~DJC2n!*AmW zWllMPko;@AY`1K$89oZr=MH(qB8b1Zdp_YFPS+d1`2HfYs!%`O#6sb8k2j3bqC00z z&)b}?eq6qXl-`067_G16cISrV`m?IjbtzXM)x(A^-p(~yCP6KgKfjy#v6hfcw4N0m zwfx5QS+;#a_*q2j8q^?L8EoRLL>=8_I@7DEOm|cbT{wb7VABk8zysSdbL=}T+Zg2 z-{a2Vu%^Y2?^cc_TAZD-Ww1H<*1xZ(sJcv0{s&5qeE( z|9<_+fpc8}%+KI?Ki%z*I*WQm%wDszaX!6^{p1kt%LlGsQm=03e<}WT=pJzXstfJn zFWStd&=3H~4e(M|I|3AC1PrZf2DJm9pq@w_j@EdUF zRuKlZPbOsrAJ0^Wa=EZxl)oluZ~o@%*lC$ZLJ>wEQdyhkBC{>xt*CJPwlBdtTg)9d5K?NAWx=TDdN=6S z^II(6?>R|MNG__UIB&T=6|=KTkM0(o^4O9Q-n(wZ>6d%MNv9{pQ2eoAHCo5gqYM*P zQ=n@!{5I>lb=~Ez_Oi810VA1W1D`vw@;0S5k4I(0&7_!j0_EJe{RK9N$lv$r^)F5D zxch*I4zm;dI1&d(OTqp+jR2hC zJTJ=@HgZzG$hAC}*d6|+x7>y%HwycWwx7AbGwl?!zKxtdI8?A{DD0$^t}!u}3_%T$ zh3jS2_jN$8RWVXF%7k7M?#HRd<%EUl#TJOWtA9{sM|Ufws65wyb8~7uy{k4${M%O( zmppg7^k-|LQ*`@RXSPBuQ~i8iIFGzDG{RwDej*^EYsKL2a%{_PTj^ zSfj}Jq@2QipQ0xw>?kLMGHIQe}ke&JTb&N)}xd92I)_? z-dF)#h$~Btxv1z%G-g8m-5sf@tfai?yBO}Vs*Ga zRr^9F=os??=i(+LM>)-JGaVHXv~peM1-pB_QhoDc=EP)H`DDfWl?pXK^|FU&w3JJ; z{3eTr2wfT$gKV2T!|N-ZwhKN}PK#+T3OBs_KD)ZUXKS@jW{+@7irK=gb}``-%`eiQ zML4!dCQD}&3M*jDGLCz>pB2Zl zdFyN8r^&1}c7BBA=*?VfBdp-I0@fc1zta0ag!Wn%bac&Z9zpd>oubvvHBWHv-4F+b zc;f7d0>La>Qb$T~f2z!Q@_DPq4-Z=u89#rc@c2nVK^l7h5d1lUF#y0wxVer%%*n+Q z2_<+UF)n{#YCTJ=FU6qvh6l3r4)_O&QZTpQ6p3U}t5%k(cMX})GB;V@)ZSWb3<%h( zDowmcBW~2q1TiPAx^{crg-L%l%93QF#m}BaE|~M7pztLrr_vy&;tz83wGM|53`6vu z=ksqCB5$2yvAZjVw7^$$jUs*K&7tG%o58T6c2n28c4b+s5z*@naS(0`&YBOlKJc}( z{ij2PyIvF}#<`Yegxzsi@yToWHZv+AuiM}8-L=9!k`H~w9BXuT9mVySy#Cnim)Ffj zFPhEo34A)NJc#}>TGE7hD$O4_1uzf;um81wJ0!u*9qxi42m?w6HTXk~bdeY}C;^>70v_qhBw+sZDD*`5#=tY+pn%gmn%8cZKnHZ_!72~xxt%J%7;Q?cx<#H1`) zDkMtk@;1W8CQL6bq0p4`g^QJ`Yp9SNrX-XrJ%H4Hw%{>;w6S z%>Wh&@cQ3%#ADrYaD)Ru6d{a35TJiCPSD5T_C$>S3F`&o_lqBKFkKKB0v?G6lo;G& zxRWCe>+VYY6%i0vU93II&?_~~K$}?USj@i0OTd?SW=N@cciy_ai5!-QT+=)SDt}?y zFkrCOWL?t9yZ(*)RU$m8?lg~4=PX_vD~`-Uzb#j6KE)a;OcDO_-V@Z9N$RL(rXQ4w zz518oj;vgwf-B(>p(3A{>t-o$aEAUcFEf;g39HLh9WLKZ)$U~yj7f)N*}Ur6iexQ+ z(;>iRI*cyrDpZAE8LC{5rt2|~$bAR5;h3Tx3Mm~4QNyl&PT0KQs%$8D$8HnX)1Y0C zf;9+Mn7KN3zmVR}CDUW$6b!4TKmBgyS`#b%FVW*S&sXJadT#JsIPf5|9o3(EA9ocL zK30I%k$g(Y9)rLjWW}L|2nQqvp@gvq#~498ht1#|o_sK#_q|3U=5Z$4Z=U54eyzlg;ze{q?TC-%mRw=pN1W)W8C@g5DC?Yrca zl6A#fcj=!8^y;##%s*yDSl#cCBD1DrDFf5si%}#aD0B=317lnbk;n)`d1h4maF}9&O5QP zHla3;D<$=$rI>36o>u9+dh8i`6}PQSo`C1PbDEwVRK7_biqlo6z8|)O%Evm^Q_{p5Ly0pM763)~%^O5902#G`Jf{M zBnaHolnW?zrz7{bE$zLaFEeo^QJ+ixI6r|=kAtPX!heZ^!kv`do5Ef0^uj-A~OWPEx5M&QVxZ03b` z>bjk5-0In`{YEFoF8WLs`lBsr@&r)i(c@d0`=`Aox^R4VU-x{P-%lp5xBL1E8Y1#T zA>l+7KLe=$*a-&t=AO@otUsf_nAzAS=wHzeoCZ)Zf(vwk{|G3I2sorY0td5qCl(o1 zeO)Eu<>}EP7iA9mYEtTcMY`j_(acqdx+dY^eebcXp6K^E*FOn;XHduyyhJchp#1FF zw+BWMz00bKKgf+hk%=K=lh@4?M31%7sBMTiSGM3!((6ZAnijv%f0D>}9G@%_7^B^A z3dgsQw`{_1JQP>3EI0lhUnm1=xUNfQWv%-Y=-#sgiJ*fAv>r{l9;VKLg)9Bk1ihWO z9CVnvbh(JbvTBOpkDJ2&-nMkU=hG;ivh#|~UwAIpVbca{8ghz>aa{u1rl0qwcBp{M?ZHJk@%iHaP7maDxx4u-Ku5C~YP zftINe#2$zAAW97cFwFtrqOG-nw$T3dl6U#~`Kc^-{IRFUUYjLdK(a4O@h6$uj|+Km zM>&R4?=QF1o$NmRDE-va#_@EzC%9S>s@jGmy+c(yadZMVyV>2O;x9H0eO$#O(`OFW zw5VZr$0z+Z+ul95rRZ>oN*+z0M|Zpqi_ZO^#ht51%{%Ga%d%wt_ykgS&*NtH6kktO zhNwM<2kN^mceeq4ir~}K@G6l~cPR8$D7;Iu!s4^=cFCG|waYENcoEiXS-lB!tjp+# zAr@wJ`5!%|f+x2l2O_0>lpXUJxMt^+nxg11{^T;+uCLvEzdU99)kVLyNy`=z(+6<+ z30_>BM_>haL|~!B55xo2^=G;*a$@manH?kL?W_r6N`a}P%2FP{ow^jEnMQ9&lZJ|0O| z8_D(3U|;F$^SA>w>aRLE>1E2tB?~iu)wn;Y^c&QsqLQ|l=vX)Jt4>oDND`6fIG(MS ztS2)Xa^qD8n~|WOO+~{c)7F9XLbBwSUFCZf0oGdw#n&8}=Ef`dW&MW6DN^{Hv6Hd# z%lyieHk=0)XX@0vTz>_0&{G9WQP2BOP88l-1_0=Rmx}U#K=>!Uf;r+uL_}mIp#%p< z2n^*Y48bE%2sj*x#Xyi4JONC+NEjZ1cZJ~q7yY}6Km*?O$g++nl8oQP%K?UVMIpS1 z5G(x|e}0dHoR;SmG5`CO{3-8Sa3RzwQ0=1fRW#qHBKg&^JXH2kSJpAuPkSBYe})+&17ldHv9CQk^wavBJ{k z%*~dS#7+{%MO?vc$gSP7_eloqi>@B-kxynWR*z>kNmSQ3k)lc2zkirL<(bIYI<<3{ z@bl4^m*NjsrB{4H@w)w+>;vQ5`+mqqve0e;2oO35)qLI(J?Igcc zU8bM*vG|JM_)lK||b9)UVt`n|JYO0}aj5G$zPVYbOWVRX6u$fM5?I`Ua z5APqf{3?WdKX-5*J}0vG3SxBjm&Aacl#f(zJnt2eJW47>&bpWMy<%5;4g+R*sEV8e ztNh8?sl(g=C_q3UfeQZzn1AJ>Kw`B=V(|nVQQ7+EX(P6X|BVQLgVyjbDyoenK$Tow zRj|0<1nY0-O~zwTX4>TPIPZ>AvL2&052P-RHvKQ{LaAFvbC;ho)1Bcu(fQ*_F|e39U6>5GPUzJZ)Q@vUCRo;l(Vhw!C_M0 zIpPd=x5lj2#Q(UVC4_ju$OjcXr0cp`w0ECZ_sJ+t(t7i9Zc8qjV#q^1xBdniUECTk zu|q5Mt}JOTs(1(SnlrL3@wmYl;Q3QuE86xPx-M<>8edTSBB!2w@yeS=T09y}v(N7F z(0#U~=M#NDcG&ni>b|d8VAXB%=l&s-;LuS4Wcbeg2iw0PV~=nEHO|Qi4{0wW57!-)aH0sO!VEaBmW7 z-|Evd)P0ukXOOm>X^J=ETDptaI&N%-)0prI-^rsq4$7S{9gXOF$YH=RD0h++Za;vvx$ zvDjdAvWNV~9UaWiT7(||@~FiJ6->An?YbiosJQlY zCm*(S^PFfN`_8@jOMKPL9EDPbJoift%^?#FkNt~yWGhq6vKR&}Rz6qh4MF;DsiW^RmiAt({xJ1uqd&r7G->*an}Ir3NDbSyZnJLM+Hq=x?CsjZgdfe-BJY~V zO;_)Gd7w4<>g_z57p^Z`r<77!ojdV;ZdMxoC33cK_mJlgDIL`px4t(3p!jFk{M)^P zJK>Oc0@l?@7mD>j;5?8BPY4{;ECe2Ehr-%H(J&yq=Sl!s^(^VZ&1Xi`$(y7{`7m(tF{xcgjn%XQ(`k$Gv+eSZCh&cpAN z-)13sWJgn3FE9s!rp-)p$pQr%qIlQ+9gN}NssxteMynh ziJZmk9){h%{yh&`yehD|eqRt=R-*HBWc6}(!C(n?SSK`8T~k$4<^OPqF-(aiR_XfG zD-EXh5bGG3QUSG>D=adk!LM1)x>H_ml)(N9*pTKCIBm4t+IDhF6)#2Nf3{JgJc5t& zR@QahZN96Yd3JX5?b@TsMfrqA&Q&XG3cuF+5@)QhoKh~PJZ!O1QTq+D^kA#SUfBXy zy&ey%nx|S?gL}8K=fth|Yn2ah7^js^NOFyyPD6hiv8Ysc8?6XRuiP6n8;KM74T@}9 z9yVx+pVo{$F`r(;Y-wyj=)#gU_5J!h_zLL~30Z^8cWX``4cGF6LledG2g7V*draNr z4;=s~X8&f)?n(Z>qin)0Vl!!P#4&hE>EXufq-;3|^$&S5IK7hWlYAU^u7pvZROilfk z-2_)W(&8J{l9PPVV@$NQ#=EsVYAWqq#0yOq&A1gyt>6a@@%4JlHcP|M1)aI;;esU8 z_*%zCghITCJabFGxo6h7A9tZ_Zw+0;`B;D%@hXrOSYvH8w&x!NQ}_A)dh}&x zfykp7RJjjvUL}HZJ9%TX!92Z*)jRdvHo0 z#whL|HgGrSUjl=4pqMkGc?>CAz0PVyDU80T51J`r9h}Wz&a$!$^rvLy_3JWtc6Vz# zo0Ub{^5*HXo9;C{4;D`ioOQTB)seheXt}f(LtmC5I4>UR%zc?}^qocas&&{%NoFDb zo%xbfe(m!q%!E$I5^gVHf1%j%xfRf#(!y*)b|rhI&oQshM_Fag&Ig{;=dAC4Yx0n< zfA0AMAKp|?x{}V$0R>%=3n7tF$!2uL&ILws* zMjj2sAA9+K!^)Um?)e51R@fb-4JjXwMx>lH-b;S99sbMc=O< z27KV5it-_HM`N$>)_PHv^)Qz^Sh?Y7hLN+g#-UJRkG zZH>;DaQb;EGbavdU+rRC@DAT==x_ny_gg>s^h6)%2lLYD<&&7OTKS!dl^+#3ZezPO zxpK}z&MeQ{;vW*SzWZ12E}&(*+Y%|Ci6ppug}7RM+&+7EAo7dlMUW zo5)GPib^vJ(&FPaJzl|HKX@vS8I{-;>MNpD}Q??jSPEhUs!3X5wMKQPSfmi)vOk5?p#vz&AZYho-HR{$m z>CG0yUJ)-(F)<~1MIb9}+TFS+E#^eB#nW>c_O;wICJZGzwZtlEch)f)Svd&wl^4kz zD@?DkT|3ZnZCI&J4hfb26~`HBs0$lZ;$mVm5Bk*76C1&?W{SRflqe--nO8K>>o(?Q zFSn{~9X9^p{ZRDgZ12+w@B765BnMDAGU=jeWCBsaVO=50NGt;52*hp#jwnE>U@<@y zARyh*P&gn@uApIMMia{qNV*h#*J8q^Rz>)u^i%JOPh>>G1w$adM)z}nksZF_`9K}* zbIgqs>EJQlFyo(^6xCZ3gp^_o%8;#$d{ZvutGE^1zBI?+e{rWbg?}vT@pH*6`7#y{Bd-OIFBUhFEPZd@%2RAVPmFt1qEP zda8>(n?pJL=dxaw&h2Mr;SSx#_eWwCEFzn~h6XHr54hb@zE~2JeJ)1p^H5#Q=C|2DJb@W|qde zxbwFqbRU|v8BoR1O$klbhtMK9$E6DlhHYkxzC$@l4KE&YqF2S&qKmD_&}LqC-oh z8E3kfiawK&9$R<$^gWZh8b=+mc-2!ZdZKQVIU9~wUy52h^?N8)GvFhr1L^eWo5;er z!>`4D>ECqjcs}wi{oN~?hA8~4+|(vggG{o}w_E)C_TXnY+5Gi8`=>^*x0|lMA_*2k>+T5QF7 z;8PFMjtc$e#>6K#-$V*uv8#DIM%QsZCnmw~^{RYC-gWg`CWYcFdr~bcro3Qp(mAb%(5B#}XLypcdhB30yke}Vo5f0f1W#1p2R z!s8bO+9`xMWDrNwBV8%9J06HfC=}9J3~GnO;IQ!D$Y4N91E@zJrP2JI9v}HRkBUJB z!9nxD*d$hHn?W8)Olf3w3G_&B_&?RRVLhGx^yeF9ysEDu$t!_`sB$`?d|9Mp!s|vV znI8~}o=YA(9ZW{)b`%|_4P1HY>SQIRXn0I!a`Z-UYd+)hz>$LLoXbe8B=#%6C2qXcXB`pe zrM?Qi52_n%1+NB}dMoJMDBjCU0M}CGvxw!r+OLOLVyXZMQU#S%q2M5rQUFknjI8Jz zKva!%p!xBbM@}KsP2TLIg(6$rPi!{VKMO{1b&U;Z=<85DDY^An_n5}5g|Wp< zzlE2@rg4`uP!8v;tJrJC9|C=<-n-6sZ zNyAceV$#ri>!(L;F~iS!v#cb(i5QB+tJU>IIyakS&MzrjHLsD!cL-O-NQbbjtvwdV z3c5IU$6nlKD>fw{hjwUT&iY}FZ^Y<6LFxVW0l!$ZKS^|deT5phYMy3FGfJ06q3^}6cY3eJ% z2}lnFR2PAA!QyPxbkuFs-Fi5siL4+WqVTkchxk8uHtPVue_6IXHQ4pgmckatRDYZ#; zp!;~7G05XOoA)+Xqfv1nZDdYmHNAZQ{I!-S`a&PF?f4I5b3aBD1rAub+dgp=lZ6`s zuucN7j;vH5ZHIBi5mibcM-rS+NW0&c6LTL#EQa&!Ekl`3km4dg3yGgU&&UwVmg_8G zw#@4@NKqc=ARAY=7xVPz%+kdfNS z;uxIUFio8!Q%?5q)tUtV7a8oyqy;8P=~3$k2{envY%NwmSey>E39FRv^}N6a?d8eR zX&#hQ|ERNl|FNJ-Dv2~bF|YcT>w=1_*{{QyU)Q~i*~iv7eo==2nn=nFFgbGS$HUO> z2oxNP0zwhg)!Y8}?IK>T|H=O?Y#dAis?BY6Wg5aSKpj+Oi!Dt{_5Q%6+K zIA{jwZ^Ir$sJe)e1ZKPWLeYu5AL^<*9w2vV@<-`dH{)4QyB$p8afZiNqph-zvu?Ic zK7J`|#o;Zb6<;T-&3j8$I@R&gsloV7bffOjb5^S%woN`$b4*HadHIks0o|46e7>_G zXH|XW)^Pf8?ohC_^=KAQpwO_Q^C1LIV4(4OY1v-o~hlCUE^!UOXGSn$Jv*Eh`?`4*L&C94z|-9)m{9=DU_ng zdnMrtwuL+SiyJ}a9LmN+zpu8PGQK1*>Ph0MGcFL=+X%BiI*n(lR_GnFgnWC}|s!*fR+-f#c9<9(8=e)5cOs=RrycOutVN8lZh2fslXe9Ua} z&;}%R3VNG*XR19O5c9o~;B{oa3*09UG!pvPCmb-&ff)OL>O7HP{jWv*O?`57$9e-_ zMHZ@TUCRn$t0Pv&RCId7N77Zuec{# zE@waMVfFcO7@**n_S+X`1HNzv3DFle@t3(b*`gOV^yOgFqEIe)Oh27L;OSKwBUZb! z6=&iumb7Iv4s(pC+da{e%0&+Ir7Lyo*v&x}Cf!n&@@_fl8okA;%D>D)mhnfBS4=5& zR3y}3B)Rf2zAMJlyb(!r{`Ef=9-4DPjOeblq(Wa~qjn*^($*I#cLXdmMK&qb=5vUQI4h#Ie@p<<{Bb%evv5c@ym z>VN00V@yxS8C!d;Saofn&7DhI^Dj5V>Q2{h&ney}S^BVXHzkTGOz^v_Ex4cBQX~rf zRdf*%;#a4!Pq~{cQ&+9a{z(5rpF-E8t)8DxK=-&qwqF^jmuF2$XyF==Sa3I4FER}# zcdg$}%hFbujcLF6t60f}SFShC)iZ76bcqSITzh6&Jv$+adOvNM2Dna{MfpN7kNXI4 zH?NoxrIqAD36XJ8PV2`XpWG6y9>~15Y|ZqT(u$1eI+?iv*U9<&VG7X?5h>mhV6YCn z4vxwyc%A1v0uwyKj+h6Ag5&_g5hOUhQX`_YF=V1A9}K4$PkXQl3?T(&Dfc_?q}=3+ zx%l-C!yL6dUOb&9-r5U(7}ck!-)DARtxnaI!5HuU@Rme(?d_+LH@!XMzei9Cuki^H zTw^X#_jhleiSReXSzLB z)*{;3v$K(Lmi6gX##VDBiZJ2Fk_FzHlQE&vxhE4IY-&~9D_@B-R6#yqw0LjSz_p8z z-FY=N$cdx7k;{_|V~(&59YqUdx?Fi4*P9dmsbRt5N}*cTmm01^Tx(rE9a`8nqf=I+#05s)Pj^iJe#QQ12T07Tmyd3F~Yts0yI zP~ip#;Yb(T0R~6dVX-d%$>2ia;`Xjs8w?Kbfc`&iY>eUAcg`sdxHv@~DR6;Ih%V63 z>hNzD$ZxRjDOTZWL*Y4~=>L@|34fB^6D!p{+aE*q0b+PA+=I-L-bDr?T5ZleH}}}_ zd0s99x!6LumgzU89ZGM%IITPjo|Mw~QuMo5u@;>ut#FVmS*_Ue9;GQnQp;+vDa$2h zZE$=$7a?w~9Vb?>w%^P+l5u5}P{3XqrZ`d$#rD`;cypB`h}hC(N8xsdmB6|HomyWnD`d&az zu-C+5jBzkmjo&e4eUIPjJ|NNmjnk9H*tQ-|FD&h7V3)Jylci{uL9;AS%WKk`r&k!p ze(U%&BfOygz}Sub9p^sa@zx<>;k(Ooi+~j6lGGiujwW4@w_M%41XGF@Y21 z1vj<%RkK+ePt`^GmZ>+!g~Lu+Y@&+j%;b3PFNJe4hfbL zp?WIDBH}W#;u2zFe^}#xVQap_l7il`M%d6wv7MK@DWYc#P8R*3Q9SWv)%-C<8vLxy zbzsy2SxEvGuP749_Ilo_Sk2CTMCm>Z3#x2=)K!?Eb;7g+|MqU;X%>!0sJq8m?b=`K zm1*#nuerY}LO{KT8gYfPP@M}IkOa%ho-O&Ou06mO>6Uq|G44(-tyNAj5N`@4}+w8NswA^?-fYd1>$P~U}phfM@LeT9#{gfsZjtkxc7gy z4$xQ(5+FycR@y|N-~n+V16_acPPP;ZSBp16XoIaXUh^pTC^Yu4be8p{%mKq5g42I#?m*9;dMEd98={Qb~2C zPU*aj`Nq$q=VzdtYhmm!u%cckmc`a(pxh+CFp0E@ubn=dG49ACR_eq43>&mYkaNE9 zy>0JiplvFLaLPz{8fsqF$ct*X&Ppn5`0PB{wRhi44}^UtKW-LNr$3p3<<1Kn+;MVX z;<_gp7ZtDU8au=jsjyG!Ogk9N)kk^wXuEM=u-c#8Tk+8^fD>_v?#S+LYKliAQ3!7@ zFxC=a8v4I4+<#`-%pM)9$5jm^LTwc8VHA!rH?@+1@|l}!HhUtK#q7>I#3P5TOmFc+ z|7riUcg!lRJDoYV?DgoL(F{VdK~L4k)(6=r6mB;$(a0<0#a+42tXhrBWM}M6R4*94 zSy*yg(8Uq;q~%#jT)hV2VUnhDX-3_oaI0&FZ?ci%f_Sd81(s*bpvPj=Quj$gbJm22 zG{ooqShA(jo=%5b!$H?K92@>AaA@Vy%o^#6v^fi}Ui_iJS#{k=XoInQNcH)nGJW&QccHBap2mUR8ghs2l-k@y_xCpiN(-x&>a28uC?fD$JF$Mzo* zzLK%2lBldO+Ww!~$~o|U;$la11r0+iSRw{Ioj{z2!QP9F2NLfNLm}~h>CJHl+QjZ) zQq)orNhyLu3HSRC`%6?btGB$l$7=$5L;Z~nQ(|xE*+ee-D?V)yzN{);8-c7~Sl1-) zj*P$h4u12)g5-Q`xb!ihnD!3@55X6nR2@MPCi%7+Iy5&*s5+R>bY#l7x-qm%ceoEZ zx#!oK68yJ75Uehmk$1uPHhSmR1L^lejG=U^ERix5rminySv+FiY;#pSE-Pg@z} zLt%1ryJY4->phvN%1yGfLkTY5CO^f#Xj~0)vwA=NPVlBryVGeVt`Ddc2FIqUyLWiF zuQnX(UT__qcUx7waNtP_#O~FvRQtaZLl=5o7l@S`srKlHmN=8!ab71_V%dY zK5;QCzPMh5mkqffQ8n%8-ckSlY zjOg6cS*KSrO-!lteyF<)Rs4LxVEXV}=drQ{oE&D$Z;JKi7IPRq)5RaBwom-L^X40) z+zSdxLV=V|@Zgzh@kQI3^z_|t$xDsCoR6L{lQ{GxtgtS-Yj4pqpO+3|l9G+wrl1ng zx!VM1dc0IjF}=yd)wo6bp6@G1_&)wH_Mq{5s&xa2=9_4uB1LSOM^X)A;^d|RvRee` z|Ng3^Nkr-IlEUG!HS2dKyC9S!VjI%-*K%;mb|g?0N_0H8GQ6?V(m)rMKAp~U5M(aJ z*waCF^2wV86J2bz6x$!xGGw1dhL>be(Z2uT)MHZ zpv%|p>V>N*jJy!Rjh!xuf*<$r>?*ym{Ves6Ed2YjmNXy1{);(Y5X*uHlumZpN1X2S zDDV0N7Y8>FVxZ6$ND7y++Nn}k6T|lNgp8IyalPQM-o76wq{2z!NY8TcefBGT)9eYU zV~2`+OZPQ8?oKt5GIE94{|m|Z*Elo*i}A)gfhBbm))NZG85_L)@6fI;rsYonb28d3 zm51@^3t~LAB9lR5=h%G)vt>~k0d+KgJmF8^ZU5tK76qA)r=JHQKuAX zca&|g<3;T-nyoOJ2hBB13j_LoZ?47PgGn%Sr9eGS@EeKGx*$jNN+Jb?+8kRdja6?df%8f7gcT&o&iB?qT8QPQ9Xy59(z+ zMac|Zh;F5p%FCY!94ryL{sCF=%;#7?$s?-pt3g%1i4Q(ZJas);|B=JGLiG|f!a4Ri z3jI>)_`(|lH3Md~;nPVSCK+C5ihKs#P?2q3_>8`Uc{x-0y${0dKe8&tGn=`-)5?FF zhrl&*w$j=LI(2TlZ3gs3bCuL-Z+Y(V82;L!F|<-3?P7@g25=*?eaPh_>beqm@Bu~> ztScZ1?gRwbD1&1ia4tCtd(`vFnp#+I~(iJQ0+B zv8_X(uf;fs?XuCLQ2spWWZ=7^i?rMkv6+Qmb4-8l+J!(i1p8|IpA>nQTP7>t;+yb! z)>%#J{xNk>MJUctzB*TJs#9Is$H02@9SDQ3Pb6$~!+0z9NQAog3=cZ%*4-LTm6kj` z2i%I(V3i;FFo$VXN?`z zVY$>cm6MIh^>krsFG=q#gLN4Szws?UN~-buUKMCfI{LR#W1J<16@tDLmiG7jKaB;r zz^2dDe*EDZ&BMQs<_Wh_B!%XtJqHjHLqA6f5_o64D-wYs8ltY=9za3NqvXQHu3YVj2 z@Y^y_Opw=fYaQ6)`20!$HeA9Wpz$^%-lFmP>#f|3xbgzJOo`H*6vqw=vDl7*_&txY z#l0*u)-C4mQ8M2OkaU5fDlQgAISnh57D6Fh9_phjZ?z|LLItn1r|<~amzHTRHS6G8oP14cQfoxXmuM8 zI~DeaZO8PyERg8>wApq0*Wo^R21E;3H<(Fuh;-6txBWUFS(4mBmBnVrF8YJS%YbbF zymY~hYwcL)B@@ktGg|iv8Fg|zt3E>Cf%4S7WSCGn>3oZYhAGE)Ap^9 z?oG$us{+GB(Eo^D@oQ7Cb5Kk4imx7!+`QauefLlfOpH!$fJfnj!cze-|34R8@Nf_l z$Ka6;#DMreIV(}`Ig(z09V`MKje@GVxg$Lw>MA-AAjF#zW3B(N1DmeZMGTt6VsbHg zylikPvU1AwOW+X-G2Eqp9-+X~lX7_Ir{HHjwwzxgkabf-!uz=GWp{iUXFKUt-o59+ zQ~Rid@5tJy%D&)H`LHp&dj2fcHum#e%dugXPG9Y93ws%>VMt_}C|C$>$xAOSl=@7{^My<0B_npS> z*$bVoW~LOTQ|3`~yf2aF)P5^G&~Nd*>#gU?*L`y;F*6mu3h+xVKOGYnt>D?2k8>+q zpJ_;j)1lkb47)NobJZqcpTW|?r)jhHVb{|8?m#}Smz<%1`c89S7U^@T8mmy%C`w#+ z#-H7;h!`i&`kfr_;i%*H8vA*e&cpa?g=TGs3cngeR1JwF^+;FQm6#L%{j@3K!)siz zz?!xJC+kmx{2MH5Y{5WL)fJ&Q-~JaY8lTQ+|8bRjK-jepJWeU@HK*0hB!?##S!id; z_SWAedp(#hx#>9SMVECglx7jm+noh{0UuJjCxK9&-aGj?`OkbGW%!L z+C-SY9%mdO0lLn}frl){+j}{1{JU!jIz+r4UNOCRZ5hS!id=^jY}0;eqiiz2_utrf z6K3Yx&&W!E4$;WDB6~z{)CCVSz=HLBVBw&!P$L2s?qs719wFg~vrz%t={8C@m>qE! z8&EIeQ2sl#nCz}Ic(cJ|p_A(Kz=d&{c(8;Z4VDm~UtoSLAdaLlD>K-;R_EQgFGB;K za1bV9XMgJjX>nwGL64kbE7OBso_K|R32!flnTQ^~p6LgR%S+|5$57VHNix;THxth> z$%|!*@W;K^tT)t^$Hj@?7QGefTZrml&J}G@I0Q=?R{TVvaxlO9nTS>tHxc^tyLCHo8>I8Rp zwfT=I{m@rIK}cXUbaNqSHI z^n(Z9(%nIo&QkbjU@v@xd~gz5(rD^YrUN;nb$XDPJM}%_BM;SXC{dp+YzS`wr;;cN z9qCkqM@0}E!KT_@pFx5V?A89?I;O(ZDqB<36fPPAzs#0h0P5K8GDQ2GI(Xnz1RxxX zjYZiT;@Ywgszh2Zq?fbZVL(n;$z||e{;tFuKbdNDBGJ56uSC^OeVcYGzPN>Jqb)Kl zO{NZq&XC~3mdD3zJOxge*~@6+Wd-h_%dItwEaMH4DLHAG%;1DTLoK2AUDKhb288nJ zSzm1GU4dRwd%F6~24dcNs$+yKK4~j+PuzQQ)6WsUCGwi7fnDFJ#@@4G!lwRw}dn(H7ZD_bR!^0NDC+>AtGS_D!=^< z=e+fNKluA&&f%=HV6is)+0VW2`?_A&j_Vg6iwf_+Y5snN95(yAtwYrHmV)T_K0j|I zG+%k!dVBY_(6$!}6K;dtyw8HU-sQ`hFcF`8<%8ZpXM%mz1EUB_z45B}@eHQ{a`Mxh zA-hfs8VVEd$-2Zl`E2N>=L#{=W3iNYtOGfqC;h|ak_~u zqBo6UPr>p)5~O1>E!MyVV{K!Ja0fj_I1pch2DO!k3@FY(*9)NvsX+daBH@UDA9HN! z0V+EecLz|fARVmGa34E6pmPQP@t^PhI~dH#0kp*}0g%7}?(X9Z8f^|(v%T?ATT}nX zjkGNmcmwILp@}l`ND*KJ*V6MO`gs2~aOLjbY*H{t6LfzKsZTm7Fr|F|bG4;Tb&)rO z21T~o5I=zTi7%eJ+4)sjwXfQeC7%1~qBg^K7tJ4fXNr9L^=7`I#zkL)KMqkMp+ld| zIJ$cw>w?J+eIElRXsd`X_dGn>Gt*T~P>ebMOPAY-QMcsg=-uF)zPdwh${S+hZQ79j zq`*A32F9>y!Z#_NVVarykCJ_u5BssHuO6aol z3kdS3iUSr++|3@VDjD+vA23hNUXat*vLA>*>Ni)u>Wt^m(h8|mzTGwD;IPSuO;e@2 zQdDd1R?lv$ybz#=Avo`NfN1cg;o@~XRJ~td=bHywOdc2e@6?jJOs@? zffXmc8xNUw<=V_|-!r|68xa-=9UzdO^;mfU3hnEF1f67CAc;alGt<~V_GDS6(Ako} zL&uDLHa1UdjR=)dSShYZB=@y7rq4e4;_iUXwvQ!jabQxwWb1^>y&sr$PTtav8ot-3 zsd~DD^*++OUBuL7TPm{ZT<)={SXfYhd13PMEH|^`tjB@b6-c?Mi5+Uy|2x#WiYzrE zEqbTm1fYh-Ln?)SWyxcp-IP1{Q%Iy+cF3mj+e1$_o{&=FYnbAiHRJcC`Fk>WBa7{m zZ^Vs!JFOd(?+vC(uoKjtf*>m;^Y4gT+dm(QLVi|+MU6(acnBIS2jrPM_Ew9sbP%v} z2b}>RHMB(9D9HZy_y%L`r%^MQ=*B>N4vSc%-I`(Vqa%(JX{+)P!7l*|$_g#1NYNp7 zB~#da1q2t&Zq0R5uUVpa3m)9VsOHkGEbS@l&QrKsBhpknlAhf?L+M;Pd3lekO2+0* z^(@0;^+V7xDdS6pmArV%IF6`Y?%{p$sE>|0b6nCx+dG1E<>9wFr@FEApR+ysOnEGA z1Jklw-EJG)csHoSQ8X5yt|c9A{ltfvDzadW7gENyGLr&&5U_3{tEz1H;8 zxv=-PKMtpVDb*U#0u_q0NjMCw3nZR9*5c!ga7Vz^Wpre~!vn11&_4OYlN*)!Xl(Y& ziJ2BgH3p8d|2dyYYPqpb&i_ogz(ODRDEK_f4n%G8F^SD5;Yi^;PybcJuY4?@^~m*9 z&MU9@Q2?WbR}LcuQ++k zv#9#^^kde-yeWqnWICfB<~!=emn9$Z40;{LYi%aQoi**ipCWlS!fV&g05iD5eO&1y<-aGFd7D6f%{;+k9{ktK`U`_riefF>i7z-C9!r2$@B?^!h zklG0>3v_`vl0M_kGrbb8i)AOJX0_y@u2;!5$qrpnhAt2SrocAuvw7gZS680$W6=)C zQjN+XS?U+V*e&8k|F9b^dwA8e`xXL)6U!AGn9zvK2dcPi$!^^TnY!ql(VX$3=MNoi zrW^j53JWJ2*N3>g7UZ4m+z#2;yZq*kZn-$gZ4NDuv8OT~58nzixyQp{(I`wkv=1U+MtX8CFC;yE%@8AeXl-hw2lI-mGx?uQ;ppKm&V z6aB{baPYRpJ@dEo)zQWSqRA-h>L{vV?WVugO8sJ9N>=7L%&ifh;?|=W4E~RI8zOu7 zHSdX)rQAPDfSFVhvLkti7u{D~?Rr*{HTFEC$m?$!u}5Mh7?%ijTia>g?-6|uks>7H zIir@QbI;5?=aYa)7a>3ce9E4ELZ;iTuY97GNCYk4WyWVBefucuY+(#FavYg@jjPW$ zri+}kC?jTs?Ni+!qFH-|mg5b+G{=hx{O8qbMwDa<4Xc;m3yqt8I){ZpsuxX6BdWN1E-R+HScec z-~&@=0?=Jmxadj2!KakqzI$8X%{`^W#~pmY5L4Rs@xI!q+Sy{UI4}VOQ zcq#%%=H9qDjv3z6HsTaar1nwU8!H;|cw}cXOkEx`sK!C*kn1@9tmz>GCo)QZ8PC)y zW8~m=Q9+kY#OuBpI7febIJG+t#aB1jopt|WSKopl@v^z6s;yfW6lghpyYBU#BWURi zyiY&?m%=?|d5deSZXK{HU}xuj@p{%{w!kif&@acnxHN#<&>ajV!Bp51E~_S^qw~Kr z8VjfBiP|ytBk=3ERI0Ghq0E!7243fyqueg1)g!>`yqrf-5Z$y14b3~|XI4mMBg1`2 zfc5GMihVsv>Pk(CLrVCD>M6!Yj8poo^DFnQve)HzUCCSN-BWW^cYfgCfN9-$xS@Nm zph59XSRA2hL+~w0f-bAt%;nLWRfPp7=-B$Qq9TCZ zJGsG-kJPV^g6P5E6ZdVWt?IqK-XMR`7rFgy0-q&)ByVn}D5S&goHSxk0yQSPH#MwX zZ}0F4G&dIt+Yuc{g{*>=`Kwje!`2by0(Y@@5%6?&fEMMC&R45g4B>|rX<9xJ-D(*= zbqF*xF69i&DU(%4h~n8t(#1TiSN9%rE{9p_<0Np@tWE{t+xwedlE2Vd?WcDaB~}{>7(;(&0>}fHDD;u_1YHJaXLE1F5z_i!c?Fv_JFj{~ zau^HbUKT1#V~>+^4h9B29T!dDpG>}6qtbQ;t8w?9U652eWqm zsMlDiT?ec|V9B#=2Y@aApRo4V!eRt9z|(n+Imj#u+9WyNtg z0S>2G*o!ktRPS!aGJ9kT%lW*zLy`vJm;bAe6Lv^`kSqPBlH4K@kdNpl=+XB9CH z*5$84cvUo@d5Q_cZCvbq+|AWgLG#81dqew8l4zsG;i?(PN*<3m z!a&t^Dru>vUi6e65mU?%lIJljNH7 zHkfeG_%dLmDt$`~7|xBOC9|R-1|uZSKK5u)wLya(skIzj77ch*FeL{p zfaLc}t7jsnCpLK_1pmfY`wX$N{K|jervIlX79<*a{p+QWyS&pduzA&_I-VxDRfZtx zs|__h9o}o#Y=TmZ4w-@uGI>(&+h>exbdbde4&ACukaqM?&N>3 zx-~uTW_91y<(G9K1v^lNWJYJXRO^KazFUJj?cOiik%Z1&hs_&+nvj3bvrti8z!Iih!3e*>S`rIrA?D}+ zfEMNCW}(9cNLU#;alhscrBQ9-*QdD8H9k>uRlrKA)9lJ)^iNkFltr=gmE5VyIcLXo zo6LyPGOt7YS`Jc0H~&hbWrTC{PPaVE*CBm9+8 zZtFdo^biN_FUq~PdD#o3YST64g&d>S2Ky7}+dlSGL>#?%N?=a-J>N&N&0kTzRV-kc z%UQ?wNlQ$ihV_f7`@yg9g@NQDo9XV-N$+#9HNV8)H-856u$-a?kzn6QgU_)dBQPTb z6A4o`4K=_aD$zN)MzmoN=fbY=F#=#N}wM+q07CY2XhLkuEFx9v_ z5j7d(5%Y;iA^TKI8OaeW^)-N{uHsmy7N0)h|KsrXmD^?=@j0;-e3$r2CUL&ML`c5z z62I3k!Wu#FQYWM5I@Lq5ZWGKZx2rs+t*U&(OM5fVH?H?-*VEflq((Ih6@d5BX00#_ zO3i7ZnXfeC^hw(ahEEh`9_!a_#tKagRHaEo!s=wQa~eA^OzCBsKbmH@e|Sd&P{j!> zs6wT|*`bcS)mrX~PUeMfbjoGTh^VYJHlG$@Sv)Xb%OTC8{#18(6)-#$Gi6Mgak%=z z`oTiWg3qyv2}>(GcPqFe!bS6_x&(5PBNqnT17QdE;^*c2Z?g;QPd+wP!B2U^d6|7|7Wn>UlYIJygCh%7ak3>po#th0uIlRh@2H zq3lZ|k$#v}T3JWOVpC`3GJD+BSMDOYa z#lw6$37p|Vmw?0N(7a|?aQqI(BcV(Ua8ihYzs(iNV{QS~2sgAHI3{!$nw z`U|2P?Mq~atpn0NmPQnSfJM(C(@XL+`T@Z+A!OCl-cyf@-w574^)gFd2@qbpJ<~q< zcCke>ck^+`ThZ#=&_q=Tdg8C>!jaPwfeNFg(?(9Qm+DtgFAn>vDnOs`$ft&`c8(sw zW9qsCUA0~E45FMphR8{aiCy2(L=yd4S~xTeIvevHZ+K4fhu9wa$vZH!gQupKf%VG* z>*wKq%=)3M91#F7<_=f$1RD};IGB!uoioA%WPP~9l%1`RfR+g08Zw$1ve)GRtrI4P z&8L7t{cxBnI3e6&D(Z5|FgYEVj;9MCaYC{W1lY(r0K(w{;g&le^aCU;MR2QP&lpB| zYBjH4c5vRZ$G zd3@%^d(4&3(l`X3iLQEeC4}OTy+XQL`EoOhZESTF@FSx)w!Z9)^rNDqJF6;HBu&;` zJHB-(jL%p#wt5Dpm-6IhRpPIHt<@Ufd8c*XvL2V!@P0L!M@HOG6vAkpCjgQ9(p_ep zvt$h4*i@VAax@K7_erKkQ|j)7YPYR+dZ!1));$J7uP@u#bVkbBn7BTyXn;+t8$?|w zte!VjZM`Et6S=9t@9@C&uKc`%DbXy~ICT=+CpuTXKl5fXRtjG9%N!`?aN`e( zE1};bh|e3pHlX?A_JKzH4=QGvZjI%@W@x%E3O>IltA9mUI$GPfgFn*$8Sg_wo@0qs zfbneahB%4`@^o{CyqZS^`)`*^0?roX^hH`>&4}1ck;TW5)AyfDk)}Ck;3jAK*#_L? zPTDxR95r7?&)KHjc01ok==T`Lix7>&S`l^1EoghJA&dPiFd*(25f-miONcOhn9l?R zY+EvB6zAL&Dvj|Ui@g^$=|pefSb68bnIU`b{icK`m~ymj&|R>^Z+*@4Gi_g{~F1Mt{0-#YTxn*on~Bi`%yNMNnQ zW%$pk{^5=TSxZ1s2|1^&K;#l!PgPf2MnOhL7pgISw^w>p$p}m~>sBKugM`z>)Gayx zg=^%xC_r!xEIIW=1s@h4w>>ZhA3 zFOe%=U?4KJ$>FFbEGaE;OB7Fn>DT$csp9EX<8*b8{g=1nWrzh>b@V(DZhKR}GAcza z7g&x-s@5-fR-_vi6c&xlbMSq5`4@mdufC%An#9R)P4>~iWJ%vN?J_sh8Cy3+V|hUM z#Fw0Wm!e?&o$myU6p}}WoNckEDW-7V1XXJD+Qosn^uQ>*ztjj&GwN8oCK&wyWhFRq z0RYV%D4YLZQd1NIM;^reR5zX0299V}h}rOe>AL{e__{5urGWzP9ZTqFwT?vVz3OY? z-6zB;(^$e4kY(~z3ZLj)FM3hxK3ybTQtvTfi@w8Oi`MeC3VjV4xw66G`aUx(?)!$2B@81#_+nFgW3S~x`+PoC#t+!c-_`VlIm&dF@BBIDc3 zRT&HtNCF|oB9I2YTBfe!dAEgG3m zX+%9jn`^XMwX8xk8D?~on>MmHMM7t{%;aGECB1@13c*G94>z`)wyV3|cb!_uKFeXA z|03Tzs_7g5Ij^QPD}vgSQ2%n~*PzdRI;^Y3r9`Z>Pc)t<7??D0;NWwt`M|@|+6HOs zA!rSkvDO6wr9UcEu&%V=+m4+MBHUmA8VznpC?Ng;-b#3s9d_@76R>=}E2^@5a*YHZ5Jtbv>%j+L@=kdC2mR9DyM z@_{>5&n@ggU(fWytBaGWM7+0pV)M_HG>qkppiGs0>}kLJ!&3d4Xg4GMx~5@13G-l_ zFPAo)U_8~Vx_~iiUJdR$DwZ20=vI0;R*t()b*S5Gz0Y77G#>o1Fx6%*v|q-i2Onty zKo#ICZc=Yp=W@`aK+s05&(rrHXoCqS_rC@!zHiZxUJ#O{9=r2UkgNm*m)SvO4_r|b zrl_HI9h$B@kb;VK1+08tCFut_iY7w!x}*m@MK=k5B8hp3^!`PImFvDeT=qoPs;#Sd z-<72A0Gn^>*@3!nhcWVgEj&ZTpXG4UJ3;5$TT6HiuhtTJmF#_5+H>f`=HECl3x2(4 zO=h~Sy==0ab7$e9zBZAbOE%Zg70^7_(1gH1v(7Zo=8I|6a8G%gqL5l$L^s+?T8JUj z?kSJ=EFjZzrhwBmX;5S?uQz$voXKTYCAc{9;l|g(?WUmj3LKof&cfGs85sCeW=|^G ztGy4z&tQPYp`Qru3ZE1`aX1GuwHoI^+QIj-(C=Vc6X+ke*fu7`TsuTHnD9WJmzzFL-S@KoCDjdw-2s0hf%GH=0?!f%)PLV!Kq0Ik1_a#R+y&S<%>Shw!y#NPu)u(_9Iy{<>`>q>@#lhrx!Cc- z0!If|Pw?a9q3lAyaB_ypU}s}OnyP3WmmkH)b_2J^mH<%olj)cs#msnBY%kO z<;~5va&?)?tn(IuHlP>l@sa}N_EPT*m)?d|u@Hrk3?}iutvktM+)L!tJ|tV!=1*3i z+nKbj9cN@pFBVoB^+YF)yLbFLthy z8?8O41Pf*WKB_m4`8zmz+Iu>C*f;|=nH}5`+Mn*8Zjg-^bWWf;{6Cf^NUI0|u|T)~ zkH5)Zs(`=RcR-^6?F0^ExE!!3g49G88<-suxH=IqIfbK#LF`H2Mv>hdBu; zuI@B2b3z^A*pZ!w|Ja0~=?m~Vv0~n}$qH?SIgKUO-Mu5&Pi>c`$TZh(x%w-LKelvg zN{x9ybJJYux;irfYns6#j(z0yJ|q2#Z0AXCbk&2pglLPRwGKgWO>MJ2`PS`)l=Fzk z2+{M&ZcDk2?76KNC`1Awrq+dwu8@Tq{h%H)x|&We zjlW9Y+XQ(K2q>)JNF8BuX0gUfstf;0Oi3i+`7Zl5%Omo#?*mFfrD}`#K3mW*XC{qz z;;mft9C_Kak>%6Va4q#gwTagZnDzO$2_G1K;`8VwE?z2FkoAm?ljEp26X*Ea$W3mC z;?|p^ikKZgXx1))Jgz4oz|L*S2BgEC&n-gxQ07+5EFYA)m4Sq&nOHf%<}&cOhAtll zd;#)^Xh0#>d{D$6UnbV{5!j>Uq-0_384r6;FiwnGesGqe_#>9B7{?f6^Nz zh6yWFZ%p*X?9ysMuG>jBz4;*((~S^v?zPk)i_w>NS~ zDypksJKQ|{)jje7#17cj5V24#KOk=TcuRcW@iQHgF1qJaY2CZcT-nJ)XVqt_A`+G# zz6ofSYEjI8jJHDM@pLF5EAnN7+19kH;D&TFX>~=Kt94qS^yZHwBWD7O*IhUJHB;8e z|Lb`n`_aB0l{a4oy*bvtTz)uK74Nn}m*F9*X;uGx{+r7|a_$G43RQ6&LU(}N_b*X2 zVWQ?Fjg*|`+6$k%JjK?|*I&Kudg=@*>s!yy&H5JA#AkH_Vz+XG=Bjb$_wlctOdkNN z2OUJmPJ+Q(8NmD;fa4dg1PXCg&{%T=w-OH@5Eo_xsrEGcp;6|T8QZwl$GpVhD>7%N zQQvy*-c!1!X(^$n2ZFFFpBNXQv||ewBW(hjp;B=vaDC?ab6WW%dt6_RcB2o`k1l?n zR9j@f+8($3H8=<_T796TX-UgwZ83-3#uGE;af9^~iL*;R|0o@8x&qVo&+5U^Uo=CP z*i|Xwg)wu^3}|{+3>&HR(B)5QrhO=z-nP#&I;M|#pA|fl8?o1+R8@0uN!D~TbN1~) z8nIKUoKl+X6FyMeZ3@}qr3|)p;TXu>Tns#MmOexCH3Vy#zt(=1l<^a;FI6qkcI`Hh zXdB}^T^DWTN=N`p1^FnB?F8I%K-jqXc*3>iVc3v@Bc#ePZ>w%+klW%W0}&kJLWLF6r)Mr8*= zJs_wcjjXEB#8kL|Wh%KywnN>sO27O~dE1ou40}Y)(r_|6#-&u%0qi3QyJp4`C+ZTD z%O42>Dz9fSG;0BpRo1hbPI}+Rop`{#R5K}urYRo7hFKUF$z(L+hI|Tx5>%kp_px^r zl%u;7!UrS(TKiZ6a~xK7^2bmUm&^+2*3F~&sg0mx+6ElcC4+^3@(2K=*-#ZhjO7uO zpQ6Uwt13~;K-}ruti=sRth-jE9_h9oJ_RH+B90etH@~}bOkLotdgoi>_$*Ii8}gRm z_XbriyR8FO>=HRI6pqO3)j*Ty@En!T_p>!zJ+-T|2t3RkG6@LfxVBV8`@i30Imkvra{3cgS4{&MF+HEpeFwJwZA~4ltJw!b}5R=pu^gFM>I;Jes|U z$-2%5HuPxU9C4ED?|dJ{$LovZW~#5hA&Wk5Y1e_7OY54V^6dMPmM*)r?qGOx23fz1EZxADm=qT?Z37_m$N3E0oz*Rty>>~dcYJU zU66#!#s-5O1^qmfn_5z zk2DFoBV<#=grP$n{msaEJ*iMZ7%S*ac&Vwr4%2Hj*2>4AFn zkIOKXN$%-BFZ+6R;x?c!SeGiH33xdjstAK|^sg zWu&erD zP%0f4gl(En4x7_}>BZ)ls&Kn3SIk}B<;hzIt)hZG=iwrB8850mZm5IVH^@Tp4d5n7opl14)yo@xS)MvT&NK?_Vzj{pJrLv3nG2MAew4egE z+eo`l)rBjg-fgQ1M9*%oxz!@YjQ(Yl z?OPwRS~A&wmCo-?T#Y~Uma?lsf7pouqCu=i?%4C&1CaKuoZy-^2&e0AZdmq5>>p8& z+FnspQ}y02%?7DZgszw5a>yybqsFrP9D3AJ+zxP<_OL<&Kq3s#RS=J3qq*A?X?);z zn&8}eTWP2$=BHB>a{)FYTGin4jW(LX(gM*5lt)GTlrH>KdigWQqAi+#^9MfF4(z~~ z9F%!M{(w{U*xD5#kqJvhgbViIc@0t?&vh*nx>QZ$o~c?AZ%B6_O3X8*O&Lq+Z@O<_ZXn`VF8W2F zC;8Rp!44Fu!;jB2@TxS`zVB2ELE5)$#)UKOD+^kmB8+V*C1puFqWlXcf~<*HULe&Q zB!%a1p2-c4Llwz$JiN0A%xOT8?Z?{;(gxC{FI0$AG1;c;8-9F9SLyD&V$;pR>)X#! zl&0S^%f(jd0b)%yk__r7%Z#Ai@xr8V6Bq7h^hv>j_JhlWgd#AfeMIU2b6QLfo-;P< zr4RWz>0R;HLwBGmT{i(asXgq0K(2%XP?g7{p+U@}rxi5TQg-$NxljCa5Hhe$oE&P_s_hkH9vE*|(9 zdm=0b3E#L|t!J7#NLxPBs%BNPpGy-JcgsJ{k%uDSV@-pbE&?bW@gOn04Mro-gKLGie5C&52;G6(0@k6p2bt*j6Mrcs{wjg- z^>*ZkGJZjaNKR80rXvdxEshk>San_hcMewycwYnq*~hp{b+TMY0>k~(7ax+qfP7yn z2a!j>e&lzQA<6XP9u0)FzyU~$&WENG?cJ+t6BE(Y@83Jr`^-uoA)uo!PYjR&70fdc zlSQBNZMxAQxE63`Z@+%=9;9UP`~$Vvv-lts;veE!G^`>%-gky>oX}K!wDL=_;7RGkYl-U#~Zt@P$gn_v7;jLLKw1x{lRe>!MGwhM2*T9)(woSs(A(gyBt0VzG~I zp}2&_y@!X>OoTNmc=i5xw2Gln%mEa%dF-V|&l0r3z#T>#GS5SPN>E<807MGP+!@?z zu!R;RF2kzlqr26W!ej4Qm_{>j&~SzaN^uqdZ5ffP_|beSZ*TTBaODDmqtM``DC*R4 z>pc3f?*1g+SKIx=2MYeagd%(|SXnR+=?6mDJj^KOmBJ0F^NEI-4?gC|tTSBMxYj6Z z<>9ne@J40B>C0&g904HFys=k3hgmIvh@$1OpQ}+VUSnK(PA<`&uQ!?koy`%Wq7&Z- z?vLMo$SGYk-tbl=a#C;^$TI+SKwSdqT9+C;6z#EVb4Dt_VmCLoL+raihs?Z+b`O*K z?9bEUw~~&utxEw+7dw~e){7FMV-6~_j>TL$009*oL^)xZb?)ZYKuUoZpmD2W+dIX1J+3WS4>%7J( zjMjV;L*}N8&pvn)JA1SWlJY*`!zpGw1D-kM+tl>!?LQ8`fk?>bB4G!jUQ>}zz`8&v zqGR`vj}5{D45gr^H*o!UdjR{So0pBd`H|=6|EU{VVy9eUKIY;E%D6@02j)T|yvJF^ zz^%-az69_8c6E7YV_iXZ7Z6TaNnLNm4^QUy6^y-CEjjg>c>P4g0KduLD!q4ty0L!b zc;2OY4aV?A%2XUReh0&L(&th_GjXxy7vdx9Hl*dpn*`ZuxK(9aGJ1>p`?d$Ukv+kd za^DA?h#EFd^&Nyq%2Ybe`&wsk3%A(^pVKF{%Kls@wXHIFY2Z{g19q}&h8}# zO8Jc+&TAArW*2Ggauie480J+GI}jVI=?8%XU7m)V8Bj;B1~6Ug`AN6ElmJ?%xI$p1 zj)g$^fER%;kek`@3-ANN4${X3Z3U8hK$wU*FpC2HFZkl$_KPO=PN>`5RuyxTugr8d z&abkX{~{s~=ZRace>QqOK?RJ>%kMeu49otK2LWVS)I-JQ`WnF zO*F93He)Ek+wXI3dyjM9WN}5iNyuH9>)LUVY9i#5`qpjFBNzz=NLxrVGE*M3uzCpd zkq%1ZDcx;Texa1-CW#)H*@|GMF}DsA^NOb4uO{G+CXTVBo;p?EgnY5maU(Q`?&b)E z{%reC^DWbs904jkrNuA$w?ie-LsBO@#AyArJh$ENbUuxW5XsfAicPaDO&o$V{?uU8 zEmUO^_&L*YHsMz_k&5%9lN2d$!P-Gy$zx}90E+?|&*1H+ZsP$$!<^JTkw_&?NX+=) zx0tbiqKh}Y25G3rGr=O6#YzPdL=yBtoLViRH>B3PnRIj)1StPbiV!}*c3_sIyURRS zTUQt9m*T>n@$-yio|Womy=PjMd6r=(DF~WV*pBeIVd&D6|Mv8YxP3jB_NutLF%@Az z^N(D{@3jV{X*FE>g48ryY}t3dxhodaVNK^VhkH2z6=KY1>G{HRchShix84KUp>DSx zZsEuYmPzdvh^yLcJUb?2x>5Fwt{r!s-cMo*vMPABNMLY3l3fFO#+m?}Q>?W{pATPN zC37|<1xd>5Flb7Gt=@m@H~zRXD}xmREPr58@PKJ~f(9Sl!PVT(8DjeX30R;HbloDU zd!BgP`jrKb__YmYEr=AU^4buZW774nL6qaGPOB5$^;LsBFvBwrBC1(UFNz_DmN=R3ZNDM|||Gy-gYaWEj3Gw!WlplVZl zZL2mC|C*0`jY?9OOY&veKwvKN`Fuec3U2N@-4k=xGZUmuqwk2@QYo?~;8A1XxN*jO2h5dLUx9RDm_6u)Fbb+J(qas z0tjbw*=iPO8rzx6?r(rZ9|fxC1a0)$LGc8xSCqbRJ$z=Qv%1Y%mDp_u%pH>nr|oY5 zI!Z(HDn7U!(znZ(=#Gz4W;yD5j@qxJ6>OyID@Rj3-grLNs6Tm~351Sn^jj+2zk-wU z0ti{MYGC`4>|tKLBYhO7d_%*?6%_d_Y35%dJWaNlm;yc%`OD9HQ^$V$CB7H1B@Ws& zwf%7@u)qG=xdNfKz`fz@0O}}Jgf1E?4}cv1Uv*F01O@RHP2DSLT8mQHd1yRN0U&d- zB}V@nj~{OCEUk0N?vBCRPIou3<5@`67r{ea?i1E9^F%&5y?N{OKQ6KY-i|QU)I#fg zFYS#tzb;DZaE?^w7k^2AQ=(0ab#8z~nOfch9@qq;z1VA<>)GAvBT5A$xsGL88xyf4 zs|w_LnZ=UPau65tV7rN49UGnn#s0d2*x!qgh8Torkq20?Dh>%6amiIeL99TunN6BR z&4@W{UXKm3dnog^377S1*v)@V^y7trAhXxa!xpp|EP<97z>J}z38WbUNQJp2;2)X; z5`ZVxTbBPo3tB4q(ZcUFCre09c7Is8DsT(8@Dz)mG6Wrs9TOY{vxi4T`{Kh*M<;su zxtT?M%%o~1TmRmo{Vcd664j*e4X<4K+I2v=N;c0edM90NoF&zn(Cb?Y5CuX3QJg8l zL6hqD6A8`{WSa#IR}Jf#j%in+4u~J)3M9N7%-Y_l^$Qp;oE)YvC&6>2fEnSmFvdFC z^Ssot@k2fip|ZA_E}$&}g(vD;fVdac4fTiMK`g16uQ#qyJRE%8f$*~9(VB1`8~_l z30D9|xsN$Cf|Sx=D@o*6hnfi!PDc}+{?G%lvYzfZ zDPE`o)p6y+#t&^rFW-g|b215x3fLVPL<;=e_ zHSw4U5mM$OaV4GvN{6y9V`g~VqjI0H$;Pd3^d2`Xb+#vms-U32uS>`(mHa$k=Z|T- zC2li|=rKDQOO;8kp&MiFR*%;Tq__{7sBAdB4*#PD44wVoyN zWs4+;eg)~tj#aG#)`_!?6Q3i{QX)MZo*Y>6=by3h;cMa|C)+YH?oo+sK=veDb53s<03v`rggwEzj@0d8SMd& z*#L`l7A(@SEm1op+6F+z?cnY}666E!-Uw)Q{*Xy641I@}7Y}{v6HBtbXGP_Y1eRheL}ke; zlKR}5&D*V6?n{ieRsK2#>BQ#ug)2WvKrlSKCK0^>CZY|4X_yCdr)}7(#olC=<6Y7x zphr!ARYOZS!Mvi^_HWN<9vxS0^f4ivXD?=hr_Nx(Dsm_AOlSL|28fKAJpA_nw;-mL zyA6f?Fk0PUz36ul3W^q8F5t|wYwI65JZ;De?EtAUf`yKCx9yOC>0Bhe2_F02q+`^9VoIa4-+?CE2F|Dc&!gA6#py3H7#OK@Fw+{^spPd8PXnN4j;Y&ZeWBcpK8QidQ zk5hykbrV-iV+Z7DnlQ_oG?KEjrzBAi(&?8a0PC5fd}kCpPu^m=_~-W*V>WYiEHCSP zlVz;0F&Ll(Bbq#i*EZvyKtodo?Gv*^y3P)Mslms{>6jg@DYMEz2AlDVqPY%Bd}o}9 zt|xE&DiPvATp!K{IY7RJcZ8+CR$h6!BGmsY2sncWB;H5$BzPHiFJ414LR_K5D`Kp- zE*~DEL647{{9Nt_W8Ab&(R<7`Zq+K0wR;u}h}6T$26hCHmJ#p@=&v^ssTn|^ak>x z1{Y6TV}jo(=wo2%<&}dCZ=#WWS{~0oG*d{1Pm*D9UcTALy6wGQ%_p_nJ{ap<hAl6FvTBwof-J_$c!d+7c3k!3;nGM`=cng zvqd{tBe6Oo2*3mc?|>e1l+F6PbfU(?+?uA(K5)F-id7b3D{>``-^xNzI<2l>@^=|4 zY6}bZi97|w2wy?2-U+W==alzhQ6+aKPjs&Q;1Y0k0~yB>?>vfIQ zR44iJuLG!AxusD$L^4J@T7bN2XH#?|z0IUI;K-vlpZG8dgP4>H?8Qsn9 zU^<5zj&B*+g;+Cm%Y2!2G8;^bFK3*Gd=yx(r_T<^Wg*icm1aeBzPxgQJOAtz-eLw! zt_eRPD)ExUUhQXc3`KmvP|kU$%lCryqHoDzBAfIj2j5cjB;}lHJcf`-K=fF5~cH=bh3-uR)t6$~vzgso!jtV*f zimDV&uvBKcdGa`0H#gI(c4~dcK96_auY|Noqq3$r{leWJsQhx-4mqo&!Bc|OMK1-6 z=8Fa3F`7jWfff31&HZ0dp_;rWh_G{p``Ea+VBccDC6`uQ+C1`U^+R!NtKyQfwsgmCF1+92NuucDBjUK?{)+T5$=l+Yd$nf6r(bKVw=NKKMDRzb z(94nXWE9KCs^aTtqeO?D6Ys=Hjtk|lDLCbO$7)M(5+%_1^xJqFK3SD`OP=QPpIeLKq-1~8O6J)A55zlYq@6Ly6fF*(6?Z2O>|1JsW z<)U2Bp3c_r|9u6$%Z!ikZ$+PtyRRUTuO#-(0sc+58>@Djl)W&lNA)dxS@35)OMj~5 z2M{waYk3AxugPO;ZPix!-?<9HsQ`;bcs6US7Per1iBde2{((I+m%xcBSt9oD_ecmZ zS=8fdg>+gKW@k8=XhxzaFSEQ$>ibmLzatGVwPosK$oYPGT0N$K#wgpjOOh`~*fOnl z!$K1!_~mVspKwzVXd$yb;*nh?$bpZqbuaqd*oh`#U1*;i6IcI!2AZPZuP}EjJ=MD^ zs<`>}$-t$?y~D%kX4b~>^!Rm-)^|~0Nw8w4BHnzEHiBNOV0*Tp~OC1{25Qch}kh8Yi;7d^BmkhXFaCSt1V+EV8b}X;o z$JHGT>OV-|3GQF+a^Q*&XqBL~eH6#}XGf%s+KT>jwUq3)QJgMAH@>n^U1;umMaql; z(huZK`a8@X?tH*fJw3)?suijBkQyuvQiI_RbW>0fl_;&KslXX^+!E*>m>3$iLQ%a& z6i9i2NjVp-joK-_8DqV5g(c3Vtnr3B%O>twsq#voCo;19Qw?5=5x1rr|2(hxo-NPp3Z&i)45uh*sq!Z6j;HhCs$ zAPi%vgoI(mq>z7lNE9fSy`>n>{sQicpZ83gaiwPCb^cNZfqC%pZ+yUqbwhigZGGU@ zkRTb0Y<9QtFn71Nv9$&4SZmO#baOK2Kaz^IU!(w6TQC@IA(>dFpERZpATF>v=l=vM zA{EzDBSt}*pI$OcfBMy(xku8@tgKfL9$ub5XQiJwz&Il^#-8D@`{2YWzkHoD+{6hA znYf`JTa3>33CU-g5vX&yZB4Rhm0#*S$ySUYxypCv-ORK15e#}GbeW|C~i3u7jO$>dsMatCwm>Dm|jP7iyK3l`Zn*eIqLG1~xM5>fDY?HqSiq!`{> zBb!!@-rqz?A@rqJ%48y9PyNM(4m|qJl9;fm^n{xI&unXSC zXZxm6y_5kNi@})qc+a)Hixpt~yLrH^e1KdY7z(g+f*w39;!^GjLAioAk8&y8#`OE0|YYUJ>GCf;pD6WK_UzO=`kq zy~7pc5TsS0uiaOD8^U&`s3v_iaSpwm=Im2uOzPy*9<H> zNaW)ddY6%EMM=V;)gZP}1^p2g#FiTJ8m#*Tmm)k1mLDQ-vMj;!BZgnG zzNyXkmqhisrGAshoBSY}XVwm8YD}6;~cU+5z&^ z^%LR&CtS+zL5|4(8V3IAw{vlG0Nw$?{|BZE^}~+^g)6ur)NHJ*5O5IN>VN>ANpR+4 zqqU*h>32swoT1y5oaruub6DzsD*yT!4WZ%A-4q7Di(GmA@q{C~9K$ zHsD5N7ZFj4`3D~Yly+3#m~B!&EhjG@FfqX&iBiQs`q4)04&ssqI5eE4gl~)O*xMO* zbH6C{pL=e3;gJ+SUVQ|KQnib=l2+hz87AB)O$AxJOvj~9dZ$_B?+Gv$z4#Hceg*B@ znqqocT9vzMd}d-*f8m*)3QmhA&-wkDr4_sOasP}s60W?MDV#fxc1mLk zu~^G&o?s7+o8`Z0mjL2M$%i}meMx_WE@-HJNMQS=`4g&}0=xw5zQF~%L&m{!;DY5~ zVm#&n1Q#_BA#MZrMq?S1e{^n%sgVHo%|t^T#YP6wwV-ySzLX2}_A8hI(LTQd4u5UC zQZoB~&}a(-QfLr*cqE1PWePb*$cs21EOxMXGvzk3LSJGyt<#HkXO1lrchi@%#wJV-@ki<2Uh1so@FF?PUl>MI8vr=8hvP@kVaSUe*Y0a${kEJzn?fN+Iy_D2@sm3c;i z>pj=6me3U%hTls$Pniw?Gi=G*!2Oa8H=)jt*7VG+9qbqU5VF%|EjtAf2+H4X(~N(# z@@WqlTUDra9yax(ucW^I#M;lJ@y%3KOs?icy)tEl^q8^vq+weR6bc39Xhi@7!Ha}e zq5aZnpk6uBHVuGKs8c3*kb_C*5@V(#y$f z6LGcL&)h7f-7;kxhiu=M`;PGJ*A*t+%P|%8DFVUabDU|D^4->IG&%TF^pFvg)2B4# zJpv2VyQNRtu|c;z5at1P5X9p?7K{P9{XS@58ag`6;Xv>M*2TdJ<_6p`;1q}U(r-x^ zs@Lr<#guR8C)@?TcpPI2R<7&k7D1$;HLKiMG3(*NK>*X7Xs3AXxrcgYtdvCW?tk~C zw>7&Lb8AVy%%xxO-kSz(QujwriKX#tZ`cwq_QqgzI4B&cbNk<{KV#}<`WjwdtTY09M^12Z5)`s;r6+BEA8O-mL< zy{b5Sf%eR!r3-?4wp=Ur?>^$_+&!Nq{yjj;h4u6DcmGD8SteUk^gXZ+qTqAv9bQ>W zM+Xk&c>uq+J4g}(&3w41xG=D#{^7&Ikp$oVd-(R(mmnyb>;bA3XipICKL33c;y`qU zV_$_Rgcm5YoPZ1L=-=31fnS*KzrO&khiMUlAB0ae#RWoI{1u0Hh9 zZukK#DNKH=i8)Zgz*T|f>fo=Ieed4Ha}mAze4K{gRG#Hj%Hj(b=YNV>7n0h2o=m3Q zC?{d%BbsI=(SMMT*GdvpPe%(z#oyW&NkN?KLoMxI^yljbq z0E7&;z-l?@aJ73iAyVSpLBpOz0U`qRma{}6j^F35h>B=QAs_O6YA|E%CZcoO%}^={ ztx=$PkdkwG^V69s=J!#wLe(8M&t|x3(HV*csrZgX=<`yRA@L7han6aj&pq0S>3T!5 z7ds#9S?hpg*qu7%WnDH@<>c{{A@qQ@N8CxtPK9*g1)O`iwBPII-NdU|{~upx9ad%b zy?qcxK|}>q8Wj)_4xJK8cZbp(`p{hh2HhZCDxfG`N{f^rQqm=*AZ>txh{|t0hncVA zyv+N@8DOp%xvuSgp1t;3_x-uUr)fTJ3>w>IvA8++YEx2f-wzYMo2}M4#QF4!*}-F% zsST=V^-fop?%tTBBN!>hcmIwbc`FooD=NZgUsEDj-z;3M<-|3l781fL_^tXwR>g$y3q0X1Tsky~JcS>!RpJG{W?K>J}5G;702#fjJKHm89z;XQD5`YBgx~f$<9?pxZ z`0K5^FZ~z#rXY880eJ(5mL~F!MV`*RYv;S;*rejsW^f9)-SL^w;lLg*^wLmuDzHkIPH0(YHTQ@LAc!2>E(X#(JjBgv95jG$3 z<7>pxAvk+ojtXkJ53kb^dm9|f2&=PB%rEl=$&VSU+18i!GKV?6Xsa$bCgUkpA4;31 z*!(tT)~Rm(b}`3lHlbT8M2TQffFN_XIJRDtQkS4|-j#Y8Q;f3^v*sU-?*89i# zg?k46{C(e%Xh7FM28F3Bho^<9vbLt224ahk`A3hoU*qlQc&+~=3}R4q=KcG zkXX5@MbU!$s93H;)xIDKf7V-95`P4O*g2k%r!G9e1!bVA0+q)V#f28B3r&5gkE)W} z7Y?h31_0Sor~Cv>S?6P^Je|=(vWDzIoAgIop^xW8vt_!}Oy4rjPx`0kDozb#w8efr zvaF@7?Py?c!#bi}H_6|&X*)i+ma9`)lzy7ZmiqN%Wi zU9zlYZ#aQ!l8yy(*n4@A9CqtXB!|6ftYwD0czkykUXoRM0noor>kfm;;J+=H+&nN! zQ-%ywYc%+Qv77XFQ&O-}l4klA4P>KbjA~HoZ$5aO531Wn45)4=asTpQwzMY(%~?I? zZlPB1!Z+ERt8AccCJ1bG_pNktCQpXPc5{rBCm)#UsMr|^+UiTZZ>5jWKT)rBc(O`% z1JQ@4V2RxWm26moqxGZtQ8!r@;UcDjaihrPypP|`suQr(EKgL5j-z)alB>58of$LO z!0FUw;0VC(6)dnjV8cp(fwPb5!iwE`U`3H+8g5PqeuLn*Au3BnCxO2$n4@c?Dxm>D z@c37?^JniGar(PjJ6KpjhLsbT2;AK;cCH+*E+EtT=Y~UAq5Ez)Q!qn0K%vQ@C9Vdt zOUS_b7iQ;GWESpQLr3tj44F_l9V8!<%tmCE5mOaNevT%FA9BSv5D$X(w;3OSqm20P z886Ed)LsV&kBNMb?j>6_33&G)ZU-=s#0)T_OcDK(G+X?0 zcN&DKkNH9;1n!nc0zD0@jM&$`Vs7gWbTep(xPl5FONg>VnYp_`(-M1v{?4-P_!jqm z*0od?VW~n6M>`hickJ-rKtCj*^-;;ujizn~t2C69>>y4>4r3~~StA`W8@O+1SsrwM z2(%Sq2v0iCIyxpTs}ylNN{(Mxj@`SXPJ^E#d*F`k*wySa`8tThhliAa<%O^CL-%J( zG%?lR>{=ozrHZ10pkIvQVu8oAmLR3MJ;^k(V1za13!zW8c1_2m;fbv?5wN zTX!CEeN064n%xN1aH~|(N^McvtJO56jtQ5g&UY@O6>79sUS;;Y01Q9nBi5tg&yLHo z4~V>EmENQdc>QvoTJ}EU&0s*0d<~8}R2;nfd?_UGIbp!18wVaT1j1GDzZ(h&oe1oA zXctpQz!m+~a{BW?HnHQx^#yUoEuvA9rXTB-gPiH07zgKc z&SvfqbDY!d9&`mA_s-5P^B*Q}&rq(&Kb-j9uJ&Jok{3?vLCaSx8QD zn&f^X`QYbm294E}!`Rql*7b(=uj2Uc%qLUx=d6A;$|Yr+89ZKh#w8*qgD0|3jIH%k z>F#b2s0LP!z1=*f_BPTC76wUy6cpI!@O5)D1<8QCq^1T+-u`cF8_!Rnrq!)963PT8)UHcWTbNIA}(Is&HJrt+66y@{B@oeOo=@ZtxWX&bnR}L)zpWn zUWBtD&CEqoB#ujz)Yi>Vlz!{w%z;Pm-bwnVjSyHF$i=%W)~|GU+$#!gbqHY0>X-X; zME7F6E^X&|l>(+Q=HhI*<=U5N%SH-}jIIZc+TZfru}>@Z>a!)M=?tx0t2KIG&TC=h z@erewJY-??Fjr{quqI7pQ>E&$ibeNlQ}2=~y5!R6UQ1+h^i`ECbgK$XJ#mz+Gb1rj z7r{vFENh5!ST*(NH!xUy`M^4PA0E7J9~KW zWwhI*b`a)!xotl8+VBc5b3C(udWk=aA<1D=C0w=94aS|~PvwE1bMb?Rw@K%nm$y&h z2@&3PzVAzjH?t~^G||t5+#+|lvpc;E&mKR%^Y+2TGr>p`-3~rzlYO?Bw34)vxCDo! zIG8$JP|)Lq1S}wyqW+?2!UjXBVW1NXcYKtjxU!ns9~B?=-zbCf7j4JxWd5(3j|F}w zKi@vT(;Mx8l#)E$9ICGNh;Ih@ZNNo!v4C$JZJ@gRL%7KB>w&zX@qyE;eXXg2#c<#- zNo#`=k68tajv4Ka2i+o_7)+u+LaUi7Y#2+0FDkH0*``R!F*YdrrkRH4tuH34go#Lr zagtIjG+_dK)BvI!W_u8L=V9za8>7}HW{i#Q)7rkB@9NNae$Q}VT;mr_C8l;3l*7Sn zEu~aX*7OS4h965s;CNVnH?#_>Jj-A+_R%w-hwV~{z;kNB015&Z9FzAhs0RIK4$if< z_cvtb)i+8)@w}lz}ufG5I;j+@NiIakL{s*>Zl4D@8 z{xcOu9*$&epeb|^r66`565OypWkjJTt*k2b$9-7?N~6D3GXJcIkUsyu_d!~-2Hf9a6+P7<=X2Hl<6#Dy(>my$t_ZL z>$voQJ@KWgXLqhI<1BOiB?UG2^HjO$Y%3l$inlnHn`dVsvY;M)HaLBHm+t$y*^-8{ z#rJLu1cT)nTXOHa91?cG$>A_VnId2JUK0=-3r@H4heh#Un3R(;Zi%`KJc$}i)d~|n z67`-CGIH{8FdmJszQa`ZIwAu&oqg|TDn;c9`wpD0yGeeJ?F5?!DDNE7CvG=CRQGKz zjDqnf|M?msdG?h%D;jRS66v*k659Dd$BD_H>fuQ^f>HJM93T1m(&|k=$vVw2-?KYW z?>$y8*sXDfc{J(_XKFKR^I(i78ivhs7wv_A&AUL*2bvuzxsk(}c0~ASq{rtoS08fr zvC4oYkihC{kRYA|?LnZPqziOK&lGR%(jgi@V+e%_QxCie%Ykq^eiuCd5MU`df_O$2 zy8PT6nx2r`;A(BGqNHZ5q^YG0ixaDe2>XJUw4W52b-(OgJP>pzNupdJAEkewox5K! zE6^b8(Go7}z-i!XUGCcbhRn*EGpYhIQ_O>{iSKMQ-O-&I7^c~-;IZ)`xq16zZ*4^r zZ%JL1W!2^S(!fl+(pH8Gg_;k{ZiBVuCBGUC4kIB!ijKFA{p`sr2c%TbN-Ks|{p5}= zxI0K48NpSEy3|-+T%~Pjm3?TCQkeNZ+dAcrhS}7aQ>SeS{km+84Jcoz$A{yQ#V6aY zM+~s}*Nba3@NGUAnDM#9S`xLHGPheXN?#G4DG9#b z;?bLPglsird>?ICh)o^Gr&i7B6Erft7XEi_dxz*|XVM;Bo8lbW%v?ETQ*+g9A&}14V}wnF+}nzWN;=w;-ZN1tc@SES%n9~0T@a`tJMzrd zbp56+2#bh~HV=P1a`)gb1npMvwz2qfy=-_lEDWN;+V{={904=XUAw{b`tNV|Rs|pw z0A?#Kde=n$E&;F|xU;G*2W~rK;qq|dx@P}?ZD7*3zdrS&-!orh`HOw+q;U*q-(SS_ zZ7p4%dO)9YO-VW^mSZcU>BZ{o$a>tVr(4oQ>%2k9M(2FEOi3?1ALQEQtyH z)@CE)9+S0f`eJzok~Uy5C$~MW)f?S6M%h$77BeWwMf3*qtF5L#-lJaKt7GE1PFYZn z^bjHQ`I4#+z3_l%gET&2@Wn0F^HY|#>RJDP}&2`+GD$dkP# z(=f@>KIdbaJVwY06UUIDh=f|9MVUnV5$2j3GJ~K+QmTJgFKjp9)(Iqvc3&B}p%FgH zG5XZ0agQrn3TX~8Jwm)uQk3I5!w^|Lzscd*HM^MNf=lup@;#)Ds@!YivxdHdUNao; z9>#sD=oZG!!T`A;1cLZsD5Q_;&DY)CmdhlxU-6xP;;*eo48sN@gO%CWI|`R{P@)6k z7*RT^Nung>Flu`{2s|N>H{_*nV&_lvnLnMqVzGhJ&W!EkmCabJxhQ9$aIa@4v#|u+ z&ZU)CNi&)EQcdamC{Nwo;C1lJO6C!yI&WXm$+SA_))m9?tCid1b04XuMH|r_W$O;b zIM>wV$pUL_dTn=RDb{5@B-TPziQxe%cN%gxMSH8;oU<~Ezub=!y}zV1dA}{6q&!i_ z+6;g4WpTWpcKSK>FJ&M6Fs6i7iYfSKHAnO}S3Xw;E%h7l-rF%QeMYcQb~QU&HKB9HcG5w=2(~C zice5hwA$*y_7DkrLu?)gh$Fz}K|IT(cJocL{j#c z?C6>?ab5c(rHd}aLTA+r&j-lpGph(qzIGTWwNgp6J~c61xRxd}rJ2u+Qj&>MWUq5b z&E>oIC8PaXPi)aj$KgoM4d-`SzR6L%r+}{MyLcm=-&TnsLJe@zCp3U9fHO4CU&yJ|Rm<oU-HqkJLl6>pI_1` z%~8FZqH}2HN;yT6K{zCJzY<_^)3@!@>b_r_b$xzM&j-_K5LIv0o5~F&O_yfT>YaOV zB81k0DN!S{kc!7?s=$9$KVUf2Cm}W4wR&qlPKYgu^EkH0GcxBFvk`DihG#pcsGq-| zAScuAOJI+VzT3?v#MFOx^RN{G?cQNk0f*Ie&Brifc$P%qv#;_I%WAcNL^Bt#s{>9H zIun0|Ho#2u&n6>0m-}l)%&gskD&)!GVT!FF6&--+0gC?8sP<#uAg0xl4lRb?Va8^C zoeb(n-TB5QQ1o9^07d_b&(~jHoaq50$R;-uEC*Gs zochU}*Qal5s;EZZ^YYA9sw}dzp5(LhI@z0`pl~!^eKx7HXyX0$%Lu)SnE7K%KK#{9 z6thnTib&g-$B);0-u4_P7jPNiY>Kw@)sJZ~)oFf_eepo}%ZX>-hi^>w|KJp7Tx)TF z{4XQz@~PaLHs$V0q>uS<0-(1;cPw7otMu26?&AbE43EzJBFMZKgn^BQoN4>Mxz*56 zM@u3qQjlP~If9_i^54eC$YUvQ?Kxtlw}t{KOXp0?k7M~$Ts}RNprBEL=@VpXLPN3O zNMDr0X&C@_b0gqx(F8xi=DWmV^(66+CJ}R&M7?jL zWSE?IXGn1;()t8Xe2SOyw&%5H(;rfkzz|BG7VeX$W;3bw=&6jo_JD^Extd45q}}(* zJ1&Z|o#kKW!?p$Waz$MSJ^Btk0h`6uc!$GXERVl7RH#^m_kv&6v9rC&+2{q(y7hdI);l;i#NQuP{k8at z3|~y$Ek5Ed9QuHtTE>NYnDG;|8<<1R+66VmL|yC-P98us|1x;5#y3sZqq z<)-7m8X@P$br2_jsmd)VbJOJVI%#vKIjoj15@ zz-pwWTqL!f!MzX`W^NsZsWNW+oOawY+Cjl8CAQhazGzva$i@PdZC6?ap(%tDmW-!7 zBs{{h2Yi1f4{D6kr!ttNh#8mH?M{%ig%Dr3wh(izHSdup>1*^Sb9}a3&YP*CN2;Aa zDcZuARD5Um28>B<4Z#OEefoAdJiW+!Y~SVZFooO`R}NcaP%~rqBXWf14Wlimer#PfZeVYg6{tpNz9BG>Yc~@c9=?Yb zvP>hbr@2k99`*vke=2+l&O1wZs<_mUELbREPpBzyG9vi&L+6WL6P;(ct0+r?|{1C+YEly5qg>Te3UnvK36~WUa(pe`);3Rha)gxg2`GUF*N`9h?(2j z1MEmauUdu6luwc{(h_>d&QtN{ili;4&|jS7Edj`9>UmNaAb;qnGqQ&KcOePeK?}|0 zT5UtZ{5%w87|qxGd^bbgp*t7t&&5Y@tV|{6Uf|NtQOcCKR;HcT0Y{Hs4r6+}w9{Vm`teQNV85_nIf8r+|`| zR^&nHXo_=aNNXzYw5EQ%L=!PU)8 zr@()ozWqF=u9uW5!1Grl*-BXMW1Ngf{!^zWt3yJPwSQF`T;1OAz)hd`SPQdQT{K(H zf0ZJ`9Y$slT2CgFteUMQ!OVWtNPcX1D}xo{xCt=Pn30|qr_|$uFZ`9Pu8oZ5o-(LB zrFzQUR$+qw&4<7ubzA#og`~D+slFYz?S%%j)njPFn@X7w)6lj;9n5-)$naxKZfMR! z0is-iCellqkxjTNv`9k&8dK|bmNRmU2qvJ5IHQ+zEzi*g9{2a4F=@_ z+!9j`ISVhe5;lk#6vxnGf>t!rWBMh)WYTm;)J3BzY7kS+L zCB^kv7(FPL)Bc)mgAz6bjI4CgU`~SGt0RYGrX=6cY&sV@y8PwNa&F~5@&P$Xdul0=1r``9QM?EvH^U+O?kn$KQ2)WdAunJ`<->ipLcHf$ zkHYHBs9E%X9;p^mL*fUH-;5)7Ap9{6#rpLKM1au58;KMPh_ZCw9=*7{21*jdYpxuM z;_7NxjY{X<30UH#c!xqbW_U`w@C237bzWs)bLq)$J~*QN^H9Uhm={1NIn0KwNk3kt zlbsEqdSgV+#ogP7LloU^q5s7HaQa01k2X*K=E?W7zvRu|&SaaR%`aRsO=k{Ln7=m9 zd`~?!pZ{xZqc(l#I%VJ43m3RQlML8DR~iudh>6Zobv&4Tu}(WDFTc`#Bq_>h#HPS4 zz+HZHDZj5gxmKC%dG|uY6ZW7?3k5v6ly@vUI$pZXZdu}4G?PuI`?h{4G)40Z<~E%c zBf3(7M*zu%?8E^U)z^OQ<^vJdhUo1Vg696)_Ww~U(86NvE8A0WwBQor6yi{F5;sRX z?d1X)BdZ55>VGKMyCZ3;!k$U4o>1IOToDX=$XmIW+T7E85=m|DpO1fk{qn~4CwI$p z+kG5L6=BSl$2fh`h9&J=P8!{QM|i5f?a`DJ_gqW)4dQ2Jr-Zjx!FxB^KlW%b(`j;p-L%P%4kVTyrVp)|9eKyyD%i#pPHT z&OW>lI^qxBxRKE_mMuTC0DbQ%43QQjo?r!^U3q=6n z^LR61dp*NZP9KkzhSg>w9&X`FD8X(r^qJaT-i5rqB5OJKgebR#fuf=PgE8i>>{U7n z1@3vulPQ~gNB5Fby%8&5{_$#tGwU>2rCAN*n~wOb+GY{KZUG8aKID|GQ}=Cb<#&`m zkGOp8gLXBAeZygo5D|c`3y#vg^?~*r9QM$j z!?n8&;?0saesU}r9)X{i^s{2Ax)tdN?0eUD535Va%F{(^KZc`p>&MiHq=AX}c{Ka+Dx@)Vj$_ROgHY(Z8arO#4&&GQsm!R z{{hp$8i@HcYlq|)o5B6%is0C#Tmq+~XCeOaun|r?V|Br&>IHzq77+s+HtTluTUa1u z1S@=LpIMcKgA3Z8!xZCUjnaA z$RSKH;Nj>3&vvL1x+8=oV}sOQSmn!iA(B+N6o9?)^wHNbTz#c6uQ9K1xntNm(Ie;* z@0$8$1%Na^h>F`nO%m_$PJX(*!XNyAKL1sIW!cljBFf-eTXVy)oTO0Y5;tT1?`rR| zT*6*(tgIQAX968C9+&LBOLs$s=tt+;&X6*yrMYkvk#F0g%$cyhy7N-Ut4Wx@y=~g1 z!u?48f#m08^KnxDiG!glhsE+!b79>`VcmWg@Biq9o1$GY!1snzPDNZ+MiZr`i`0+U zWA3j}$}a(7{!&S~Qgl#8^n}c^QhqB;y4;|jx!G7v+w zX3R$3vX=wlrs9v9(z$G$a{nXK^#^R#x2F3Bq^7u|6ND}wU_T90eWyg>$W|xyU%S%~ zClBiPZK~5(LtntMT!s%X&)-|>&wCBD?A)v&aTWUE2qOVHF}4^EEq74WVbI_%`R7#y z>51<9{)bd23pYsLuy(Y!Kq8)yAB>S7kZ1RI7UTq_NNc}P`El#?@`HkL=lWQ}AyG!x zr}dRJp7|IXU1NK1Z%Nfhs-uujiWqflxifn2zTww06WkytArW6{rl4~gzjbPLIiAVV zyy?Q(t_Bi?#B7$KS!T)s{#-Gw@{i+s^+(ZxFICRpN6qWjp*nlcT#G;IeQbbKZ!FJe z&=1mNG8QQr-=>V2((c}V8)Zl+E=`^M@z{&rpK$lJvqrwYi} z{H{oa^pSR53lem_CYt3wS<(iUT{**wx+D)xrxhGl47m=OPoqAGEIrO2W~_$`j@9ZLxEzQB-=^SwFW*@$5MU%6 zQ(I~KhH@u#@njH-Qa#yNRHvZP#;y0o}3X(C#iko31B{L+-1JLI@bAcXy@3O)Css;#I~XD!<<_?9GG6{Vc50zEeU^5aBsG zn4vLTLO>kXQdW{Sb#!zCAsH4N6Z_lEVXII{);?bGRDbifj6a``q78k~dW;+Cx|`jAORQZh&PTe-;Xzmm%9OPp&90{fz{!{vKi{c?Rg z#9G0EIPA2DqGy!U6mt~2u*K7B<#W2IE2>lr`zK{p3x~R-1Xd-BNb<;}U+9Q1c5g8CwDXu8lIx0_6s(kn(pd3ie(fR4)ibg; z2T$IfP5jU&x&?`&N?_Ti4m(fl+(FoQ=u1J~)@K|J*HZSTcP&3@@?Nsc-IU_eiBr!( z9eHH1x>#zq8s^M+ozZ&IS)B3tLoIc^`;n9*2DJm1diBMY8FxadCZ8k`zN)s%MN{YY z9U0iNB#W{e8&nwM^J{;=Le_wqFj8loHBmoC%4k<@B zWwaAm_uU-fi2fJPw_7Pgu;^J33M`y=s-qt={1wm~-Au7p|3Zf!g2&)SbrL$+CuJg$ z-dWE^wYvqKWr{|rUGl-+TgKLY*)N-<)1=P7dn?(L1PHq(Ji_0S*)}Td_2s&f|bW+}d7uKZV5N1fGP1 z8+;SO)sY6;w~InhCwgDirxLyIDVKulYDfv zFX~-u(w$4?jxbKTYV!6L?6u4jk+QYZqSM~bcLXa^n#rTPI8~pR<8k7z+g_azY?=MM z^a+Q*s}k;5LDZoX+GVsIrz8*rvVEZM>G2~L=K2Em1(n^Rd$vbC*BT@y;5|JOKM_0@ zos+2Yaen5zk=xqvS|Phr0JS@Zx@*lD1BeMq0?Gg@+9OaJCsoKe-|=TMpRxccr!JjJ zagX!;t4cj}WJhGLw(}lf$1=NjtjTGbqW-pnnk1`AH**||RBxe7{Tgz9LF&ky+bNIsZmraL-lzx%Tvjn)@ zKg5U<(4C9Dq=NfGhMI*WLdH?*c;xm(qEnfzGC#HJcbr1n^Hexvm(#yJR($P=nB(4ihEfSV@qdwdYV~b4eXdj!@0||TgJ*uUuEak!%9k0k(%!PT zo)oouPv~lfl_~>3>{b(!=3i5NaNcxtWAHE_)>}97)R^lx=*+X1<~bI%cwxY#t-0R`FI7FuP)!p@T=%KD}|}y>&CRnEnz|HI2wqH`Wh^Wv&6W5>8kTB}&Ut>b+3a5R;^Lb?g^D&zP>AQpb|?AL42tZS=1 zz9POxqtq?rDoQ-6PZ*BDo2(#j)i-eMdqZ`~T~3l!&-tlm<+n2>30t2$KFWW#MoEDk zjNL4?R#VnBed)FN79|gS=3xGth2(A5fgHVIft0NyufOB#y1!!2n7Rc%`XLb~*aXSKH0erfY=Y$cv&*=iw}yk7an2YSJmPlu=nE^y z=i!TWYUZ_ZZG3-oO%YwlVi#5;Mmd>cB^8l!pP2!Hj5V-r)%Tyay7%jGzJs^PhxB(Z z4IfTqjqoM$Y1a<=Fc(yoke9KLFgi{=&vIU`Rz$Z6LR~b+A19{nYF%tcQad(eO%8f zT*$FAh885dnxnXc01fGe{zHnphj7QbyE#12X5dG~ zHj6-o^3NOo1^5m7lHp~<#Z_cEEYUzbM@~?<$s$S(2P7=g6zf`26SisPJi&NXooXT{ zCd%G@g26EV3%W4T?v_0Trv4n;^&kW0t)z7pv2S}RhTeBoV_ zv}it{8q(6qBiQQBW*B-@L;mE5c{ZJ$`61!$xiC!M>D5s4itss&1cL;P`gJ|lhZa}) zl&34#<6OJFFEC1sYyNWI5i$N8uAe3`j8=G@rXTn5^^nBo)1^U^)jooA`YjfdbTyi} z9cS24>{pFkQ1H`Dkzca>qp2@ zP0DdQnY$^Urg3k=wRCGH7LG*hZD8NwiriI{=2Z(N4oz!mFVtWDh`o53-zdC}$VtJd zXCdj{XrG;J&)_c1Ph@@jp_ zr5k~d3~MG+twzVYuh+a@Bu?-(Iizpk7PeGr;%&;!re+f6-jRN$(0$Tc>y)u#L$fSk z7gAH0ySYAYZos`gEZ5pktcB>#Og`he2Zh?t+MX?HPe%-iy97lsFxFKvzS8=hQtY?g zK^=WT-Tx*u!^4f^M}#5;_{lDs5VIZHniD1twr2LGZpQY|L;2&?*p(7{ zySLAe7eAm#;xGA~QO-yq!uDQrxXU}egU!fd4Ccy6P&YDH#s}k)fWAj3ksUZkM(Ha* z4R`BAS3H)0<06}AN#ZLTdj%!mTC9iGa}E3awiAihmF0VrS(^7w7SJ1R+xT7Ip-#OXD%`bXxer2YxC&9O{s{RdE#2CTCfkMzI{2&$rX+Bj; z*~aYZt0zS3J$>G10zNIiVvKII#erm9+Kl5lq}2txyY(PU%sd9!Jv?8+2mYhVKrZRD zH2W09GIr3$^X8CMkU$m)5#gPXS}Ix8YiI4yv0F)zJSw_iWbSQGknvJ(CE>*SQPD3E z1+P!LTtux}jQR=lfPfq_uALS2ymTbjA(R#7^<%Xg(JiPQ53dA% zP=4Fnq^m=MpYVC47v|q52Us{?;TD2SEHEz&JTG=`urERejunYmFC<<_H5_6@l7B+JMcf(#M>QwZ(*8-= zSC!)$TQa;lx|0IJM#n{Iom{h4YJYZqE*GR|rpD3rtUL3ZMoc4x*2m;^&gJxr>`Cn8 zUDmgm+;a79;~9t|md&vDGwWhfySpWDSuY}5{dKHxFcfh&$-rmd368m=8<5fAly(8s z{2x0{%F%4k=ep;k`{$XANZI!-o8j5EaPYB)q#p!b@8O8Sx`+S4^N9<*0d(WeYZO=q z#OOnU8VTaQc%3Yic`7?y7u1I!ngp4aDzuse5eRfg#YAv_FlpDheisjd^NiuwKdF@#b%cf0Xr_$@S)7w-Va| ztsD!{^BUJ5=v6%kF3z_R-b{Vl3t)-E@14$LDbSE?5U~Xm=nLdU8^Vb!kw0$5QA_s< zs>35*?SqW%H9sH}=x@7KNPrD*DA*oW!e?O9AuNS`m&C!*8uD16k7*95K4TbYU^S2b zxY@@x`CwK%?6}TUW_bdvG2p_Vjp|Parj2? zkiM??^Vfndg2y80qyTbM%b=tk8b@NKmfeb%k2mb7ZKXyznc!ZuQGcGYwX0PM69c}P z91MxqU$#YI26ts6Ba#8xT^M`JnR z`f2Mt-m-2YfiC^fZ1F|Y=TT!)fFiH!QVY=&p~o;ceDntW{^0TX-Q6`vHh73=VKSpb zV*h_dGcb)DIIt!yX{bfq{~8j)(}-NKq_H3~*)YGNAk(UM)_@iv-9`b6OX6AB+n8dD5mu^;j4DozSY2}jsqo>Qlc6>FnuQ5Wz zqWtt9oL^@qD&)M7_}uQX#!96qcqy0rW*{_-jN_`Plc^}eTUJ$zRnT*u;SL^sJj%ty z&IiUt&cX%jk(g5StR+_XSbdcWnvw^YBR~Ozyt5z>5q@~PznLK+MRQx!>f_e+ zME&xt*4VreyzU%?*p>Wag0>PMlM z_AerExZ$Dtt)=?!S`Fjo^sh?oEc~DQ!nG{jeXL=C4~y$&;RPcYZ#xU@tpNh>kP8YN z32Io1pLR)eW%Oa9r+5pCV%c7ta?$opg59z;5~_nOiC^eJtTt^uwe@akqxn5qh#*(X zfxbJ(8c(nkIzCO`ni+9zTq0*WE25D*5Sp->AeuXM$KsT5D|Hx!dYMaOcE3~EoM|hG@_nMoYgFmA^?sbNp%$HzEa(pfy-UvuL4wS}>PI%RTB>2kbbSIS2A zlLvo%n)!Ur33o-h;T(i%Q;?()${&=4-3EL2vpZ;Fqf3(s5ZU(8UDoeo_0YREz z3tFUJlR*q^@8WKlcj|fSmv=NoCYGU`*#4rkfiVf8vyoYDG$F<$5IzJRcW{f~G&MEK z!6m0r)xW2I<2D{D>2&bVUtfb#*ml4aj0pK2_OzZ%CDD6TKHlNLv=YD;mES*?XmeEi z@?OaD94dDboxY%NKRy^?*DkFIjMHYgS=EdsQ7#3N_2UXilRlj5jvjv*EPpo zjTdBKJSV)*nQV*olx%~iB=b$``r={#_A8`^+66T}v>d6v#0;;m_Ej`PaNXwa4Q%u) zjH!C=hH1mvAgA2EcpS(NL0dr;pd=sLb?&%)iWbEY;dl$;0T*Vh2t+P7*WPo6SbWFG^laP=-*EMiw8px;XIWv#`uE z3w{sfJhkWy$rX4=x2gk}g45S_Z*0moKkI=dnO_Y?gdircBhG zz=v|QL`h*>+`LgL5GH92UsX-PTMkWa5GtcoL2%^a2{v7n1lk;R)zQTqmaiyQ-1Y$1 ziJ&Fpj;EIK-8LkyWHE1z4#rr#G55r))xD!D-TM6X?u0(QeUsTy(PPKocc*`R_T2Z4 zTxi1v9@hg|>UebF#;9sYHUU$muH%Z&Q!A=_vB62jeyyyN&4XXPheHOdo7BnKipo4c zzm3mfEp?;IRyLRE5%Xq{S*Iww$sTfq{o*UWQ#S31EH!yjS@XU<1y_5HEsQ0ca8mHD z%8YIgp*2zRd+8Z!B$w4%u5ECI@uAkdxPS$VIA7(H`%iT|nie`&n^(TQ?BvwVpqB8* z39Xs5nr?N}6cJOGwDY;)Ax(%fI$In0?&PJbfumgHqO4=dF+V+33teB{72N$b8WaDs zl1}EsJ;j3$tzn)2(*y?=Cg^9oLR%7T!vXO>7Iv;&LfASMkx0VHS^iLy>6S`*Mu4b` z{v!gL^o!1ivp^x~>ZF8KNKz>puUG8u_Z`$^=#Z@*Tk34jEymwCFZ5)*rdJ7f!T4hxy+R77}Da{ z2-n{%%Q)Hp^??s%h*8dmt_?3c{gKL7f|A8cZcHKEYG*(mx$?ZfEHH>%5tnQe=I+u) zTfEaRO(=Z@)iZP6c`D7Z9BK<&PPx#Cx(Rp94?`hgH9KJx~2_|xTC9Q)aR9<_GZs_X%*+)i6oXT zQ(7o#pCMOP6enY3 zUdgLuE{(Sl#nY-vEK2n8!RcUj(n|T-fwP(bq1K-(&E3u-%FWq0W4K_lh!^j1$y=ge zC6>+P%WDXO>m5D}xh(6u)1RPzyN*|P0}rzp0ZEm(tgtk{6(s*y8gK!j?A_tw3L6pO zo`Wt21=m+gl(@2z6pEXZ8>~Q?YTYUj)e&9NUy#6pDt#hA8soF!(AFnvRXS z_K@rBL1cv{k75sS4UiSey67@bpE`p|{u(csqL3n!q<1yH$T5i1bb&zvAO&B67W^&L zmNJ>GrO-X_nx3ULsACthlrwQ^1G%dZ=`&hIC;}h43~a1 z&UgPZ275Qy7j`29l3rKj9~_qMHekZ|qX4eQhz?6tlLQi9AEg7S?L4btDhAh>}uBtJQ39=LwzSdzL4`%JO zfAkcjN`ym};ZyIQa@7EjDrCA0=^_TC+Ku@V-q`e8>Sjy`YBLdQEF;_Hz(1>NrYXw?|b|kr-%OQ z`aOX+r;)6%0~t|RMozK;XMiv=I*cSkh^%b z?BL~ZIh>~ENHr!BGjV+i#pA1%!ZK>j;znPu_C(zL?Ocu(+UGyajj$l1W_QKsa>A}x1TZCDgX#loY7X~oPt3YVmx z+NfO!=U8b-CK>=&1p}fr7;|i!IGtOUU^sElj`mFZ0vY%COe%gl2O39DlngG{W}i8) z1q<)&_QK`b8i8_kQHJ=zq984_efzG~?&kK6$Xm$F8Uv}Yu1;vPznnP#yD(AU5%41; zDLj5Tl~SBX$n|<;kxy|@>($;)*?Ml&zyT(p5YmCJ`X-_Uc9$bRB~z9$t@@!hs&>qf9u>w zeLdQ?H$xNDkJiOIyptue#-4I@U$MPO>@VcyBGy&h7_jG^^WZOCIWG7(A{15yQU2^3 z;)Vf8C<;4>LOHoOnsXpO11A*Hmtn_1fD#Dwr1}c1^(RPBbwy7ucIoc9f}VB(FlY|;--|L}JO&pv%6H01BT!@g$`+T76=z-u7pcGN)crIP(_#t5ot2#d+QpjGBp zF^vOaITpKisG1Fk<-#U=*tHwqszXowoKtNM2Z;M=lXVeCKcPVSvq5J(PWsRh%VnQH zvflLtg3g*JvoLBui=Qm!R@FAsx3|^w$mL1|gQWr(EMMw_(YcbX%D0)X``qryTF0+5 zys3Un%Bn@@W#={@$Q3vYo?6pqJUnC(J`K2XBRGe+-s3W_g29qi#L>BCpbLWO}(L0k}vUZ&pq9Yd074a~Fx;ekbTS8C(c^1QGx5`xMz&q3tX|ip*h; zadCmZra8*h42~vO3|2w+`?}XLN*t#TG-SinTy<|7e9sT+h#w zzm>nn#Z~g+K;X-P&GDKijrxt=tr;d^LXVUhP{f`Woli!MnuW0yIt0q*+T4GrQ`BFi z)vGtoWWUqEqi}p&L;>@#PzgO(G&td$zEB`GtCy=~--`RxGtmE&;A8RQZ7+Y#BnUBI z@^NKgiSqbAzRo)y>-PQsq$s4U$gFHKv$Bee?2M3+E?oBIvS*6ynZ0R$_SOc zLPf|dLZbYRH}~hh`+UCc-ya@4?tkvb-Bt1U5)>AAHJe6o#K* zR;c#NPd3MFQZF}D(s|A0yOmKB82D0;CSvQwwmjwuox!?rDFpZ=-+%g+3o_l7 zTPR9powRZMm};GYEgAJd;v*^BlPc!9kL!`6zeqD(mVTo*rm_a=MWWG*i-xoX$f*pUSvW!8+fm_^SnAhA1CGXbN z1nB~%AIEwZdh}=;86^qi%n20k#%#;7UP&U}LspO%8#VnnN$yv%Y`Z#l0>n9by5iW{;(TX@V~P={wS}3#fE~H9x}~#;Z+9| z{1sI#1;PJ*wjlM>femLe}L2}^oj%_*P*o-?uL^W^MYzp+2+_C9`al!v>@pl zTezwHiNeVYbzLhYW0+F&Q4DPyEyrd_`Ln!M&*!9Bp7LeEM_#MEm~mVfa(MBXiTh&L z=lguwEnP*{i$9?^eJI1rx5fsaCED>E}h?GD6Qb1PdoFuD2p^pFMq^FY{g z&^N|SI6wR-L=uMRs((>4NFqbHF3*>M&2{p{rZ@DiUy(IaL^d6L3u(oCM>fW;jeN#Y zZ>+YUYnT!XOy)>Xs^7mt!a4n~WOoggBxKntWf{i_=Zd%(l-Kmt-9b$0$@^!f@Y|u= zEkau?FG=awYh5Qdo}_2hObGRUX^d36I7aMQ_%kx?Rd&aor1k}W5FT=9c0o33=nITP zeRQLxx2Q&ZLi8km%u{M=w%i0!+yY$0yiq8n(qdn_hh%&YNXFIUx|5-qaAqPi^aC^K z&0))$+E}AZAVU=ce0NhU2IFF91udZec!J>&b+odF(OcFZO1v|dD4M3}N()u^qD~!E z>e99p)SVmge1cydZ$6T%Y;fo8 zdKl0xT|CdX)bmPW^n;Gc>DsEf;M2*)Bu8T(kSR^e%$7)Sq0LN{Zb(?=KemaBq`rz6kijbLOyk4H1|azvJU75J%o>XbSRep0AoV7W@4`>4#6tioh@WPYNA( z$BtN3s+)zR2g2I@E8W1!)(T^9Y0s+#zyIfp6^(ro;(dIWX{^{(SM_F1i`^NNcX9@ym>*bKhyqJ7Up` z5;Z#Ev2)u!^D;dKFMe^2E9r6PXkO;`008+RUN8M+W^DnfG)%gPLLui3tkxN!F;C?S z&!`{z0vMwwaD?pr!0+7$aQWsEE&e=Kem)=x76VZU99WG&lMrp|0wKG+Ch*|dTaLdU ze@&vT5OCcNDzk=!qw*V3&ybk(hfgDbWK>(xYg;dfoq2gb@og$XdpM;dI>Q2t`ci|j z5+aLR3WE8%m#!#NLna7F)hl?w{CiqcbIHvyH8!R~Kqsd}SY8MO)m#YZw2)(kWI+() z(aKUC_4X*uPo<8j9_CM~0kQtx zNC5H`KtgH@$qh{GQRXQgkANJY*YRK>E*K)Iy{cJo=QYePyV zrBSJF1xA)FC>tXm&GA!@*W1IGgn&7gfI3@!_LJ@>m$1J0;1Gc0p6?Urtqqyu#}^>! zWD(;7bQ(l4KPDkLR#36~a`C>uLAr1HOI%LtptgUhL68gZz?ZTc8faLKc2LU#WyTDy zQ3SmSt(Ctw(ChHy6jKOf>Y2QL#cSz)882*w~4IwUgdBujhoQC?rr|z z(^)jtE8h#eppk&o2cJvC?K&O%*!0&ZI_Xha%jw#p46Stx&jzN;81lLCwhP?mAL>6J zeROT7cw$GX3%t-dhtHsRhUP0f#}X21rQ!lP(u`uK+E?Phk($s?PiZi^DQPK##0&&IhTBGey+|W~Sbg<<{ z*+YLWJ?an4H*v*O*~D1~yx~m6Pt5WjD0PUH)BT`nNQ}hB7VTth{cd|Pc{{Cz3|;`J zR}C-ZiQaiJRMc1|9AjW3+_|i>c_Bh!L)D*3M2kU}u+UWPIaRXgkGUIqU}Pn$oo3d^ zrQ58|%U~WYDJeN0-dn3Vc}~COupH}zNzGD)!;87(LDBOa3jGm-{g1Ac>yEx7F~X>@ z5k_+_ERH+pBOlAG}PF^qk zggItPO!7he$#bxy54$rTZD)Aa3qx>U&l<5kh{)_(J+!%~_z7s{n8UwzDGTD@6&u&? zh^qULA7Q-YDVoRB7w8+c)Jexm7xMNTIjx)tDpckBmzb+@+SY8H>AA+AC*nru&SB@h z^EcmW(r}Qo3arxf6&kdC6MkSHaz{<@@uK0Fm{FC!klb}tVY;iH4o|*89*xCz!HRb3 zwc9uCJUeV@SUZ!5_ePJYbC}~(Mx4Y7>Z#pRi$Cc7)`d?_-CDay>_a4tdr8#2ih%lj zvr$xA{BG3O$wQYR)%ZPcJUmHp&-~j;L2Ykm37iZUUNfwV31rYA8x*E#@SfUdltEnu zD+LY*cY_8$X=adl>EbciB6{q!J?7uDr_T}5VfU)yTHS-K+gmlEC+pTz-!<{CC@Nb? z)=CIDi7LBRRgXSBuiDiwWctouEhkgHJ74UCxUUZ78e%Xj6pBxZ?jbX!~b0<>Fi{ zTQ<6JBaqMUUYK>;KxF5vl2u3fH8fY2Z9n&H%32q*Kw0*VcwS%GcT{WpWXFBq8wc$2hlnMi2B+^0QlqjS|#mcG=9HeZ% zcW?>wc?vKkoQ=G7>ICJ!W&5f<|LYrR8pDQx#({$bsRQOLGKdPkiG-saDHQLBAXHe05F(c1-)na>XsB=^9 z#|QaJLN3-BZhfDgZ)I+;I(Un!e3-_S878e*H=eh*jL5Y$>2Q)u3b5!aW;u<_Xfi6B zmY3x6SdUwJxW_{e=^>IpUf2Ae1oD}qnNe{CCsef_=W=or8Mp6-A{6NRy}au@tMXZAHu3IQTMnSY~!FlGFKhi<^YBGou#^#pfNlg2pLOl3mZ*ge4 z_eM+ENhA#rmpO5#Wk9ivS7yGVC*qj_hrjiMi$iHt?Bu=ColCd#79>}wONl<8&{})9 zE{}TpgdW}8?ce$gcU$L;R$oNu3BP$-1NNF0FJn?JR4L+%tDnflZVxf#pCICkc=G_0 z-mHR6$}8<4=cypE!VTX8Cy$xyop1D8+f_zl5_JHaED`Iw1S_x&KeRLltb(bn6>xy} zW?qd*u(s@g@kjl(T2UWNedZ=1`!F=?Z>8skt`cp{n3Tsm=JYdc@b$h(Pu}kP^;$Dh zOX5Pe`Qvg+ACBG^BZFDwRV>SGG_D~mWPNmU%@f31^D9N-4E7RVKL%{l#Gkq{#u4Ip z8?deMoMx6>1GE+Bk@b;0wAwo96&sE4-fdH=^Z1l;OLA|kqk1ng5iB3cy(O15XF9%S zUC61Hp~8JDseU45Wwp@YCIf!b;*9$@j3U5SHcH<{cTJDQvGLg)CIOF;)M-}rukU{B zl3rroap5Q$V>9mcK}2v!$m)Tg%{414TQE3dfrE_ZRh8D(a#6)j%(Z8m=<}Tw4aMPtY8;?SmGkW21XXI^XM(kvR1xIDJN1}xrwah@31&F>>Qkc$ z!f78~ZF7`qC5f+maJRfUWNLW`@-*!nhryKdQ$Jw4Ll4%|vkf>Ik{>_3K1tB<3V}^B+yUUw2el8o5un)R zMED{EZ^MvT;89qj1D}CF+s8UP+d%N2hokGhUia@R9)h(SkjqVL%!d+YfDc1qvP0F5 zo`P|g(b#9pneN?{wa>1ovp;(tB$<=<#0WObS?NzIlSle|I~hzulHl|{n9_EwgS;=E zGfB=dnanVlUbj|hxw`-H+uD+)cv}#F2<@-OZSlOdoh0?DCmhq4x&*tgZR#9DRnc3I zxms4`g_jWKFV%S3YYG;0Qluv;+_Xy!%4F&~67>bvL&x1kIEebp2d_wJ){zpyLWrRmPU#1T8CHiFz7X`2Qt|?aLZVuA#TP6l4wxSXyKJb z+d+EvHKaU3F8#~IK#T6GIQ;g+$t0H(@8mY)&*Env!S7Dmi-0~?XT}BvQZmn_j<*jU zT!NM~G9H!F;}SfB&g@=INoYx{Tr&e{*2taIY#WefWvY9%+*}XfHyVf&LK&53S=Ahh zk15ef&)LM>NY2h<*SM`;h@80QdnfL!^2YPknt(6z1Xo0Am_7pw%GXz(Fw)`S`ygC< zm%XMmBMzi=9o>0}K(by*8`k3{JA@%V7iL(Y1ND6vxP0KP43(rV8hI>pt0LGQdt1;b zI9u&0wXW^e@RYjW5|~YES&!A@i8Cj(E4F^S`bOc~lcS*2B6ko)jI`RNp1m(tX2*vV z&(lT-5Z8eFV^w*4QbDN&SPR6F{Ld|O z&>RbzsTKqJS_bQSLYxok6X9;U7(&0ovHMf9l_+C8r6~q2JM_A~hdafO_QmPAZ}R1u zLqB{bM7$@rM(OeU(5GG&QoT$Q_B#e$LABw&YVZWrV87)ijQzZFr)Z-#W_kmVfXwqAPZ%fTUbx56}HThXls!vl;E5A_^@W!#(w_Jbu zIPIwPg=(Qe3c^bX1~+r`Tg*t6aL>#G=V&u+P;dfs?}@kQm{=R`BiHsnC3X{W z;Obs4!5v5WBv>9PVd!32XNSZa+tlu7`2w=`x>Qw1dYndS`>Ep%(`FOfx4&|IEvn)c z$g}&#sbxWx&c4=b_BzJiL~wr9XIxsOP>gc{zxSF9hupQ z_x+Ao3ip{0{;Y4jlYE2;j=-ZYOjzaoSHA6CNw~GHQ}r*A-H(Lz z8455xR<^u*^LGsJfWfcwUtR)r$X2w)T6oAh!kFGp9by&{R>GfE_rLlpF;-A+yP$YA zEUcUaAioujw*DUx)>Jghu6B*w*=>OewBoA@xFOl$h7|Jb-oGLJwr2Lw=1nr+2Mbj$ zD3q}rza?2+#~W1C@M-9zo4@Wt>=~*E5A}WT(D(}qTUmi*O2c+M-Jo14!kp?})$2Ns z^JRZ^RnJIV35nW>!#?%Sz7{XLMN@09x;l+N5vR4m+fN=v*M@2{npsDviW*WXF9NY%GkU<$^BvG5n`|EC$KrX zJhS_9<-Wf{xi8SW6%|53pZo6`T~n+rsKf!Y3D`D>C%K@opP>53izJ-2`f9Zu!sCo+`-ll0}|XUc?+sbON3&OVx+`Nh1CL7P3 z02Kp1JsvUe=^12Y(tn)pr10E%*K=R$(0pU!SrYBS%NKuroNK=O&`-frWcu6mrI_EF zxN*=`*p!Kxgyllswg>i1Elo^$O-y0-uLY}Si$-#J_s=;f&7qZ0lyBNFOoAR`D%-X` zgC`Fe#Z%hXA*1+IoyC_2NXD2|{pJJ$f;Z=AJ4uB^bDV1S2o}UtxO{16Tc~mo&VE|u zf11%D<3_Vuv8+k<0<9}fQ>g1>E%8TgnwQ*Oe;;l4=3<;K8rqbhKFg_n)RJp`2s?Qe z?OU#rGgIVujpWx+{fLQWqo7Pic`pA1Ji_NMsN)83ytC#ozviElFF45F6T?ZmXq;kv zABNnzJW=Bm>ca`sUnm9Cl9;3$gg-Z+A`Dj+5kpqx&ksy>iW-^y$0(Xa!|b2qA+!LGrcpu*5Pjp0 z)x^%O)JnY?XEtA;->Nm)jZy+?Kb0}%@mEKF>M9=H;R#9LewD}RLVPiJwMJ9&7**Fp ze(`)a%LnhaT7PEejMYgyZw50I^WC^wVQ3eG$>80(Y?EQ)kvS}8eYMuR@7%m^crmwu zX_ZZ0X(Z!MRD*&A5ewr&R$&(x3iU_;f{D|g8wbsDck435)o{>Nzj25XQIc>Vlwlg; zkGO1u1G&WSZ;>z&8LIBy9fHVBPDrD}6Nn1@0PBG$XAboLJDWi6kEIo_nv0etoI87) z1Q{*wV2;nQeXvKi(`xT=Dstq}2#pLb^yzWv^LHH3(}dRbO{Lp>x21gx4RWe#7?jIJJ*)2qdGhx1< z>|Ct|wblMHd|+Ql>x!bB0x0rKG{H(@g$C8}|7?X`+Cs9*LKYQI zLKrDeqZ|TRO85+rC3vjNEU;%&|Fl1;L)_(5g5&)G_2u<=Sx{na+ctrF({i#XzCK5& zOy1cer*v{;M9p5Cv2ayKduk=?4(QdDmvBc+43^g;9S~5_D8H_}bGpq*>O)!}$1W+2 z>T`YWbf<}xl-H?)ZYkHI1tJ{PtTAG<^D5ukZZCHdISrE|wg3k8SB0%(WNk#0;k9S; zw6iuS7raXja97 zM%ziw9olmMPm+cQuoD&EXH3`qz21q;tckb)g|~Df9R++>z(rY+7W#> z=~s%xVaB{0u?BjLYCOD2^F+ed0$U+qbZ9^E*t1oC;8{w|bX^?x1QTaVm*NvE<*kjX z?zKo=`rc`)2jPCUsk!ob*1kquuWnttDkYftg?3`SWWoLUcGu(kMGNTC)8_-7g&!u1 z8HY~F@*1QSYC<65vf1nD``@!OlU1CbB}l%q3OgUF=wuQrC+6|;%zUVE3>tEg&xbbkd4qwQVFaubR8E~t@Plx z*(h!)TJ{eO%f95iT{k}+qWpBUmHyg>=EgG`n__8ah^tJ6bR4ap93D_ zz0Oa~dOtLe3|9bc8uawkDgEdonM1P zGTJpwj)wbjM=SO$kON-s8ol4#hi>t?8*vDUl z1w+VZ2bv{Zp;c>*LAgSP60Z{)W5taEx+eD?yd9JpFv{6;WbSE55v7KPS^(HGr*wXD zWo6y+DVX||`lyT#U%!`xdTq7h!@OY>i}k9M&>b~_**B7FzrQ-5&wldHuo5Q6C=&6X z?cTnYQWT2E^disRMT8;eo?at~%2gG&BecG@YTB9uh8u<* z$^#2ZR$JPG@tQ`z*s~G@E^sI;a;(&NU%U_##NF+yRx_U#cLl3zs{cH&Lijaqm9$V@ z>H4rzUq?3M1v%`P2d~K6X6s{)xVL@+8fZOZv+6yEjNk=^3YEqlE?M!h;J7T9qvFzr zD&2PYbXT1;mc_wheUs+WlOIbWhiknL9SY4Lgw=3*@#jmdm(5!M2mZ@W3ubGA5rjtG|<*!U)B zT5^M=gPa%07@dOoo4zxGfpknY4C1BvDQ`}$yP#FMNDW#7EgQU;!(1ZV%vC=Qc`S7< ztQJ?Qc;FoKuBVtJe5;&RAjs~Td1+K2z*n7O$uiTzv4Xe*t$nRutwXr6m|2E=Cn7# zeqvF%W8NdWE+GM&xh3;vE=8D0-Qz;qZrLno@>h(V<-C%~O|rJVc=B_Qyei~j)a-6* zdXIib-XqQH&IMn5!iD(*xz>%;r1zD zS)w47$)*?QO<&o1)-eJ;lxtO*#Gh$hz z1uR!QzUvmg9bGWUrT!=R{f`UK-UIx(Ao={y@4q6ME(nMSA0Qh&@S&rO^6*4KA3+Cr zg=T-jG4?c#0Hg*0+;E5%nK5q8^HA;-(ck1&PqBsz8B$-l8|Hb*9-+m~A3%q;iDjOFBFK^mgEvJL1D?B$%gMi=O?K`u(FD>W`%{Hsith7)&aM zSohG~>YF44SCWX%Xd#f$uQkv}z{}h~387QY*O>}Vv|OH3xhO{|`EcCbA-TWxLMt>&T^sLKkJqpV?A3lP+v#+z)$Om-1p^T<5eUKRy`7`1bs;h~L=Ad6o1RC6#{bYE_jZ(_SA&q)FJ2O!5iBwBx&X`}XD# zEqfJYKM2=`xrCZ*~z8Q$b0wh zksOaC_lmbY2goxWuOrJ1st$**8t{C+x>mPY4}!D{sdcpPLpaPs5+35{B4{P2P0n#|b84j#gkAzc>S^jGYuFx0fN4?%q&XN-P zmj(V9E1Y5=Q>!{G-bR-#lt7=7J4+dAXgMC->-C2Bz!8nJwTYvZJFmK|>fd4_h{E}w7h(8Pt7_eq`BTNH`>Y|cRZ6i? zf+FgnGihXhv<_fKV?tQJmRa=JA|vz2oW!Pb{7@gRyYZ1-T;AGQdVa|W!L-Ns&yg!I zN)mBh=s8(?m`mF+V2sl4CZ&FUo`8o&T&gMI^Yc;+A0_4Q9}N=1{J330^nCb~;~8`3SNO_$d>sK3-|X0a5K+l4t=R)5a6{i|Ykfh&c!ezfsh&yFymsFYofA5Q9Gi8ERXP*pZ)#3~`DEE^x4%ZF zr_FJRYwfjzYzf*2jTn;XZw?7gx)1lOqngDO)Um=J{W+rOizBTL*B;(KW%$lDc7NY> zL%Nj$ki+q;G8l-8xKLFd{5{!OU~SEWpj8P;aK?XeDDMA5sc(y$rlxrl?cJbPa=xLM zI7Ch3Fa(>v%hdUT1Kb?NuithzzpZ+p_j+$5(0d~~c}{gKhrai8LX`PX^W3r7%qud3 zMat)+lj4s*Co;a+ku~g4O#bQ2Mn$YnyKABI(`l~RgtgZcGNVd$^pzg zg;Gr>2=>yNU)rKO=7-fH@v%9K#CB$qG?Gg=@7^5-<%Ad>SRmec1uq?30Q?Ug{Eu7C z4eJ7edmA7u@sVb*NWAu?`vFOjHP73Z66b9yUs!3SJSrxT1MkvKkURzC%^$UtJ8)6VP###UosAm|Y-Fsk(iZ<~pTs36Pc-rhTz<}fEBq5RQR`m% z;Azi(vOVI=zOZ<&t?+~ILbv>578@K!OT+Ea7Cb$PHqjMzhrr5{#z-(uAYJg63X@YE zlm#Cn-hZ`Y>aUH`7q0p2GPGEA+hfMi>1LA3RU4klW---DiK&=2h-5~mS;xut5!t+B zFTeiUb*wSCV!XUA-g}6Ns^psUO*?P2@i77gJ^V9$i8kw8--@%ps2z?y-P|*9k;J)a zT0Zg6BR3Zn2K!@3u!%x{qxikrrCr`_rjHsHw}!AMlY+m&svs|x16Rc!3nMBgUVAg{ z{mt^%s=&Zt51)1mUk0QHQ_S9pf6sR5*ZIp-x+`T|U+|hhr26`4p)lJX z92K>hI^MiXjWbt1936`q3p3hzy+8NU8cL?QD(XwA6GT`0q95xnST3wb`D`pZ?c9(<2(S-8!d`>7$t{;k7^eU!41)#r!JZo9&jOK0O&{rm*Zz);0YdP9>CZ2uh)ixu zJ*ek7pGeW$Bp4=ZGk|keC=pTq;AGuBGF&5}f=>nNLU^S0dthpEi1Ov}V#& zi>N7DZTUn%(VZ~#vmDUg`A~FMyS^mD8YC~v77Zu8DJUH26F)^|r0CzcUVXSh2*H~& zXas#Yx?Hn+3AE=5>B#u)tVeJytOz2;I51)aAVL;)RxbO&K+Y&98*X!FZgZzS^5eb; z6Z9BJ_A(y+?a0OF$EZ!?FO5uR<_eNq1^@z}27Io_?0n^2YQnP!7%9>0q0fx5le)rBMO#8{oPw+VBICJ$b?w@%E+ZdNl&4B% zaVUmg!K8wea>2LAC&IE{Eh*jh(sUyyDn4iZ<}s^@fFc9%y!#KxITLwVfm_yqe$qm`~4t;fwWJ?QFPt|Q8#Owt@UT{p1<4_-N_PQ@N$ zrur1O+w%&lYc+vBNAL+1>fyyA~&6Ue0%oED`kc2ZO0^@7aIg zm`xV#YGtQuX@%Tgyjs_;DysdDcdR>nZMJ`b=ayg8<$LrgtZKF#sz~h&suV5zYltcZ zz4c^oo(;nS))h|?Lhkcxbb^;xhIzKjl1Yj#Jf}6zGZki%kenF5^deL?k4N-W%~e@t z+jGgN1nf1;^u~>unz)+0Uc&gie(uNe5dHIsrDe?Sbf6Q16^UyoRyJbmrg_}wT=q>> zwJc3NN=#)|MFWkhL87V_lU(Sf87E^y>gGniSGdgIzs2oXzu>-8b?xk=US*1)3jNDi zkwN#i{#55x$#;vFn@$i6bzH$x^9=63T)yvblP{fCk6UjaZ1gX;{=bUMuVI~>F)mJU z@F1qfy)2+B>e5%PYC;$nx11gHP~dd_vjCv{x9y1NU66^eqV`qkE7IKZI>>hX2Tr)u zb05Pp8X#l!sPi4>^jHi_k*R5fmQp*VXV63$!nAVUx>fJ7vi)LPtk~+9w?0Qw8Y@LHO19A%-?dkt6jBKj5+mkN7n1=H|X-j_eZU?jkIaG z%j##}+h;tydeugy(vCh*S?UAbuejfQ5g6`tt@vlkS3;*qzFVh3?V_pPo(gk_ey#) zm>4Kcwcp{$goXEhO`xCEs6K}RH(^Qk| z=`yxG*Jm$^)#?$8#fGNIkX&Uh48|>cCC}GrJvbmkr^mK^v!viO0gvlWG=GO4O5hNb zVro!|eLnJO{uhH-Lj9vykrB}knWM|zE2qOkIXW@T`u9QzTU zb{Jwze6PvcvZG>NYVHuesP+TD!!z~Jy;4<`$R-;0r^a-@+Ss0zUR}xFk=zb?^3ofZ zWk@~lYCL$B_6Q#bo|e}3wrDqSJ2V@<*jBCLpPn2 zM(T;R-!~pJ^jtg}ggR+D8!i#7LCZ}j$GEVpd+3xynLi5-FvN9Ua^Hm4peo<|2)05k zEc1fA_}JVrHUnOVLxz%q(0`E}M{#<^{@OK%=k(ZV80~0#hFZX)AeGpGZ7JYeS=l+e zV0fh+oncV1_mKXFVA1wu8e6tN4*&~;gu_1wLexm%VO03;xsJ5KnNH`6IE)q(vydm7 z*J$waC^y~ODIuTr#SNa7q0`yyznZp*vA*4lnII~M-e{s6E51prXh5#$hi^Ww6d$$f zcd}wrOEf-nWrrUNRe90oEWX_5{FVsQlmF#@R79S(uV{Gr;?cI;^9t98Fah2o+VY%z zt}e=|kAIYmZ}-+ym^a7aN4*;|*x>qBAM{1qUuI=zWZHp#%kQ28t2?XBs*v7aMy_iF zHq;E+yDuSO$|4?tXDZ_c&cKq;!IB)9CS~aY{4{4zq!>fnVqj`xW$K7E*)xK`j`?Hq z&@pTkBaE22gky&_H4QeNP-@p5s{D8dF>`(S%l`4RCOB$m@xBaE8(k@Wl4NbSpm5%Y z3g?ZT;)l5UgQJBL*|(m(A(dlMlkz%MBdPuAcyg#5*O(Fa_5A6u_X`Dmxi%Y_qwDiG zpuz4dz=TCUr;_*INljfJ{Rn>a?#X#IdVQ|$cWM=}S1y$?MaZ_&dD`?_LI{o5i_h`7 za(OL!gb&|A0be`oqTD&osxr7mE_6}lcB^b;j73_`F)RG={Uj->TdaY#l>13i>)__P z*czP-YeWTW^iQ#1ovg@!E1`RH4qn(=RU?3QOZL70hL)11UF7SNN{pCJ}!KV z+4ej&n-WlvMZVB0@0amFL7wS`$m)Ps)9J>^CKq}VD1FuhS85oh4zW@3&DsY#7@6wx z{W!7e0C2~jiy<6mxrEOq-xGO3O29*6;!d7!?Zr})qTJ)Un(Xw-!&l(!Qgzp;q`$#U zhfb&SUc;6eqm9JDv8Vc!J8SPm)y(QBKTmB{DXbfe=m~hd=DNUTkx)e4fc<}EGjK}hp$zwW&PZ^d!x_24>y+aQ}n#V=Itg~ysUQW zXBk>JhUXxU@k8R^$>5Lj{|WLw2_Ze_?!Ek%vO^b^NJaSoP2*K?Rau&$(e{wpCJbpt z&S0@Z_7s9U+nbIeX+6kw`KPOhk4NCYFUNoFf;j<&)Y=Ng&ns``2vh34Qsj@J*l_H# z9B&*R9Fs8C5b|2M4);gYAT5HS82ZNV9*JK_avsk)*-JlfP=8EHG{cd9YlrFNNjq~< zoKcd)eU<;)+H>}n5eT)vbrx7o{Bi7VC8MZc=Qib+B3eqM2L$IeAUFz!VudgiLwwHJ zEY%HTS(@(DVz!>W!wb?wHSxi}E^B|O5i85RUZcgSwNq^C?QxTYtuqetxDd@A;&N{F zGnw5PN#G#fQR@+-mF%-bLRVh-AqYRdpdNwuDtp?ZO-{e6J9xcs0!Lw2B8G?q6U&q> z9bIf~u&z417HCLnbawOrS^$t&{yx?AMiCvuZ`Az?8&0`|(OJ+LJaI8o*Srp7U=2~G zJ>tsF6ORj%dr}9Y9`)bt6i%|rnmDm+l_c@U2;5y9(l0et1RZO}<;dtXlabXa>Sj|{ z3Aq=M!Z#J8tU-8_@EdEEH7$$BtFc7$At|T5Kn+!!*QKuMvq-30b<_}FglbSPvCd%Y zccbxsdEtu5f!@(Obhh{DXI&Qs02uGeJJfYHvA5dCeBP1!j?u)B=;YR%W!hwsQe&^8 z0t{2X9D9C}?Ohn;IM7La^`$$uOu_JS3z#bddy-Gf`#1jja1WGUIl<)K8HUexyckyu zW}g%A$H0&ucN(FhwNfinM@e2osA%Nhi;%WN!O}k01GLDLHs9ay%7le@D8sZcW20Ws z#>e?bd)e@jCQ_M`0ixh({$70&uee(~AYS5;gyM^by(NQs@wrR=)!tyCK{DK)-WFem zCni9P^w(&6L@cj$6r7Hjw;5xn>%Uw+-*a_2rNA{LYV15>;C+2!J4WXHf&_$~iy zN!ymR-{a&RRA*V_juH}U!D`VdB!*mt;D?|husF?-u={d8$p0mcUzPVEWBfpCSdO!> zV-8gNyP@oz`1$z3&uotbXE|aq+yIvW_o&m}CyH#GKl0U2cVN>qw*`vP2Dc1kXxQQa z)hH`zNC~LM?$sc$>)r{zifBY`>M6wByu)-$nD~Tp#{J(+A73)!fE)6qUo4PM{(atj z-4053TOjaYP`ql0FG9-_^}kuzX0~lsG%Hm&w6`L%JD4j{F71OR4E>e&#KOI{kW%5b zHnX<8aDL{4*QH}zbZcbXmsdvegI**nxThcb_|?6I^kc_n3?rJU!hV3eR*FN|uA22b9h~$4jWK zJDaKG7ZYkUFsLVD?)33Kx>H$~m7mO13@1R+xMY$OYeQWyf6FrMk_N+U zuV>%k?Ilmzui-sp3w(ZPR~?2DgP+duOlqx#xx@M(GS7c%@BfZO1y%s85wDZ08S+d) zYVUuD0k>@JeWBr}U`kzdov@uD_%S$+DLn_ofE1CuZ2rV!_wBalR(&wbkDj+pN84V= zKi|a?fk_vmZ`7QHOl|xW1c+ zQ0oPP7fv{k1S239vMU`{2+ zjbNv%z5)3}4l>zY*Gn|TS=&dR@=rNDw6ZKV^%jF9~6faD*tDMaFSj=EMV zEk~tv>O^?I7IApS>x-TZEC^zjL4!b@*lldr7#PQy2;*2OF2wZUDq*j~b>MkyZEgoD zD&T|hLLG?+&HmgqO~XF0Yep2vsr|TsGx4AE7$}v0pT`jYNGes;*xeKy?Gb{1%U+`3 z>5DZB_tFb?)c1ng6(Y}(r-nra8@AJLA>dwfrro>e_zg!kY;-r)PCwg3 zIc||zeS3T|G3pReCXvIrisYQ481Xpq5e6zKOlEO$&GmHo^6IZ`C7(ar-ALos;z3xKuQq)u zg(B6N*k|Uc46;2wxP=amQ75Hw&(8TI;E;i42qP@o`L%z0R~qq)@&=FL`M>hFfo~Fd z(;V15Gy%o}IM{({ta9}Vx4g8LoaWwsLuz**Kfhq<7`C&eTj4xx(nts&rhwx4EJZVM zx$Dp-ASC%y0V86pAnY|%8*)RqrRp%zyIRpam!2;1XI3c+GIxDupKk9sZ2$b)Bv9ej zC}-}{svjPAwIkq-M?BHD)`-u&gg2jQUQb<3aO(6hk!h(4bRNr}a(_e4gG%b7H5bcq zQGP5uA89s_keyq$=>IOTPo5@=hNY-g_EodvvNDm@uttj@PO@Z1iE+f{z9V7Q)O%-{#g+Y~c7__4u#eNHY*KqHR&O zw!FyT-Pp-VK}O~Jwf|nB?k;B4QO}U5;i6l(3)V5iAuJ-G#U_Qbi=>I;Pc8QH!jtR| zo+6z2_|X@weI`1J1U>@~5& z>=pN$*ouGKmbIvxy9C!{l?QQ5&%8|eN%7YH`}6dromE2GY@Z7n;acXDV=^QyWtN=6 zXGUa{d0s_^r4KX8WoGrK4ds4GRg-eMDFbio{OYb6mlQjIHEru{@S$t#C6f|pzohuh z_xrp#ISCM5sHIup`x8HQIwa;83SX%BJ_rEz#nn zdqiI5Agn6IMTAu%SKc@Fg;Ut)R=S;7FHT!fP}XuLU6y>~8@px5|Z2DUpBoe&eP}I=#^G_UoOyL09#e&DbyaZoHeLf zX>!exTULDZCTF!KnfP_R@507AN}mq7CKMe;+RyW~gPk$i&#ja28A&QS9G)<-LRhu` z8)5#7>=Bn60FNz#FRE-mFtf68N#(~9VZ)mTFFa%Z)B8NQ&F5n7Vu`}?swrpz1u1j@=ftaeTCcQu>Nda05rkd8|&3MPp>#l4Zk0{F(d)cchUn`%gWrN0#yFQ_Na~ z)Y-g4^SYlT+qTqNS{k>=2RZb_Quour=ph|!9hpKC^rK|%)z0s}{RvZOH0-2+lpw!< zDL4>Az=2d!X;>Qs2Ik1Eg?7Zc@`7VR1;h}5_=S^hf9kl8elaCxc2D$nTdJqA#Y+HB z+rxZVv7j%<_9$1Se{)H@zyuz^AxxI9;UFbSQ(5rt&dk$}`Y99J_6L}+;`ilqrRolQ7N191CU?9f<`d~C-CPtUlMk9?=xpq6$L-hDVw1Lg-y=6QeJ!Zo#cc#%Y!F^`m_>dxrvSq3bKBBkD6Kx zaw^vlg6-pSf9>6)iy9|#Q}beUHob&gMc>41ynge`HhmGFhHm7M=1$GT&s@FZ3T@g) zt|oPsZ&lo#Saq!QS*89oQqI!7BQtU&ktx;xq*d*8^ZTM>H*XVlmAp?W>{1B)6NBD z3oK-atG44sJf(1t@?Zc1{O6N_Yc7%^hD3gGyPzz26%@6!xsh8~6K#+D z6Z`=f3u#k~?Z3%%wgT{@{)+$H3~Tlh$j#s^Vw|$MyR!u|wquO%=gd+5=$ntD9i455 z3g&+CM?_nEx{B)5EnEHe<#W`fGgBA1ZD@*4vscxwffw*-9&yZSrbmzqL7b zC|9yNFtj1@219IUC%PR8JxMlK9vCkz(r3^nc_ zcY`2xU_W4ZdX51RQxlHQ_{LEaklp3pA1suGn ziy_gQad+Fm-|UX~o6)jQQ8;;ALT5g{(>HiNn{Cq|?3{}ue(n2{HK9^y?reRoI(@z+ z2@iQmd>hZ1oVq#!=bsC^ie5EM{EK~h9YfkTIcN_Tf3`T&PMAP9)k4N{US zDIlE^(jZ;Z5(3g8(xUS1=P>U(zQ2$8!?|#-(LdJgXYaM{bw>v3k37gmY(SeHDgx6( znJ;zfKSCK#OZ-xTz8Sx9q#FAwHKm_ezXVi4yx0GeE$7~W2|#=3OjLKW>3YriiWfgVBhC`N zfRD)cVqSAWC3XeYLeuYQa}E(fEclN2%*76QSE)G773B<&X7wr~f$vjh9u&x#AD|2x zFPquGU11=~P#kSQ#Q_4%uS&$_R}*xH5nOUe<()nl^dZoO zbClw04q6vQiByGa$z*Q6j75{R4g(`Yzc+>3UDK8CvH6UVv4B|rG4uM(WivxWU@@yw zKR_wKz!}eZJz$(c`mHK13!yeu$$t8bZSW>$!hv@9v?;vTLF!r^#iD9uabT8pHFxo3|M$yH~NkH$uC>GoYKDfm(I zxvoDnJ=Z#$x8ShC!nW*D=ak1WX_r5Hi89lpu9UvPs;2Ix_WPHedW|C~PqJdPoP>(Y zM1?y-*Pm-%#seM=$$We8LXXqaf6~kUVC~-YFFC()z<{iI&owT#wy{KkX5A9@7)ZUu zRTR)nivN4R^MgMtBzVsKu4oRV&%nVKxO=-lwlfn07HHt+0zv^7cNjR22s;y?u(ySX z0a?Hy5wO6;p>g#3GvXgRge8`3)F@jJ;Wr*VO&B8Sj9u6s_x)0o1)hnHYQtKdN5cvK zb(#$`s%SIGFX@>srTjT>)xER&8L){62Z}l}hLvUnG6uG85ADKv6550Tm5#|uxLwpcWMqb_1Z0|kM>7LWh8?Aus8Q`517 zkw-X9Jq+4@ov|YFB)oD0zvovTGQ5p>?4h>0eal<==M*#32hR(}8YvK(29Ye!o!l*L zfSav3WRKxwYHDN(@gktP^sha3EIr7@+_e(`gF`M;f7`Cj?=Cyv%48imN+(mq1~}CYBB9}VLjL)sn|cGjO)kJDxw`h#Fp7l zw(?1@AyhoYz=4I_fIeL++bTPADEGZuVuSR=qI*V&{7B8cJ=4#sl1vWVR~+5OfnS5A zeoM7D4O}@67Zrui)1G@Tx8jz|Onm#RQf}1j&D3bI%qK?nhbAjbimhZ9-&nDYE!Iw~ zFXhy_uG9{uGGFd)v|mBo-6mr-53Taa_oW0t(DuVd;ueF~4Y)_NU#p&BIR3_(bxGg= zm4ntE=PIayyPl@Yyp68mri=YVL?tWF>$E}#Ha@rvJan$o+zK-+55dD~*-qLZl)|b-A zk$~{Y1J{*m67qesS~z=*aJb}npOpZ0TjSG--M6ZHAE$pSUe|B=R!gR%5S#0V?nmqwne$0(YU~NWW7N48R`wCKnq63I*-C z&sFar5jH?QZ3TRLEJ2|6|APg|t1&uyMr;kv@mER=X&+=>0xtNtbLD^?vE?(e0=WP` zg{A=>Y*Izh%l&{J;qd(TfqpXTIep!DidUlDTPC!}?LZzeT?wnQFTs}bmIHtK=gl`n zNLIn%JTD-t^Y27k+=36Tn=xdf=v$aoVC8K7Oj0{jbDvAgo{6w&j zG_*}Ms22LQamHP^yZ24ki$VyA_*nMF%oErc6ZX7f^VN8ob8 z*yiuyfEycERc~8}H@Qmlmc5kwIB!kAU8zqtLDtqSX_Grv1!6C`VlE+Mjw_7hLtfJpz*7ep7Em~k>op5uBnqU3fcRB< zLQ+mLK>$R^t0`@h6;yFteW0f}h}nk;GNK&{L;w&0-b#PqWw}nUu}7fI9SHG_{`0Cv z0tW}kI@%JB0&zm_4z?&XsNanQAgg$^<^j2m+!|~&y0#=bi=>w}XqnO&1XX8(Zn zg`*Y97tG`Ep%s;p|p zPQ31SGo6VjonQKxHuGi0OXN%ia)e}Cl!=y@B$0m*zel7$vhHQ%aJ_CtC z`%QOYj1$YZfi&G|J`Gyy;V|)7rjJJ*hBdF6(-Vgl;kdw>ONKxZn7ny=+G z$zGHjz!B?;^H9_{@(b$)jDqjk8$2~PfcXyEw`1?D~QZJleFECwnB zw^fh_&KHmt12hE|_Mi@O|2c-q-FlhT;ZV#+l`e8M`F2L zs)>FyN1TMQ1&rTV?MgrY%x;VHv54~t^phk9k7i;pNI8#I;7N^}Dpm_o&H0!KZ~D@R z9r%VvR=?rvQ+jX0m_W%fqHfFd-sEoe9K{UM&(qFhWJ7&wOw`ru{x-y9Q@E3vg6{@` zecPzVxC2dEGUGSrrg10 zRiR++cHwK_p6xj`n6I$N=mG8Vvw-d| z)y^j|7w%cEy`p@_gW+pLXc&WQc{OeF^mN+SPpja$zBfjsOpqZc3M3^w_fC2OcXfAm z0PJ3n9c~0f7eL7RkFfV&Uc-0*)^~xLNE|MZ_NI66hyw7$`v*PmZqT2_99p}s%c4% zX1$-3t@xkha0Pcaec-g#Pd2mxBAM1zG76IG*J_$7fAez(3Qc(-Wl&~WTeXU6G0Y$R zP8z;&1YYM@y1>-4ulXy4a;#eDlW}?3)H`|aMYk|+HJ7mKaUzO7P0zu;F%62er2dJH zwCo3wmR=VH!^1;BX&|ECxdHncWYQ9-cNFdmh!W13*PH8|cF%dI2Lv)(f?F+0EvzRIsdR z_h6y7V~*J|*T>=)sRk~s<;vJxV4jJ({njr#cK(Jnt6ssRtSIAKFRTHK2@*W!wmQK~ z_nG|=y%eF%>zI`)UN#=eV#6*tCA_IBPu|@icYOC+Ud!!vX{bo0FpRX$7{RG~KcCNVjCbNl@lSRIMlUQ?&pj0J?k7633Z7;ELi2NZ@zi`7FxiR> zBYZ%Os6dU*#X|vHY;Pw8b&^&Hu=)WOhUQ}68UmbUs7Srw_PNs0mcJ;gCAg_Os_!;O z_$68^g+X;nJy_5$yZ!=E%k`IdQLbiEdVhXD{N;X#Ql!r8> zO-8gd+tUVQ-}G>?cKWkqd~%xJCf|yVqOhJUs8pDCLb^@Pzpbns*3(0B(bIId!Ek$9 zT15pgn4KSxHXseh_zii|#|57)a$?`#CYf;D6TJxhy&i?)LZ(f77s<5e@>(|wfj>%> z(7RneQ<|v9ph!>~L``*0r!6n9C=U~tl~F_oN7O@CvOk*ax384KpH}OWEH=!a00kITKJ<;BsB=UCqpD@k z4`Ni6MqBJ=0piDN$W><-AS&j-P(bfo)EK z`&|hs`7!MEXj|T^x8L}g;>{Bf*2KHtWA?wWbL9;t=kTWS=U8+8vP zJ1dL6`MlG0wDw!0n4-o}eZv0`Iuc%K2aq65}jOVfa=Z5+*hOt8*#clr8@;Pv)aXTM7HB zpDLLXyH;&mKqY@4M~CTW(Ba3@62Fma@!*W@YyJ+Y+eElWHJMpHb5tOtt2cEjlS^}O zQjorGyaz3~Ecf9~SVSkMMn_W3WiyR-vvE&CUx32=r-BZUJ%Ix)D2S<@E331e1FHkV z!5la@p?6tL;QhZ6P@&a{@=1L({Bn8Ri}UJja(We$SJ`<*#g-FvY$Lq69s6@Fu;a4d z$GfZRcRXEU^!%@JIZNXhI|*r!=i;Qd6whRM6;UKo6_mY{V_pu9|Xnd9nI z&wX`m6E)f67vZ^sOa`=-9lo9*lmy~8W_^J|NxJX*bDs!FmU}d6@^n0o8AVR<1bXoE zh$(zn4C+u^6npUzGCBu_}u;O|t#A zfJH(H*`P#|Hk#eqqNFy0GWMn)17F5a+^}_UZ2nXH1P~(tvbmT;OM81;8xYdyawdV2 zg3_(+K%Wh2_4khEHu&4lH~R%;1z3w4_zPr^Cg>L3TpoPDcIZYuE=tMH?>0&MFz7<* z-dyM+NOGC*+Eo`P#=h8}$Q|=RG;(pwxGp* zyPNjC3u=aC!05tILPa3uLmQ1#8LH?{B@UiI(M5$+aU!ap7OBi=ObO znHo&Uu_z(;6T%4kI1^%&b|}o|CVw?>B2^)=*^e~Hf_IQsTP?tf**3!I=K^H)tLwW* zxUe(xknuN6gucptydy#uoJp{aKEFu^BoZ4Fd*HBb2Qzgs0l?H5_w8RUVDtE+;vIpS zA{SdVC_{PKciRa7`)-IBQ=HZeFX#pEN^aOsyH0Z&uB zyy>LLCsg=`@UjN8sco;xkegd%0dmT9WX$Nx$i zg$qIqKu&-E223-2P^4ATxPoV{4Y;;`;$~P{w#|DE`N0aqn)S))mALfq+$VRN%OJ4OO0QdH7o1pVmOh ziZ0jmo3c#7BQx>|#fRStbXelqi5^)LR?XwCph9=4NZP%fEcF;34hTvLPOJdl$4~wX zIJg%+dVOD^fOsF2k))>&zBc@7v@>%09+3?y#03t{xy(1{inp|aLGBH#W@w*dBRgPi zh+gU$**gI1S%k+w_#ALlgN7UNk^G~oM)I(@Ip;+hR%2T^W4fm@Ba&da8KoNmH-Eq8 zZ%TxP>&(Z8USG6>`uO`n;-#J{Mtk&hp!Jpe#x673ucC@m(rc|}Pv(ZnFHDHoE@~bI zEWWVBuB}-N%B?Y;D0DVzp)hn(;r-Q2G2$v)9{28>PKYJ&Oc4O6i_(PPHu&fw^-5{S zieo5i$_6}0n3UA`tOmOtSL*cTWJ|u^=XX)dKIEcW5cl1P+`pQIKod1LQm0waN=ez``g7r5CCkw-Gk_Rlak{@@`^Q(ga~(}0HM zbD=vm$lt*faA#mLk5$o&E(NezxB~f%W_?B^SIWAw=rmp!&ye38mQJmd{qh)u2F_um z+>FFPxcV&mzh3O$u0PC=q+?+Y;P(xPw(2G45GSuP5MX)g>BQCcwdvZ-w8)GYX5cHj z=XUtT0hD4D@A4@IcZDwJP8oK8+_*P#@I&_!D;8Jc6=b6QZ(apU;bfl3%MH)7HA^cr zkg-MY4_O}2x(uO%&@G$(v zYXm%n`a0kNX8>mOu>X_C*b#AU)D3Odw$`0mos8nBzcaC4Bf`{0^33++qo)NS7st{?&B(uK(1#<&mi-obsK13r4lOGT<5Bpz(Q5#zs;&i`HN~PHDi@e;xiBeCSYt z&K=9)fP#YnS-?P$=meApfJ5hCZ-jI=J!3ljyPm!^<%sb*<0-czn5-J3h2Kg9-QQyo zHVL1UW$FGA2`pcqL8!$YU=AEp43gAJvB@qf*=W^1!Aklq^(lh*1gjpFK~Eh)N-9C$ zM~iz?QzM=6z+da#BF=dTev$i}t?a3@`(p^{e5=pw%zQpM%WKfAQn)db14q0 zkF~TACgx6-FbP#7WnkB$X6|SN26dDz`u>CFgMaC7+@^xhwrh+dbu-yAWE`jf$>EHU z8PPg{roV-Pyv~`!-&+ekDOK%#@+Hr$@bDY8$w%2Xr?F}`zZwf}5(sTfN-ek7dm|2A zb*1jEFEtXomNpTqnK!3JyTnH8hY*lb3<~YYcglX#xVJ0%V=WEn@fNW}b;W)O7cM+$ z<6?t5s056lM$LBeURnz98pM;PcjU#VjHTba1AA%^>mS78p;ze5VTTpN5QYgcmX5bR z7kbyN`hTR^kjZJHTD?(EU0&rUnZtI_()Pj!M*Yhak3DHFyKA=o*{knUcn~&NRv*Ox zY6fZ3&m~oY?KKi^Y6H5C$LiptQ+EQtIm0j>L)$N4J7T7X!}#{m%jqxAz07YvQoqf_ zp{9}XK@>J5$NR2;yLh=Uew>x+Lq!*c5V|3tV(Mp&S^z5Wn({a>0#V&qXkcx<6n@ zzqhWti!Y&7!SA3kRjQFkEK&6Nw)^Z&?ecpndnHup7$vz^gwWHvq4f<#w3|v8VCO`d zECrrc9KngXOH#=v>=*0^0Vx0fQYx$L$9%)EKyjeA;9NC=860U1w?o}O8U5!~Vrk0kg(-bp9sm9&lm)>?M4 z$XyDy-D_zj>!RA_Os$xrl72BB%z`B82wQG-KzrRv@n!S&3v#h0~5zI7^4(7?_Spp zkEpcLh)G$nTZR%Hp#%UO`57HPd^(RM0WysBYNNICM8)EolDE;U9(>Jz9QZ{V>V-BX z?r4%yUe35act_2z;BJW3GXjbWn{Vhxar^um8kZ26LCv;<0-|dbhQ>`>o*Ef+c1{@=qsQA=X0Mo zKy7of0Z<`$CxIdX2shl!;WoAiKz;^2tQ~+0#Rv1P^rk)+A>IQ?#{e(#vdSH?^y?&# z3kUM|LgEH*ek#==Z0bn-3f86?t&5D$h!uMv32e-=`(}~|H7!@RFVS<=@I*Dg%U@}a z6x9L$!18N(VXKe4|CnmBZe>Ne{Sesa{hpMr*P0Xp^dc7J4;?mawF~6U*-F)m0zY_Z zha)BRgst<7+u&mrPkG2EQLu|TS<0?=3+|CTV@WpZ)MuE6Rm^#-gwytsQ#0@m|)C$BMC5x@m!fcTFJ%0)jWTI>!f|%*& zJM-Es1JGEnxZV(O=1w{BxODmZp=nT4Mkr21yJh4~6f=u790;hvP>oP2xz-sQ6u{RT zQ6)%gRQ-9=kT1{sX6noNXy;z{ox7r4xnC3~PVx;M<(!!~u1NVW3{;p9s|v|{faOVy z$k8uandK-uY^dBI^|BrN$XOB8b``=cj#YpEs9RdGf5lE%CYboZKX`^cX+m7eN`WdJ z`wQizqn)NyBMh5McI;c*znz<(?L0U5h}p6hUG~qncP{GeU}XYFfxJbSBS_{2?gSux z5$S+(`Dc_yE6f33p}1fZ^JTzmaEzXZV|JF{UCGS@x-qy%Gi07+8*hImr(0bYt)%qQ z4`#sRxIr*S7hoOeC@6|Q`kA@VAI!Sdh?v0*}XaihX%`{b&mgjz|4NZPb%<;q*L3a(Axg4%20-PH0S zjo{M@^v0N>{Sx;?XdkTJ<8hsS z5)}S=)o?GiL>X)kc$tgh+~FXj56C0$z2T8WUHllCrQ;eu`{NLRtV%{&ZJ?3VlnlQE z5$2F=Q0RA?yIzY+pm*$EYGA7M$0i!cDE^zs0sQE!t>I=sAqBSs`QMI^iuyFtPEVbT- zxG%*>axrhb;Q=^3HU*RN+!!0TUnHPtZGCVIWb(i!P0yX2_g750sc})%3#Bet?QoW8C_Uo0_U~Y{n z%IJWgXzC|qSYSa+yUD!WcT!3@xe#yr#&^qoOqw~#`syzGr@Z6`!jVjqy7;+)H&0|_ zbGamFroJY;?ll?lJH|9y^SJMQJ@ugaOI9<#uG^qFK3BczXH6+7E04G0^XPuw)t3lPs?dZACt*h!oJhBbY-004NVcOs_c6c0U$tz zK?F$O1;>+5;jf#>&3t(sM!EK7fq{l(Pilh;I5xOGWBxlHhU z(f&+Ap**}&eihIlJv+TMBJ^HSyD+^POtq!<>VHkLVFhTAwp%S25Dl`WX_v^SiBpK| zSHllr3epSZo_(+m83qNy1O+;m2WjqX3#6I|6H{Az6rgPbY#AEWfGiA*YKl_>Dt#PJ z7kTX{n7ML^-L8TUY}{E-6?13S6?%eX?lYv6Kp>ei)j$CDfGdWX-l^zj&P-ouujSTFd=`(#43 zy6g=%+n;7sL`r%UbS`M(>qnm*Be@(Ph>)alMO#3d|J90D?v z5)mpKp^5lxkIH%8k=G=+X3($1*&63RVBW&yegs!-ugm0|f-OAhyx!O8Um9?W^ODuK znj5#9-U(mISKgnh&_}Y|zK2=KRjGUo9H8R3Gmgtp+xGIm#kWcJk%O&zS?u zZefc|Kf^Ba&=OKw_Urv=*`W|=#O?Ye`}Lp??0d9Q%Xazf48vjI2(g1I2|hRnTyQlt zcLED$XhC5McQ?4z?0|6xM85?auL@m+W$;U=4}e9$xH~};*LWMu1!{E#;!%8ativ`M2f1OeL&qr zuj#*h)Ah{Y(WBh{&R!`to+~fcZ%>B!bP33U={*{Gd|tq4em%STz12Z#?-2i?;?X8C zSntd|PqG4agF4D{84PN0aD^)%oIK#>2w)R!=c4=%Y3P5BmW5nW$#lDnC+=M=u%l1C zTR3zV7$gE7-WOjzw0(AW!;{m~L$G~*!F<#gIroM%F+*&lZci@8KIfvyaGEQTi09R( zp7Scyf%Kv{QZ)5G!wkMuc#Pvr8JgB*LkALCh2#8c%Qg3G^k(jVio0^ zj9wa7%Ees^peWESLufl1IM%Xt<#AUsZR%1?Y1_PKlTT)He+_%OYKD({Qx^Bqm(`Tt zNp)=S>e9Exo4=!X`73@%K7UGR_-?@ypK54BWq~NL>ZfVP3$4@BheuNL1v8hH?P**> z?t|(;%dc~9Zm=N)VH}_;ASwK>pV6P*UkneNlsT~e3g5wqzjXVlG_?M*9})x%k6`nf zWLWRXvp>L`VY-#L<*Eg|?Nc4YWp*rTJ}uX7>^oWP*hv$$zX6Y*fVsH1hS^UlJIp3G z-`AFC%|)}lYcfx5LK^jcKOcy$uh2`*sLpIwAv{)Yjd8P0NU6u|6>#Ob721UjQhQ}Y zB6;_51XszOv}0DmYG<7e_Byf6pKKSCX=Bsym#h2KeEGU>2}Mg>SNR@#e0%Qr#wqyX z961P*!R>#BfnzexbM$NxINb8$I-cS5GR`5nWkw#OYXYovx-fuh&ZEHV@_{vzoM0>)o2*(|wdPhUZ{U%O2TcIE+~ z&*sL6$q=B*pLue9jJ&U-+5=AGpQ&&LZHv2m;WbsTVO1Y+DyVtmhESo(u3!fMJUKzyhhz9JA3Bc}{=mI+jXXO?mTsKb)1~~_{ zuipI+iT~(P0s5u}f0#8W5#&N}t}4*P31w;nvGE`u9!-P?^sfDFIZAk(+pkh0asBB_ z?Mz2ArI0skH>xjap~rq+Iz+LbYu{%J$y%b^P zbwHDTmGQOEZP_*HET&yfHr+g7SN{a1Z_dw)a611{&9pA zk4B+b!txs#9b4-hR0WOg@IjtLDQcQWiB0X-N5j)^-U5`t9HfuneF^hg^X8$u;g=)@ zkFUo8mz;oT1~B$#a<*PO8~X)6!|w(pubrOmOEuSb0jG+Omx2NB7riq*_j_^%Bz_x9 zm?GG;pY@7jN}P(EO3=GXkFzz*8qi;rjZLH;6 zS0O2@63W9DuB)e^m&p9{mA%(-7o}o6JIz@vVi!DkA(gP*?z^R4_P{WOq~Q1pUMj1Y zi%%KaBj*o>14sk<%iM~qF3j*hIsnLt^jx0?D_}%mXNKZrXXl4m@R&je>EGr1Len@_ zk3eZrJ1^PY4(+q$yZt~dw0s9i!su|W->VBNwt=>_Pn{SBTLCG(8 z>S&GFdi%oNr=5a=yt1@it}7L)llXyR%ad=F_bR7Py6TAg;)}09klNbRS97IqU)=1$ zz|aC`m%R|kZSbq@`YVe@)<+jX__Q5i8T}LO{|O-3IKaba@{b0`hzt~jiSeArUP)S9 zO$LUh*eg5PL*tn(z-6Jf5!^gzH-X-;g-pl#tqYpB`RgF~+kP2U(d$7(l4fHM@*{bE1K*5dyA zR{V91k&lkKN#Vo_6J0p@`*i7|J<4|!KyH~`Vz!OQia&KL|ZYz!&z>s-l;v%M`4O``wF>g-?+(h>iX zOadJ@E5v;H|0yJ)SMXfwr#0LbBzT>f-b2{#*>nfYEC2xnjT9_k3RXTx7DjpTWRs~K zE3sEYYvrmH;WL4=2m&%51=C$I&y!`+>dWAIJaw;PzVCTurp@yV7V?sz}I0nF}KDc?xDCYQAAt@fRKw#RLuA zPC%;V8{*OR41BIzVBPkxRk&ZHND!9?O?Cw!s1sg^PA&lqK57`Vyc-)?&x)H~JL-+w z71#16kX%yOglR_&LA$Sn0pGgnOax+00Qh^S4Z+|A#Jm6sLqmPeIRx07oek^>+}s)d zzqdQI&Iga`Nss6Nst8qgpL14%CUO~Aac_he>&)zyN-pSGu`G}@~3 zrtu~2La)vFYv0*>ai2)Nj@H;90T}?bA@z5TMWxxkNq!>W6NKAl6sT8NO(AQ&mcBUs z<2inUO4%GG#b?J!=(POY%_l-dHk}0kD1Ff}T%3zody6-J}w>Ks0K?>*+LgdZai82T43fx+c)kp0OE-1XD3Us2!3DjL~d zcTd0I1C@ZBF3-K90Lc@Gxj~RFOi_^&-3I&>i_pm*vAZnhGQ2hT8RB^UBLq$HX@UC3 ziy%Y_VB9mWMGAqOyC#mA_Zm_KGb|$ex4u<_Pl>xrs1YkA;rtVmTea0ILCnz{zGm0y ztLFrYoM;v6v{Ld_Ni?Qn6^>IbEC&Yz*xv<${rm6RIsRBBSg$CF%YDkcE=lM#N97u~ zs%MbE>Z(eF8vf3;CO$ajx-GY^s?7o?_mMZSpQvi?re4A&xZ>eNswe(%;QkrzuXhCD z{<5GXZjNNEO)Wo8wV+wf)VNA%Lf78}l!P9X`zJ1VsaWN&1L}6Y+_f0W3v3w2zxrGZ=S-4iJDs` z4?$|cM_z@X~ggW%_0>2B~G0L*mmwu^2(I&J_ij!9j43Ey4{NLKGi^m7A(L-0YvN zn?0Hh`)_LXi;lbe?X`w)LNiaS*{>I4ad2Ezf${>Cm9(UgSuDxlVwRph+&$hy0cr6l zjsFd*U49*Az5Z$aWwtA&O>KPj4`UK)#}b57UHy7zh@2xS0V{7FY)_Kpxj)6q<1>B6 z|14WD`=(fSs-p8ULCxoWjGMPPuG96a)d=@^JsXyy%wfAr0dMrtOs?To*R(MZsAAP8 zSP3F|^uaOPWu#Iwcs_C`KB0?&G8c1VKPbET3%JiLL6emI}e?Bwl&jlrf z7MJ{d=Uhe>2oJFRHg_@ud6ec(4(2XE6mRPeHtwJ~H#cL2e2mb`OFal%A<^oDAycu2 zr23j3%I;dxq4;PWZ?fjsU+9?E&%BWN8|&_~CoKKrTp;k2l=!95uW0GgJMAo(HzREl z1*r(WllzaBzk1)BNz3?Jn0SKE?6nrNqR#A4uK8ZDbEkFi z*5jbG(xlO08(4I2VuT^my2L>G^1GGiO~AM# z5r^gG*Y>{I?Nj!zRU8-}yf3g*qsff`ojZTNvw)kLo7mgi0BNHY2=|7G0ZSRc+JO2| z4hWf(I78+%(;w*8x~&;1$J<`6pc6}1gBB_=dr=w#B%fXpQi?QooIY%z{-^}j;eNjC zCT51Em}D0s^9ySE#gtO3I6kVawL9#6r(8a6D|^?Z*rD*(B z0k7;>km{COWDrgXdt&gIzAL!b*?vBfqc#CLZ|S>D5xel`4|M$=ty*hOtQslKmdnMfAg>g)v2t*(1P`smx-AKXUe zrR=6tR8}~)h*9x6jSrQr zaU?^@_1a7YFvURQ9Zm$=uxSH$%guUj1IxYEC9-x0;{f+``dc8l)2O!e_2Jdm)HHJw zfzN+@*W6;zEH;&X{^}ZbnKYa8%TNotBgtUWK{AcxynA{Yh=Oeaz7$J_YePwZEv1+c z=F&)iEYMM`s8K0t_cRwU<8`L;Xktf@!_xk;uCCorKTLz+F21|r3k>#D3IuC$GR|mBljMZW)Y2P_IyB?6ToB55_rD0wFsTlq5kVDdjNo`?UX$boI!s@nZ_oi~#6k zBLS5;7dCMNN;Jp?708lNrZ6QtG00sN-2+&I7F2Roqr6~YP|2iw<^W^b%&5JR1&?w{ zI>Ya(lYd%9$9s9bPn`Y}o-IO4PhZ1~_$smNfA{Dv1&xkh&{kqYnKz`&y(kOs>)}2A z+;Z}Z7asQB*pf!L#MKB29@BJUb`{+)b6%|bhDvjSmET-LbqjGtN2&HynI>hk_E$?* z4OFg@SdUOCUrAZ7A*Vousp1N7ixt_-sK>ozlE_}Eu{(9@pi~TCCw(+Jvjw_ODg0sI zD*-ddWy}klL)gB+;u$2n(`z@5_(H)6BOuxRf0Z8>YoA?on-BQ~Y6D>d=kAxlHozTe z4?{XT0eT%5s|m^yoeJcP{;rhK!9d~-={Cri_yvXSPr`4^C2n4Y;O>Ls$f@fsqw2i(MJ+$@}q zsKF9t1%{!XyU)vq7sr>!DR5iZVD-EXR1=B_4Q$Dro$=5D#I!)||+N5~Cyq3*r zAVZ9j3bc#*ocOp4KhX94Ps4+HYBe!k#+$RF(=e)}M%z~h8GbbnOJ})rSoPD0RR&^e zlkWWFdZx@CgBQ`6)Qo$%`~lO~KwQ{w4BNb%t3|bcIKDP~eJu15lMogOOySVFp>qcq zz=Q>y3CJE6rUqJTpg%+VS^n)0H-2?@0U0r{mEUy57>~U)5FJ@gUT*<2NzB68}x3yj_z))DK`Tr#|Q#q)=7uzZSv3rkPQC-F%6U{*5~YBOi$ zk$mg$A;~rUsv7l?GO+f>l8~G+t)V(_|M(4}?(az{Bl9H*bhYwY$>S;u3TrD@WuKOt z>vd-5ye(P>CL^xvaujOXK^z2+;A~?XT8l`)m$-r>_<`%&AQ9kdG61e-=9j>04$jcp z8+RW-1hvPf+faH95^yzpWbxj~6`?kH9>TbP+GH`0X_ux?xs=+EkZXN^rv zTd^LXr()o(jQ$>JEDhyuunmTukl4|y4?x&rsjw}a zRQs-UkL>3}D)L5Ew8%eTlJ({5b6Zfr0j#sd3@@$yv|%&SPb=#=SRs|P1z!G-2me36 zGlZ#&l@k(XiA1u3GXw}X;Ee~wn}2q1|MDmrO?@x!0E7T%f$TnT*3E>5GvK9U^M7<8 z2}KloAXw5ODI5e#rU)u6jA~ineW4&Xon!OkG_D)hRuy;NDKo{c{ka&7&;S>a#YH^T z*y4T}$_kR2cn#LaBj53X3Is4S<=hYJ6ov~e^@pp?C`QG`Mu)hC^P0*_W7aUJu7qO_ z(hyluD9e_&n}vvs3Ii30M6us7hS^V7nD0IKk3<$e*21cM7>M${B8D&=R zw!7Io*{~{r?+uFm`)h&H$?uxNJ5&0W`1Vcn@a*7JZ3QUI6u*)7 zi*Fc;Evun58Y}m<7;#;KW;hErofMEtE0247ILGwSjx4^IV6`uEdX-D7G0t^L zxII3NM`Pz>JfODviL*k`6xXrY2Gmw#G~i+LmWvz(6=Mb!JJ*z0PE=7IpaWoa0$BSJ zK$LH1X=`pK4dxbLz63Rpf147U*58>Lp>iXe?%8C6kd*wTlJ(;jY4)F(M3J;A+3Y1RQbqAw-`^X@yi^uS zENdrTAN@l^-E~S^BTiK=<6Hez&7hvU7j#uWqPp4Nd-w_Lt5w1W_zfxwo%S^|Tx*2+ zhi_@|Dc>JX>f0>0cFXP~@#?WLv`cQg1_de~crtr4*0VZRCso*8Vq_U!T%o#iKz8wY zhf_#e3uX1@^LZDu zo868jQbfC%sJ=K~Dt5@;`=z!PiZ{quiN-VxEOBTPYd&rVyjv& zLAq_yT2RPd`Kn*$ZG~GXimK^5n9Rm!KjMC7G-oe!cH^CfqlJu8AFMv_NivE z8t5yDu_<^GdmI+)#C4p$!1IgMST)ma^hm0L@Hx1GxeJ(2Cs9e2C+`v#mnmBU0wU9| zi7}dMb$_&bVlVii$}OF=vf_8p7bihG=DkYbA+IMw7~n$vCh>t$i2d^f9z`wU2ZSOi z@^Kb&MGwgBO>S?>^GM-(;3>C|;x-_Bxv;cqPmLgt1Fz?RNZCd9A`g@VT7;jgb+mVwDuXWalghO7CAh+RbLa1f)t`i0L1tp>?Ae1pAKNXeIHoi*VJmIa6P554p! z$bE?zEr3Xt{_@;Kg{N8v>le5_Zs9rLMcmk+1XczbO*nTGFgvq^9Ne!iNtE*{X(ULF zNbXwpu`Vad_?73>p{>ywzYLWxamh#K#aUS8si5lfClyyFxtWx$#SQoKqW9m_@~XK^ zY@L>B6Nb|DT==xHujx5ObIAXYyK!9U>qb|3#9qLl0vnF#b{CIFBdd=OP&sa1nYkt; zeR>M199b^|gjcZl#WPR{98idJGahUaD0dVb4wI5lhZfy`fl0kbk$3mWTd`}Td^hjL z4?O^;AiyNHc(D#`eF0|w!cWb^olA@l+HR!1=9R}}ZcWs%y)1B*D;)|yRCE~Jcv5x!SC&dZ!eGT{ zfn6INB4H9>{ z_l;Jyng_4NU-;Am$ZfK9g@~GKO`jv-&ZvlsK$Mpu_k5uC0RvGrmKDS+g|ZKmw@#k{ zo9p^MfPA+z1_9I?O}0I^dxRqa_5oJ?Fb5}lXFw@AOZxk_7Xq%)2URoL7j?aYs$^ zzRn0ELa{3)uC%7C^j;QcIByZ~hv$kTJC;6neK5M8=5__Ka~!>rgHy4ZWU-Q?sRNWIjJV##n%kOWX<)O8DQbAgI)jNmZMXJzIZmczQqVnRkaV6QVgrnj5F zQ@M2X=EcO3LnTrsVunM{-uZTlZSoJSYIJI?+7EV}7lA)cmC!{ouCiBN4haD#1%iIg zo#4^hduw}ODFbmtU^1%8aMLr7zJHfU{8`s7uh|WY_tv?14tisI0O-jeTVyXEHiEdYiz8PFjBPsqMtfCu*DN z`K^n4C{bIFL4HK*PBL(Bg zscg0B;xSnh$Ow2ZE}Fgvy? zO{UA8N!?r3io>ilh)y`0oBGstm+bN=y24vV0HQpl1h4VxNr`v9Z| zUEi8N7*!@AY@jB9&3`^k&BDY6m@$F8Dm&na2E9su?;CR}#k>d);;yH(-U+6AI)o4W zm~0|K!NP}7JA!-r^mK+sgbl(6CA0yFf8xTG=N$zQJ~;canX-n2bemBnSnbnu|G2m( zjvJT7q^rd$=se?x?jf^si87@VrNl$EU-d*(>3g3bYs?qh3f#l!C7*m#uKH>G;PdAL zCCMuBr^95q(RYVF?q()^WF~1SsQOr1Kv`y~g~JaNm1}x))mf{G%~|M$&-m^H{llLm z<7jBG3I}anILZFx|5z7(KJ*m?wSag-=MwmVssRdFaIr#2L%dHbz?B7|H&9V}f!h$E zG-|jMnEqCeM=A_pdcdr7DzW(EX$P5K+`jJ;rKp30d`>xhk?8KxCqQ!WGSKdFjJRLM za@(%WNkr<(iE^#WA|2(|EB6DA*0_x4xc5HYa}A-tQf*ow znAz{(_fKuo_-6(*V_;)gfa;7bdi8gw5}1+BcaG639AQr2zNTy5SdXhKHx zOhj!O@7L)9O()F_$k^k$d6+h+&VR4Npxz%chf_eQA;i#%kM6T;rRa&=%bkmSt$L&H zjU%&n=;DXYBp8cb3i^@yu)ny5XPIe4WWCy1_R`b`b`T3h40B@?)WPASv z)_yJWR~^jM23jw3w0i8E1c-6QvM4deE?oe=Zw8bv=&JuWTVdlDo~TU#+4(&T2BbRu z7Y1{-M>&}SYhNczb2b2iK`*s6WF-qTF#~R?u>ULc!v%lz+y)T|w?sMfal@qK&txNi z`GTN`85Qd|{y)aPJDlqNkDC-FWM_p!=CQX>_9i1c4i1jvI2?O6?Cgx}GAklmSs^2d zl5miboskMjD$nP8bXWK9aX;7dhb!)`xbC;lcz<5+*QiX1J7rD1r>S{rRI1Ei$kcYu zADr3~S3hZNH!jM7xrP&ce*Mo@5V3?ZBp6Nxx=9g%iaQfGX+J)?;%9e64&WU3OKfRw z_G|Q^fTgr4P!^f+(_x-pplHIZdoG+4i%zUd!SqSK90_Wbc$_#@kwZKs zyJBE4j9T4#+j}ZYFAGTti7Ur$4Pnj1;q>Q=0k39!5AbUGh+_*j)UAtcm%=W}os!QR zLPXKKZyB_KhrDsPzc0AbPeVEgM>QMSyd3^i~J(urf~r4y#utgYX?;$QOR#Lyk80R0zcG z8e~2Cc#=gp)`ML7V_%Mv)%Qr<2X^YWxu655Q5vVPt4c$ zq<$uSzwFU3ICbwF6p%w83qJnP`TujBK@px9H!tAXGwV@Yz*42tCEuLV?rx!8{R?8&6nAC?RP29-^X~4!Ml&>x= ze`w0l+iq&LP%4Qu8`gT4Ls`4P*dG%`|N&&Tvr#4jzaJ_zstMU7|1cb=N;o$2SgDC z1Q|ceb}~hgX-_XUa@(gGhXDk8Bj4(D_mT_*!HQb93 zOkj4ncmf*{9gcJpm$titP3BT8Rk4P(q;2a;$D^kX9ZTBPI znUaGh%;ET_5%gfqV#ctrV3iGDoVDq&zfr9uYB%$SPQfrb)PHhw3{jYn>X~$NrD3}> zS|q#7faU!2w;y^Qb|W_ilaKGX&Bf?GotaB+I#uSB*H#?EcJ)#O8Bo#=uoCls6yMtm zzQWm5LcaWyyrw#O1C+!Pd>H5-Fp&0yqw$Xoq2HhUjvI8q_`11bfObSgQ1tg-!@w7U zK;R~57XWU|zvbSP|Ftch;aKqgb$}s2Os|l-E8GFTKZN^Tc-z?C+rihY?(Di;ZL3o` zv@amU-e9TO7Z5_Zv%w>`#?1xVEU!?jNiRq+5v+`&gGB_?-5Gqy?DYz`q+dwe(PrR+ zHwAPuqNy^d?-v?AHhz3_y-F-q(%Z%mAksrrm~)WviVA~5t7*Z%#^b+~)T z-7|tTSM1o;+r%vR(rgJ$PjCTZQuaABJ3q6%y#__<+*)Ff9pdcFXkeZX!AJ*ps&H2) z3=%E0{|B@b{{+d+Ce7dVivREw7hpE60^r)PM1HS{*-ORb4kJDi--W04zWVJnRZ=!@ zI2Ge`YR(>+KG!#HruXjfqes(hk#wsw$=t>Pt*@AG(--)_ZoT z1&fayEQHP5G{SnzUp^|90TTfDctSq4l7-lvArPf%;w0g6zmI1nFC+HU+6Sc(@0Yz3 zRnm;@Rsw6^f z8edl~+inZdI7Dh6+;reM-3J0E0I;hpIeKby&U*(dTaGk{K_ zlg+UIi~$&>`G5yVZxu8K*b4@jwrZ4{7YT5EI>!yUKBa9}9S>y3Gzvx}Z?WdG-u4yB zr&I@aJq4r#G~+K!%zDU|$)B5=GrN*6SE+H8uF2^2OC4%fQWmyC#OjM@#Id%*i$1h58e6uzYKo?zeS=Axs2YK7TJJuZM5 zF?>}4d4#gX18Jw@&sP&c(0@Kf0piiVIlCJ$XWt<{iU|jq6Zod}z{1#LfWs;VVUL6h zxx>AE@X?k3LJ6!~+9BwKnHeIQ`aqcVdSVuFX&DH--!ICOw}Jw|5g>yRpaiZ2UFSn; zy-&V;)%kd1z?%@>=B*vSk0vP@KTyXWNwJb72|0H(_-4C!92L6Ep`a#Y{;m6E&^Vw3 zL2Q-ouKeuHU;!B&7Bnh+bxPp*@Rzr|5=PfI<;j za3SZob3UF5j*gp+i;8bP!EIDLI@1{6mBnBnXVgD%?eZaEAd92$b|-#U{B7^bk0kep zqKUmyvaMl&^7a4Q{JXV^H&dpuGwh3xi%)$N>ypKuE{`>|dPQn5+V^T|%~B zHe}$#dNL2-p1@lL;GU-{@D@jqV9e}aZe*f0_GWNc3^_B7`qMEE6mHZ>UneHQT-oe& zPZ91|3k@KO!@|?7OZ85zciQ)5o1RV1ySY-v4}7+=qWvkh$dx_Ldo#p`t_H+#`_t}T z+zMPIc3R^x@GiLc;U1qV=F_lg_fW-|4B1xU;K!0v#6ny!U;49g)r$66p#`SN?p<>| zI=Be&YtgKSfn$EDl#YhG)b~AIMb7Ug47e?tvD8S2X$U!y{b_IiTBU%s`fZ~OMb&TFik(jAxnbTZ;+LVJDtU{AdeYQ^ z{@x8G?eT@U51z$D>j8SmJ_c0$YU=Alq}+L}+0V=Rd#h$+j`n;zqZ}X^op+0*O#kVH zel&N+>fJ*bX$>oLcbNSJcH7%gxF1@t4_yk+KTH`m^O18{ zbGlQc1ukslQPX#>v9X@pJ}gZ7F0XT=?uyP=Od$If5d_RX+`twrz zCpy2PpGB;1&AD8>+oDAg`7YG)*=l7{8I!g<5rLs3q2uBsN0z@c4zR18J=#M(gRb8E z0ZR~%bi09}m1-pJJ>yhT{h;KlZSBNpds?UDAP(11n91LqkP`HwzU7fn#I^7l0>cHY zZV1=p$*HO6#QGdd8FC{I?$2*4eU)H$lQGvENf<$c5hYtA;9kFfFfNlVeK0lpt%5!HWXvC?HC&G!42qs{Cvr2cf{KI_91QD zz!7yF(_^S4*Fv$}X!4g*R!zsti(cpJlxn%E{saM7=R~h14paIzdIM=)I~SSfxu>qB zI!4d(MM_K%j>e-BIRasZqEFFIDuN~~y&40^$2*-V7g+@0WLI@=3-w@96NVCeyAf^?dyKK*=9y#SW7Phm`+)9)17T+@ zJOJ=nrDcS$K(zpabW{7hD1fYn{z6u-(y&uv_&216FL7V71Lg#%*96IT{l!wr)uX;r&8U)7hLX?oQ=e4_tg+kYL?%uO ztXnyvE{|~C;yNi*`>JW6tR!G)Gv9x~$}=}ZMPvUFwHwqf(Rca@+}Ranu^Q)mxUPKN z<>}C?Q7dIBrqb+`&=fwS&6W=VqxP)v@$OQcz)$fzv5c$7Yb`$-U1TV~smu^g(WK?<7dE9e4};6r1d0pyz_*zn0X0KGIzg?tsa3o~=fCbw%@d9++s_s+r%;MLa&6mgo;1c`vq4fz0H62OOK{< zvEp)4pI;~cIrnqMU%Qy2t%glTJswnsELdT4^E7T4zGzg=T+F&WM9+PDy~0u0WF#|L z`i7ou0Xga5 z=8kpcb%{*rczY;E(a;zm0qXB!r4JPfzQ?LEujmwQ4$Bs7&~81IKi3 zkmJTQ2somHk}n^13=GyWx?D`Xxt-(oyo&){^Cm70H}te6t!aTyvZ`BU05SU6uunXj z49AWh$0qBMb0yG6M8}5@NB;>7!-5K^0eHLQjDcvX!Kj>AGBax7eP>*N3td z<5F*L#Vvso@;{x(G?4B_#)dFs;2vMNF7*66>9n(-44zKryMky&eA@c@cn?Q0P==@Nkj~%VdsNOMI1gwf?#^)jPR6 zFG)fsS#|xCHMYacNI29%Oz<%+*U6%#fYY`2wumZ?^szMCIcz9PmT!QMp=FM zX3OH}TVwK>BN$#&9T)Fjm6||f>&`xXJCs|2Aerk?>BpwUxG?onBDp<6uUoMY{R4Wt z|GAU@>RP<8PH>Efu#mdG5zNTnP1*Z@rf)qjq&1Pvh_;)mqa5AdEZRt*8_6$OrjtI* z4LBeGu)HF!zxM=m#xX|+Z0}uZd3*crVmJ^&faEYO z`k6bUjt2RK)b}}e)N0Qds|=|<=N?r!!`JbGbGWJSJ5aJeTF`unJlb?wb}&7Y0^rlS zxDVgD*qL0Va9a&XB+9)p$-4OKIq(d#rBLq$;w>fX^3I2yyVTvxG^hHe_3O0q7Ehd7 zvn_C1bEV5Ck`H7DeX*dgd)hn4?|b8bT5QVHvBliPAMp^>1WZ>B_BurYxWpSJ-faGnGO( zT%vU+##~g)wa-L1kMRdqtl?@>8I1?|y$$MmX>D$)+f3ei+9Nzq$hiq5%UVGXJVL1R zq;*2X5Tt2Bw0#cwPFM&EfU$Cd{dhxG`sv3Se8%Ugyy0fgMJx+fR0{DWci^ES)DeIKroKk=71)r_{n1 zDwqsIm1wR7+}ync!z@Fl6xD){cknUII!Q2$$OOgsd#}M5>)AP@cmqQ=wkslury?ohGB-wxs)N_?|VWr$v9E*aC~JNuQtDB|^V+L}kH0dj5| z-=D=;Yin$gzrRPTG}w}CMWJM_+a3D0F_*u4=S~33L2CI&o}r<d+x4M6$Z&5do3< zz>T%Ny7?VsXrEw5HAb81lU?-*1@)R_j8SfDITZtVXZ56y=CeM+45R5~u*xt7x7GyL*~ zTq$mpTmOZIIV8z7!``Q`I;FCGChQ`@3s#eM>dFFylc;FOq)x~@T!Gzo*B4BYdmy)L zAL{Z3n8#L->oq$sbn#8vPw~aC*_y~b1Ny&^T~n>J3iEcueV``#_vK4{v3807Yd&5Q#THU?96MGtdZ{cYjc@>e1vzb({UxA7XV>JUt>QvEuFGpNbH{09@o74R0bkGl9uW=T z>v=eOzW&FPAKybZJkBbZ*`GY~Eaf%Rk)p0*By-8Y%gxITb3l@^v@)Ylk=jO`a|(U4 z=UdSmg%nilK$msjJuK>iVWNFG=NZ2XDLU!ZGVYg@j6{FFIA2rioab`YbNKC{qgPj> zWxQhiFJb!E0ze``Q2gmN0+Qj4@?l{X0@28wQ5fBm)R~a`0 zGM@3Vj{dQXAbFZ%F2M`NGc2ACaBr?4b2>s;ZAt0_?`^)o@fO^$cw~P~Ltxr#EpzFN zq-`rVd`M>-Odpj*g~k+`PJX?#P0>T%qDaxqNHLefEbjfWE*w5qm9D;z%p(`gMQ{Y!V&qa5tUt7*RY6vL&Kc zq`C|f6hMLglf|G1jL6Y;SOlPm3n36_Pq4DUwZDJN{WH6SB#bOB9+T zuZ&%j#62_4s3hdzGIMY96vPfReYP-(lkHJ5wvzk|`Bp)p^xa(Q)5L#b=H+u*r^+r?Py#qgi9Tt*-suETMVRf%Lel&Ixv-5z8;A&viRH{lv|= zt9q*L_Cf0UH2%y32Je~(tzap@m+9$GTb}lu*1eNpQya73)|l_5#X5lMKrDs>VK!G^ z@Q!kKhPw(G=^7eCRrw1h*Lf}O5kw*hl5nMyafV|m`?=o~tw?CfPPlK)2+39Z+7f== zi^W3DX9XwgeIDnW5YCZ#KI44-*4@SC=Wlw{g`Tg94kf!^*Aj>!r_VaHjY~(Pp37of ztVf18@-OJ<(B6~)ft@+UmI|+_C$cB+)uCz;mM%9xIiO=T`Dk zb76Y%iJh%VNDxNy*l~*ZAu1h=`pg#fv%CEh#g6!}G`)JZ!=iSlQ<)Gr4=j^&9Dhr` zcV!STG3kiCe;mF_A2IN2!5rAMs({3KtiO&w9OW!zd_l*?NMnDo{BEr(bJ*RG_OF`kpzl5 zp{$8r=3d;Wy1q`YtbHJ7mPQYva;8)3bD2Trh{eaIizWBIiUCI5VO6C}d%P^QmpeP3 zd64arImauWPO9)KM77O2sG|=|l_Qnv64T{W?<5|JVoMu~^mf~0gFtVA?F*6c@myLr z7Ov^Pjb`m83bw82C)US}29cebp1S}5j}tx0PRBm%t5npBu30}J#M(uL{|Dd!F3y8T zQSJ~s2#yw!6uMw&@;_@x+Gnj}(&LEnu=7pn%A&&PRZV^n{_{Xa5Kn~JfwJ15NpS^N zga}IpBG*}$-j-~xzHT**aX4zuqg?!){1cG*0|W!ECKaqb2sQUMU#qo_p9Y%zwRGP1 znacRLo;{xb#ngyIXS(e~p+0MeJRKV*YhHN1dXd69F7BdrDGCugmyb5k;5FC^E?z=6 zZ_T4+Uc^IA>%fZMa@sp)Fz0Yre`GTMpy(^cU&=vk7Qzt%Iw;;$GB$qNZF?^UG!dmj z6)W6TZlTb-5bDhjRLX1W8Nh%%00?zFg}~?g75T^QtSqNST-U9YxY9P^Tt-dYl1^#I zDd-B*YlTDB)I>f%Z{GefwcFQi{z8Q89nS-Xj9U(b5v{WgrEi0-?nZXsOz&wfMAL^$ zhtrywnMcW2zD!XH=US%jOBSb>vz#)CGDHQ(I8%uwr~B#p*W27h_NrUFmR=OgAIkB1 zUfXzCZaGe}VL*36qRhp``1yLyr!wQ<;;-n@pYuL4Ea!{_&$s7-Oq=!%rn~&)Hy|!q z(c?wa8zGw_KxUdbOHgwd;*zb;04`bS!rqlFz+Q>FV5LAPc05cP)C714A1u=Wfi`ZS zKlj9-fN_PJC(2()$w&>Rj6?$~K@6x+;vWL9Z7dOm->6pe`9_GtOKR(I;(k8xrSuPI zk_E@{sH@ty*G+qiFCkM{X$(o@Q4X~Q35Kg}pC3t>uQ;YZgVkw=%0224eCK}s?#Qjp z(f(@dY>kcq;@#eesvU2*rl=L45-nmJlqSun@DSda*ndJd( z60w3~M^}5RoMrRnPhZSBX%gDd72H@LjW9hu)L=-6jk5t`Sg+H`*FmL3 z!N(si%6|f7_Q0>}e?T&bRe$h-j&uV0aD)#;>%=1wV7v#eIaNGGfq_Q&3zSnPic8Zt z|K&WSt2^;(aQLDHZ)087KoAnpfE6z*JhY1kIj$A$VnRKKK#uELFI=IGmZg&eouEo< zcN~~l?JBV(@hWTa)h4^#-OPI6YVSIc;_Z~#5S;p2D#Uu6Nt=e4#24Y)k-)j<4+*>&;7`2iVy3!-pxE z15V4f7~~iGE|huV%!Bmw5$7V7Q~H)!zCYP-d$gU5O}cIsrLWMm_CANmXZ8CTQ4gGkzRFdMD&?1wnLlY*VKfWZE=zyxik+kJ_g3n^4{W za&n5|0MR1Bu}c?@;ZDR2SY)ZCh5@^Qt;|~uXFb@vj0wW!ZK|d&?80716$pDJ%Ner% zXyqmozX)RA=RxedDL(eS86W#T>Tv~%eQ&h6(JS?1F=-N1>Oawb9VmNQ1DH1n>1wKI z!i?Z<4!%fxkoNv>0iE8=*~`^;49(ks=D^-kD;enLAj_R(ca7hcJBxi_XJ+)ZSt^Qj zvJ6nuUEP`+PRPH#U9llOy0}DjxlVulI~{lMX0tt`+}EXOfcG)=a9Tjk(uQG^$Nl9? zLsf}}Uim?DRvx+O8X72_>Zi~=^Bw36hJ~Xod{Zd{1}VyL?3j|`TNYk)q-~MJy7EiK zyfmtJ=f@xuOU%(h!e?)GZvH4~dI)bCXnQ4hMOZbPMAlBY|FFKZt_aETM}9!l8@kIz zFiiP&-^4Po53<#zWWDhT6bVB1|7^>@nr%aXi0bPhkgh_&wHRy>LL1dhe{Gul_PU+cu{VV{QTP_>OX;VU z@b`M*N^%X!eFf_~q1R8L-E~6yf*Gz}0ZqM*Q&xrC)2J|+Ktk!4s_%K!4KpQaJnN714$ znYiJ{SRy)Y1_IL)q`+w4J&7>oQNKI~t_k0-VYd z%)6a|U>rZ~{>yo(ZTf2@Ub-RqCwl8u0sSvhM$7i(HkjzzD>RIvbwspjZyriNH5$xQ zYjgU+m|d*k4*7#GA{E!I0dj0?T|w33pyz&Cze&NvpUTig=~HT&usXw<0_T4FQvV@OnWNSyvCWAjCAN1VQ45QiJ#gjN9Dgr89&J>7^6}^N&Zv$WPHKM zNJJWD4EIEd;S+119twX}m@&kz5CEXmKBxeOrWMdI7Zj3sI*%*L7H06aDt0WjjSuma;o(+cZ14<~@eZB6V6R1_v!}TLx z{S*r&0=_l|msrf^%|GfVe`x-XI32KPidPZ#KHd;tvmS~JZxp;^bLs@rT!`zk?Wm*9 z{$!C~w~d(ng%@mDnK#ogJ=wL);L?yuS)A)uDNyN>aUB{n%00F1-?B(oCm!AuDG|z; zAbq|xRknm8p#)=D7H7atLEYwj-)k@f1S9*b_nIzLuwPIhNN)zgSgG~${U>B8cYAJE zYQYPAqom=Rhbc-aPet~5LS)rVqDRUzCl2jlzH{tU&twDrJiC4jbT~sH37U~#(8`!I=*UUT&=gqPEisxK%go_ zp3;oP=dOk|fU3kiWxYj0b0gr3lTnfFPH29H?xs$^@oCvReaawN^V~c|zwo)(y({BD z&!PoIw5EKE8Usavo+1Yp1#1s@Nf?X+px}cz25+p-{)P3MFy0q?O3M&(G4nYaT_g$A zPn0DAM=07f|5iT%YJ(lAeR`@%^0S+;Rt5(P>$O?Nn|J-=D)= zZE;$i4^>vM3xd(R#VJ~auc+;dx-ZKWSvY;MT z<~gogt&?qck)yS7H=Wikf_YMc1nOHvE5gK%rxF&f&RhJjXexW}<)qZ351*CozA(xA zse|`uoK?5@AsC@J7!xGQ#R|Af{oF%?nLQslAU2eJi|PAyxE#pkb@m4{Iz0Y}g(NNR zKt%|R?EdB4*)If@Gi={gFc#tle&>AhDUj{N?UPB^%}<=MM_*Wq5GY`A zHkMiOl#^1=`C+-DZ<>pg%*AI|v>o6zE*;l(QilwENo3ekg3_wrpDkGQ3CfJTprC~; zd|&{dYkpJjW;3!L++G%DynZyBMBb$eSL*NKX0a-L6H7t2)x6x;V^Qm+Y|2}%mU=JV zn5flzC_S5lAERqlvLhL^jOnqM(Nl=W;!&m;i` z?tzwE$3WQ*oSo(4TMM{N7>!Jt$PiH5B}8<@!-~B3y^|{D8}@a~ z`^KFhILMCFNLuvixq+MHt+dY`3iVN*`1tYB`btY=y1Ol%TsTV2q^0M;GD9tV_#`;u8vdEcnY) zz5D5IOs1i1MlZFuQG@6Spw10Ge(oW%C8<^Xt9zAheEek*bBqztT+$2z&873Zm{6M< zc-3L4>V27tH>7X3;|Uc2>j;uj9N5jcdLw}!r2_~qLBpLffct_!g@0G24@VIP{J7;a zamgKq{|Eaiq=2cu>3Xa(6>s2vhc% zvR8TBn7NOy2cN?r61w}(_Sjy^yqix``7jxMlkJNVu~>Z)%Z_VhH-!$#!d(T&t5P` zK+X8gkoSUvtK1DtD;-3{|2_3(2jA}CmKw}U5c;C(SPV!xgc9wMPTl}~0|OIii~a@O z#?o->HY*F5RlyUA+kF1mGBXNB0lnf)t?He93}s?O6U=~Z!sux>GR3cMoSP4pk03ht zB1HUUpAngVcz1=_I+pVboGMlN+nl-csL4zmjy95*`Ee39^v7|xS^*AJcv<^9){7kU zJR%-?h>->VmZk7v@$earh`ie3N=ri?w3OG4O!;{EHR9;CDBSoY-~D?AAbRNY^CDFi zG9#I*VG22=Q?d_b`b|@L;*%9KWsmf(|C$6-3=ozS4 zK#mm{o;z5wpZ-pZjUP2VEz-e9nUc;3#aJcas>PixkZEOaQnxr2cZ1hw-)}q*YCUFJ zoX#lW}!>Z^8${Li&gpo-k#964d`mJ;+W%#nFpnP?8on2aQ>M=UCugDb1*dsNrQKM8f}+Y!Fzz3R9e(A&ajvn{OXzHG~qn->*Q7(Tye<^nAhL2P~g8&F1QpnhPI916tjcsU4} zxOoB^F4hNx_c_3HH9Zkd&}#c*64w*E_MLL-D!{|l+(bPDp!XY){eCj!+KHQd}&uuNwnMmXZsX#AZSN|BK53cxqrc+~=hbBtJ`h9@S@(nL-3n`Nk zFYo(>Ock7#Mn%j+TKv2hp_HrFIlL@UXxUa$&lZO#q8Q6`w=mZwwJU{Z1c`3D-buRd zv~thvI#FW)lx!g{M4(P4MX`MgEHxWn5OZ~_DGpNJLnNRBNe3E8Wf%Z7fdwz%82}p4 zUti&?^c!HZJx5IHm!*5+Rj@-gp#W7Z@c00hmX9IU0lLU*MBujg`XXRTE}DOxR;xb> zNEs3?R|h`TY=v*B`*}CMg?VuGDttFNL&JUY)bThz8j`0eXHtyQwO^zLDc>(O)HdC4Emy&@8iokQ8X2`ivLkfi!R!xK9!(gQdP8yV>R?@bV&EIwKFkgtt3j=3Ay zH@D@|0v@+*TeQbv4eQfpA;`(w+f(Zbf&wb*-=mR|)P0&n()zP@&y}UGNB`t%e>xk| zP3rTdhi1^EEr2tNu!n2vqYK9n&0@oHjq~&3(#A`Zi!b)Azo#gA#mjbp^|x|4oMojB z6Ue1{R;7_HR8&d4x$0RQPnAK)}o%#LVl^uXb+F!F)B)p`3E&V?_27wb%U zJSD@gcYYYX5d`2Q{~PlO=W7)=?N?F;sl8__)>$mZDXInwzg?DHi)eeV^r<^wg@0EI zv~8kS!yrvY^a!la_e@F`Z?K03>aI6{7<%+4V)DkV48> z1USfn1{)6Zfdf+}H!lz~=?Sbp{qRfzZ#!=gd<7E`6%-bSiHQOp0}QnJAd~vvH|>AU zaNNMpuoqGTOb|nix7WX3O&~7ABQ{xFfq>-v1c%%y?n{F}42nPSh(18Bm=tC|Gr(Bu z3K(mhj|%xH<_tzSg;XRjXI**~SUkRvOwCqney6^4mS=h{+3A%beT9^N{sq#V8mfun zWgnBAlNR?4>++K(3MehW6I-&WIA{epue=`>K$7j8;K}3Tiu&@V6eQM;50}?A-%G7k z$*Dc{xUSL!3~?Us%=?+MMC*chSDW%q-^dXOJzsg9RrM?RqDQ)&Xpn_Vqh!Vr(s#_y zUHaPjZoLoS$&GV+_1{i^dKgDle!D0w3c`q4!LQ}OCj$5tfP8ou78!s+dJ8!tK^{8B z9g1KB5#al0-9NmcHNv-v>EM?}E-9%Ie?hoVtH3I+`~8j<kjH z(``bncE>&lv;LF4_JnXjJ6Q!(=>L=CjJG%RvUmTV`3-a$95_3?fb{~2q;5he0PZ*= z@$b^V+qyU!n)OG7$GNy4={pY+Ug6RZVd+?!?-}q%>z)95N>*a3WDMe5QWuwdq2tupD$6$mf(| zA{Sx0rXVhyExx?pW7<@!zmYaw%@n-X%LqBaIJHqGHvXN<&R!`rP8EsT;Px$v$O6lb z6O`mYhd(V~yM}Z`V0A#WCQ8T?1MoW6-^oA!UU##EIxJZn#@0S;y23afieFg?F9&BK z+ciWr@Qz$}!pBOSRs7Z}4uHW2glhVO0rzyzwZ)LP$z@&Eja7d(;FrB!-ANn7VBXoU z5NKy%3>+jom9afiO`{(_#PP%A%Ekh^$1g`Y^-C;<_k`6fU{{9yTi##Q&b6zAtC=gL zqlILv<;g14*L6$D_spYEK0&rd^T4HGZ&X3H6#*67M=3dtI z4-Ckil^dNqrfvyJC5U&c-_Q7#V?FVc;~n4~g7N%?u$xJsPN4ZcxbgLL$9TaYNiH}C zg`h70KnQFX{>?E3FEcmb94+YX;P~&;5Sk7h*a+$%Q3xG>m=^GLa6|YD86liN?y@ns zt@g_gW@!7k?02HPPgEtVnZXiQMJ_?(k8mJAa(YjN2iZ6va@_kY{MKIxt#)&S)F5v_ z>SBeY>I%2?qhuM^ol={pu9ALYKB5~UUo#M^tN9$oCS9_e$$mXWFZ^?~{pXbKCY2a? zbCpg&+=vbpB@6qpwGoe|dB$7PehmX?lOgNDyqngen|9oviqvV9QcFy{M@Q@uuv45j z>gI=Ueb9l6hP~hCMj$3fgeQ-|ok@HpRX62>ZG}UR#lM0qUmr>0?xq(7W3zfkPJ!$D8i{irL1`qJDp4@C^XkLttrf!0oDz z*GnloW5GWW0)e$)g&^&~yU5EQ1_s9nSCB#md`7_<`gPM@07;TwFhg&!lsMpiHjWs3 zZ%-Rxk-z>o!0wd*KTTHVfS+cF0el*e|LyIHL0`Lv{Ya*1B zs111oszBKa(NA{#<@o7cDR-YGf70jlaPTG08?d?->cPiC_*&o^qJG)ibZB>?`$+gw znN6Y!g{>~N{Yo3U+l5RlgtR*H^F?&>mA1&9m!zBS?k<%9e(58W7hnZyM`U;B2sJJ4 z7^ThG#|mGfzKs2*6aK(bbGM{SuB}7<^v%Q7Fv@FtT!}8OY4d>D-b?xVeX^^K5J5;d3Z9;hqxLOcGoJ&Q81B#~y zsA}M)4<4={3Dy-jf4CxGpc(Ok=p|UB9cWB`*-t`!O%eV{MhjxkKn1VN$i9iR?PP6! zRA=84$FckPD(D*rJ=meXkv)y088i8!X(~8d^Lbljs&f^)o2t<=k)SuLj_m91RrJI? zP5uR1ju!)8oNX#eI#?EN=sKFkRA+e2(Q2K~_gAO5S|>w#*6gKEGfweX`Qz6*6gMi7 z)Z4MhE+!+&iZq`Umsr#0_DgART#qUql_?pJzjv`ACxcA-{)>)2PPkAGR&A}2#lPSq zy74EfCrjEQjGiD$!KPt7;H5-&){VqIYCRvGo_EI`8gVR?O{WaM(AJ%o`7yJ!r25kf85iZYvC;tLcj~xGV~Y8BUFs z@dqeAW$|dAc2BZO`555WvmGLww0~%+b`f(XJNIzC^v~)2!ovq+>NB}mhv1+8b!v^- zWm=0QK7BuI;T|`9SNDONstsQO)8nUACAyKpgM4#OH8&6WmKa&XTuiuthpwfAtT~LwDPQU`v9#%?A1~tLeOsiBw`?M%W2!=fSA23 zfY~eB7lQ;v`cM8AIG*6Zk{K>!s$+y_E468L2-B*5GOemz(`L%5+aW%A{kpb)oF=pL zT`qq{#)zFGBO&Fyd@uB@FF%S5djZ#=CMfi!c%fF{G0)1k=;N?(w{KcoC3MSakZ|S9>7w*T!M=^$6OTHPMLCi;df&OZ zpOSJ*Z1%3AD;LQ(RjQp}mqUfSIutL}tOf2^Qp9k*Ac4GOzS8??!dq^(+Mc~qV`6ox z84hGrx$ZSXOlqxtp^z8jV8j5m)sLThuK)vZ>xj?J;m+jHSD+sMDO*v7W5IdtrY)p| z#3*^YIk@86|G&}#ZeZy~nFcFSUzF+?Ew9=$Qs?9!#fG!wbC+GOJKyES+19-`we@~Q z07_vDr{mbTDbPCYZ#7D@jD8yLXLo# zMz*f(Z$6V(>1_|4@3QTI7+g@Yutz`(6Fym3EOY(J3n-sDYzaUe902NI#6ulJl#XW6 zycQPppIP;~jWXUg`8%9UV{+i|PUih-=KF+2;)IyU z0s#+XRqKr zhGbdQy=byUY8B280&~7;DynDJi&aB!B&vM2?*g$936OvodO-@Zz$+A5sG3?H(!Pv5 zQcX1)X-*5AT%6!3e8B%b#6Bh<1!g)=>?u&$ zABf?rYWQ~~8v}SpGLrBqvk;tw2oHYKM*!hrA`GvaTlWI~?^KOR@dtM{`04F|)EY-X z(MEdt0{>~q*%L&?!0}R)KjYhWTyY{4!ASmvfP?Gm>a}|}u;G*W+Ybur4Ab6!*2WJG zK6{&Z&Ryf4f5TJ7Ng`Yki!%MTLl%58dcHk{r~NwfY$0i_huy_~oYE26=l6!{#@@R) zl@^3}c1f4@24LbUA6hIX=5g`>`r3~G*@=xXZKlQV>T8zzwa41nmcLMn*W$d#hRKNs zEmyBgDtzsd%D9;@KY5FL!ToI8IfB$iFuqOLmdItb68@GX#W(o2_XtNEt-1RhY&_9) zyDf0_TwH~-UyIz@dk7g=q73RWA9(QW?EfV5`Jg~A<}c*#544;9|631+*;ly5C56mZ zf=kEhR7`|r9PW$@gp3mIfQtpzVwPuEHWxH z1&46Nx-5^~xzA*TG7da<+9i0YA=hi%V3Jj%;JsL<<)g)UtiLpky&#l{(^=o<5j_}s zq|3*PKecBYpGX<+ap!e6p|JG5X)s!9hOw;sw}OLSSD>NgmJ) z86q4U{2>~@G32-npd3gj;;HFoh;UMMMSuqh)LQ2x>OxxJa7YVmuUR)JJ?3lrBp&nmx8B;%< zLYbj?xTY`&mFCfeJOSbc>)<$7IObc_;UQ$bp76g93x%2Zo1mm9xau2l(qXsZO zUF)>p+ey+{C$sEN$?=W_6@h-an>sr*B8)yER2evsOJST7eOat1mRdWb;00I9w7I4+ zj&4zFcx{I&xpgl3fq%^VVYflACDn%>o4jh-&+6!MEdD@c>}c zIRn#sZ;$|we{TN!?MDUv`GHIeSAV1naE}1-R0tafH@FR`EyO7OipDE^>6OrB zTfzU!JmQu8X7(St(zYV-C3cO2lO7j)L4)og(7E>Atv5uzE$G-f{2KcF2>EdzZuQ)K z?{gK%`}|^0k+i#aVqcbX?6lf@L!QY?z?e8<;{~VEm{AL!taPM)ZsdE!F*LJ03zz9V zb?$Md+VY|yT@0N%3gwzm;u4`VfN0!Cbh1>DI}$o9fVg;HufS}!I@{G?=|<5qj$m7_ z$9X|772AJ{As-&{aW6 z64>8CR&P-z*ou8qz8X35QF^+a&xaJ)s4E$-UY#T5dQ2-fx#qXQ88GO=;a3(g!4v!v z*tMR|NUXDR19q(=P1Bio&3+1(%GZ}~#d+;sCdD+5SOgf~OL38dWi>AsvXQD{R%$`f z-z^>OC2&{>y5B;VTQ5H4Y|B*aj`fbuy)68=r|7(NgKROCJ2^Xaop!>#4of}Vn}U)E z!Xc&xM`-vkScE4)MGjgn7zuwlX)yA}&n}VWc?|?-XjBPQ zaB##KDLtbLh@7k;MW62VlKD6VT*HBp#+l;jrx%??gkPEyeQkY;y{DVqs+ZE!zGB`i zJ*m#7%B6Rjgx=!0mYJRD`EXKGT)9EB=6#PcDeGork*n{^C?*JQzGYSQLo~jASHqLl zZihoB+HQ(zh(WbZ1eclPT38*~f6uLSI4ifX6P@?<+|;+{tyrD-`(*5Skg@Mfu;y&e zF)p#Nl2@U?YQP_U7XBy_M>z*w~(1=Xep$Hqm`zVf#nM%f^Q^ zF{Sk(jyGiTfjQVp4?6h%?BP=xof$hj@3|r<^8)3(EN{D8T)z$D#|?1n>zEv2PSp8A zY4!0G#wS0d%*CX>Wb&30cJy;#KOH^@q9{nt?X=ZqtTt9;yu#6WYymD@FigG8qdVoy zYr)ao5W*7H3ZEXjhbRW1_1t{9xegR2R;;%;qiGe8&}Z3QsCdIZr(pN050Clp7dLC+-3Q>so%S9}pwQp+^47yr(gC?AJOyWy zBp3-65n~ZpGACR0YQ+;QoyrzSnfTLK!|;Z$bE+972He#B?r{7JbJhV`G2r0wppd8E_!TuwDtI^c9SB8tKpnErx`VmH+O8VTOj z_TvNTd*f2iUdH8v%0TS01M^b=6Aul%Mc|&!V2eTp!ILij$)ELMxqXNK(&|8shn==2 z&1mwOzm#<%ug^i5ypRtX?&b`G{)@`UvpXW8^z>R3IdP2lhWTh?;C$rahaf9*ZP%;<mfe~#c!hMVBGRauP2khPyE5s|FOF|vsYA$#x9!NJKm_DsplPPR}f zBOzqZtYjxy$p}SRWtYnD`98Wn)%A7#{y4YPRjymFUhmiY^?W`bkNe}{BxAG?jLWrw zMzd$UczZsE5DRJ{QYI$R%mK|F{<*{X3;}n6={KjuQs36ecM*}FH&z_Bxb_#4a}@Aa z9-^~oyA^Nn#G*vQoyXu#j$kEfSm#kmbsW^w;Bta6XgAsx#nE>u-($!2=2X>QDW*SE z)z+_ZxX%V#m1f&3K&Pl3m8T`0xglbTz-ddH68GHWg{VTi&y`JI+ICHjH>R}-=u2K^ zQRU+m|CKI1I-y*h;TyN^H^e? ztxX+WVJQCBiA8{P>3=_he}4-+Q`!yd#pB}X00T&5$o|(KCcFrLct^FB7j*NlxB{3O zwCO4s5Tw~cgCApuHHR-$pxiLkgdV>I=-jYgzZxN+wnARM_>CNK-5|NiaBbw7WU`2x zZR}vQ1a~?R6%LCY0E?*Nt-((D(?NVEE;y;B-wB@M7}q|(1R`;++58 znG4fjj*h!w{$&0<&VY!$6>;~-{?^um06ebv| z3o8kPk0FssN>T{Lu66n9h(vB{&`D_m(n*B^LYb=BH)IBV2T4en-pD5XXe^!GEg*Yj z|5*QQomm}MnCvMM-xt8K?i!pH8Jh~o1~h{hvHz1pC4D_iN|j`e=>=kKACJmy-KQ19 zJ{0cxx$r*tVO?Ip?RIvZ<#gcv>?ATJK;&!!{5w>>z`V5vk&%Tf*p56Ad4sDRxFVdb zxvy-GX#V68J>NayL?rU3JD-M?zvO6A zRloT_H7!s*=a9EWTXDR>LhR_rZ^nJHu9+GcvGaz*bs((nP&1oCz1J^;Gn~;uS@VVW zZqX5!)P>akHzj#_8BcSHbaX5n#{_%?UyrDnXM#Cub16=gt0@t(S-1x#bl(HU%wI() z@q2U|75!b_S28{PkszDcmCry0dh-<`1iA9hAr1tBy{Qp_aAy5y6$B$S8=M)=2{FAw zZkVMd2vvRGOI6TDl=F&2gD+nwC6`{+FLpGN&SL)U%q#iL`Hr_^5YGZas3ahk zp)TzbQSB9r6-&ANbeWHoDRYwew$5bv*QWjPeUve-_2DP)s7KPfmycR%q&A(J0QmDxSam{j*icLwL6-z3!j!(RGzAmL8kQ%`24>xcPvuX2#ptz50Dp zdi&Cg_9IV4^))Jd&6bcVh`wLsoSnM5>b0ks{mec&*6ef=99Hdzj%PPZ6Q856E@wz# z^~l-Usy;_yl6r7%?O?~#(D$%o{;n?nK6Koz0a<2^cHsHn7tT(gqyBT#{pXF|4w79i zI`e2^t?Z?7|0k>?^_9q|PKxhB^sGo}1`9YF4g>On!M7KGrGGdSF0j~reIOVKkt*lq zZwB%85#Iw-1Lk%UyERw7i8d-aEK5e_nq}Q0;w#D1d(uiRfA0`oEZx+eQBHN)*TeCP zVqb1%A36|osW-WOz96zJ{Om02rk+{iyHcz;b8=|{y?7*vXYWb3nrMol~6tj=r3rT7AZU?bq9J$FlMQwB}*73O7y%z=LWao9qwq$l!DAmBmBCr{;QC+|2N- zEgf*#l@q!*gUfCYSDgUv=XHE&f+kI42Px#OL6vi&&drsP{If{+LaTYeG;!YW)6nBk zSt>o?w2U^YXGt(?>NQD6&E#U{@v)O$4iTPOXqFx&)8M0#d)&Q)1zs9J5qT1${J^rGwH~cYRKgf!0*18 zWp3hzo<4}VQ6xZk4)$aZn9KElZsjmi{JcU6$y(tXf#t9$Ct#=Ts9n$mDX^!lyuKa} zQuv|Z6^XMz$zpBs;gE{RYbXern6_?2i$sw74ki%izC2VFk|6+F25HHs1Ts%7-Ku=p z9{pi+>+6Tfh2tdoRh?zv6+Y#eJemLoZb7O(V3VK7ZY4^J8`WFE5pI-V7BWNti$ z-BPBW>vntf)a`_74$n;%v0nKHpA=r`Si3?wh)@gqW8K|n-DQ&`9Ibe1kYVU z#@Tn{IfwQ&Z9*BX>A1;2-dyj<2AO5?HOdPBe9ku4si(_H(^{_f|RU`|*Hq<-76(~N@x z#2#}6*lJjet%V(s3!wOe(#;a0Mt?~qpwp~|-#kZjzr+hnmS4K$L7ZFZM;ETwbPkk&!nGDYgm4?+K(KmR5Wnrd&kd|pC}I%;d*|{&Z=+tS zqVQuVedUPzMA3zL3@%k!?EC`8n%cq1pX$>WDutd-RgZpmnjod7=pa;EO|bc5JPXMr z#)9FKP3)YS1xrEEY@u8;*$9E1^vKmOxKSu# zAq(;#ayC)u{<(ymEtk|tr4s|GVWKMV?2w1idPWA(26lfFU9{r)AG79(HSdo%PxFeu zp$L_tm)_4MkRv71C^>sirdsps8+NcEVr|<$)z;_KLeOZ^rI3S9IJVYi1&lx2#tUfl zA5L9NHo5OJQxNx@!<^36*tp`FDf1iIoHiiSVyNp-N9$6A{r#b~r0^)3fT`L@x~)`h~n-ZEi3gSd43WqN1CrY0bvj`Qa=+Nd|_fe1I*sqzB`x@hz&5$9WA**zG zO0a2cS!{tG{hXDQ`_;|KLi<^fP?r(yl2;D}KkYi5k*)Sx-GrVh-dN@fW@t5i#6zix z=HNYWVlAb9bX}b^=3dFFw~r-u{~){~aH}j_0@Jnb8%9iQDzYI*`I55HIJVAa>%_aV zBfjN6yY@KWMuahlOmatGqpgb-&KX)O_RuLXH$~9R_?O-PdZ?3xN6qR*i=v*>5ZyNC zqN&0MBF|hxZd7MRprZKs9WhOBXSL`-Rtw?h=mw!;A>+$zciU1fUm6;EI_zk9aM(Pg zDcpMS<`n1Pmq^BqFQ1>QN*(K=yKm?kE4B6{g|6$g=3eaSRkt@Eyk+$rP)vO^nJ71v zH?K19N6pn87`MFkzG#CJ&Z8X_5Y&m{<4`qi?(?|)c z@=D|oaRT#wce^$QhlmqM1mIi9MTE4wEyzWD`U>E>3ne!n!IFrBpwO-^)_}hPY4vtetv@f4RJ&2)JO7@2+}JW7r$-Zp72&~!NK3!U{EAS)}_m|_;oeUpFDcp#whJON4wBX`d+@!lMHWq@p)081#( z@DJ0rw5DhpO#-S9V0SrC``pLpRLEw*Amx#I)vM1_so$U`l>*1CLhi&FTytEy`}9SU zp5*Iq-jR$NKi$YU?C+Wjr7^(@GQ$e)*tT;3Jp+ssc{E)e9Auy*GF66IkQWYZiWE$+ zzd)h(&ut;XbYEf3Li~}r2bU&T;%^TEu|N&(3c-`_pkw^7-86aY3m${9F~e4NSlY7b z7HLLIqvvAV=o0~!GO@az1c?CV6UE)s?;52~)8GK?&n?O#OJzY&EGo@gPae)#?tv*e z{6?$N)05HzLmH1nucwY(?UZ+5OK1{K5jP9ljRT1D2Er6T0eN&)FQB9@ zXrX}fj6` zqZ0E9oT(0ze_3^=DXlTwXo)9sYmv6vy$Sbw@|&Rg^Lp$R*w6o2m0y9QzzGM22^}2R z22s#_#z2Vm_CX2HKAs5#CM5d}Jzr@1 z#O9%91ukmRKInTItJBsbBRgvT2HC>zkCcgQy2z5+8D5z?_5fJgEOV>8TKm;?GS6V- zI`0c*yaM9Qe1Afk(mR>(kAzffKr&M{Suc1yba}?gJ@e$Z=rO*RO7(myS{uBC_GCy^ zUT{vS$$|az*7I`|3`h1j{ve4d4x9UrOACUz6cpHDS6hLAS3rQr*&e5%azWk`oJ$s( z_?UDxEHpOoG|JRlzsKe6#hCjq?+B|t&vxfO&!Oyl+fDE3(A?<_mD;3{>{f=p$%Kx@ zP5wn;Ms`H*j82Y_u|ed{IJESA!V`|)}5HJ@A$^!61gI=l0Ft`OBg z-ye}ZzXjQ|7-DL=evPpIkdfV2?+x;Obw3CTmf3H5##qBPL;4##N_W7cV~?}|t%QYn zR5Vd%xcLR~dklFXFXBt9@YLPxk}|>taxt$hle~r)q}1q zwX6rI!3d5H;-m$+vf6edPbrb|!%X7qP#&CpaeWmD9LzX#X%xVoF~8jBh^yKyk`d8B#Fi`uK>Z z1$;!4A8lX(EDT~?*zrmSM2%HA7xG^+ui9e15f<52G0FSlL>kMo)*pv*bi8{QzSIFjNaxFSLs!D( z3Ygf+4plW;RShYMGAJ;XE7?bcXy5N*+M{Eso9#arq7)$*KNw#*Y;Kc0Dj*$bH}Yu> z;wJXTfaR-5+^*f}u4=I*<3(BhPb)8NB$PDlcn0t7$;EzBS4|xeOtrTzW}!0e-;^jk z-PpqyLZ|@)@ml^GTde1ci`bQSb2;?v*D|K66>nhOw3YZ@9y<@I=vy1RJ0jmPwTRahU5HEMT9o7oDy@-qL@aQ?JifKpa^RfA;Ld=}RV~ z%;fi09S-`>2h&}JyaMWVn8>IsRq;~iqxbzUdiBmM9=<03ZI~4y!Rk*YAo>*qUA*IC z2rFmfgtM^VQBl*9g*^u8$H-&&Tfj2k6IDom{9e0;_1eW!7Q?2}Z~+w#Cfykhua|Ab zfxyl2$0wP*o0WnFhw06e%oE8}&&D0!R_2iNbJd<}zD#}Mt~b@OwrnKhQf#BhAqTd%F=4IWDGA`(!C0u^$gn?K54Y`L~#WtcJk*;r2e z&5SyE>3&Co8n4UX%ZCQmW2;KzpI768;$1H09ZmhS@Ag7 zpq!lk<$FPnwjFWM<_>1I2yO?3a6)WBG-VGGgnz$4PXShD?s0v1{r}>|y;tIY#h>7j zdN3o$$BkP<-1tKzZk$i&Iqq^pxu2?G5>swdv8F=$AuCYgW`qql(tENdcEIIg=qL=a{`d6Qe2NiF8mvhnVb@#D4w1>aV@tDbM{soMk4NG!; z@mgT!Big6UjPK$T6fGfc{OEZYl25p|4j&$<5~S&z=+a`vefZvy6*45}(u?SYML{0d9bL~DJ1ZxE3`(J$U`>!DB4s3YXNRHDe=e;BXgn=2 zo`7n%My4C)R(7_|_z~xC$>(KTUX~5|Aze-HXYnUac;7>^jHYM!{zdF*rf8(;YS`Od z%|^savNr6ENuZ@)sPAE=@KCBPo;=G+9XJTRfN;(L>CBG)qWFwCg-0q1$vU)sx;X=> zI!`o?aq!2`cq)EH-lu8ZP6qm6LcD%BwA#+m>sGzB&)_44_GgSVt(v6a%-VQ*D58$lVU#uV7{teNr>&AW zZj!v63WKcZod!VvZKAng51ly~S73_rXu9A47Y5-c=J;3+*pwGxbZ&vjw2=472%;-F2<9ntJrB%7Hi9^Nk%|i?v$D)$_Jmt&3wf6kTO)Wvw>Nk@y?Lm5ZCo4{} zfy{;5O-k~M$(MX3)LKOBQZf*Z(Px)4v1W1_+Q; zVs}PfF~6|$*WEu$H{iku+`jGZ<4k%82O|aS)g4u6ZbOt0kezrZ^q_p{z1xbOvFg>uB!J%Xc&8YK^%`s zBEobxO_(;6HE>kaJjK`H_I;|n@q=~H{brRUhUOE26wgaEj!nLN)YS?|cemNolYxhX0syH_RHJIsPCOknRuir0EHT@ksZV46=$2FkFQA&a*iOA;UU6 zDLeyKh5xUW`TJbFg2Q-$Q`!qWM$UFnnEh+whqzqNitq5gHuu5-uF>zG4>t<}* zDj{Mp@ienE1yQcE5qM_y8@H`p5|Z89JCifgp89HCe{Zxzh%aP0gQYd!>xN(iQwwj^ zWAdC5g%HAkg{k`e2U6`XjtO7clM_dDB}}Yp#jSAO`qyPRJa!>pR8!d(;8UZAA48Df$?QKf5+`vV&_l4>*IBAB9iCs zeDjDwqQjxJw-?f4gzh(Y(BCFWr0@1NHK^MaT+WzQ|EAB5>LL>vyiQPshy2lPi87B= z22%>W^JDYVTwJ*iWAZBHWFivpHwCT7;#wd~-IJfGQ*!HY=~I_d9|?ny)t8YHg4vvX ztH;rbEOFR5-Zw&;->L0=JWa?>7q$|Gzl0a`{O6*U_g#%L;#K}_Q*ze?qb)-tJ7qGW zC=o6A%3gw6UaJ_f#H9b&QypmtOp0O8PDvnn=Akd?8Yn1$0LiF3o#au zZwD2a=>>bVttWm@{=$uHD_;gxl#CWD(XG1*0jv~)%5bz$^~-^6BtuVzY$N0Uv0ACq z8Gm$;I?E`YA$Y)ER-0ffBE`s7_eJ)Z^Gv_RrclK|g0y#{qZsR5y?P30%HJ#R_a1%rO^u^QgyR)*%xdAE6X(8Dg3ZGQ($<{x$;l%?+~R|Z5zIvNH=>xRw*-jL)quzJ8wZ$~?^ zx7-7~Yc$iB4<3x+5HE}Y+K!H~Bn$`f&l!>}G{^toY> zmIw%`pyOwabATqF6&{43c@gG-@C+gx5`4uUzQPTlN}zO*(ZJ{cyM1z&8emY)$i`qJ^B>xM%Q_kCaLO1q=D4k z)|QqF%Xko$@%P2>58Z+dv_;WZphh!8re0P!d%Q68f2}0eh^>hiUqmbAVk{`-Q*yEY@@#uQaUHHe&a*UlVFbt+@@H=Y3`THd^e(I+c@3GL=QPD z=-TdXkm9_*a$(T_wH#}zx=P3!8gu1!t^msP^GUvIM)q0A(-?^AF8q%6)lresvq{dc z*ICn%62B*u`g&Jfk%9H&;Y_ua!jW%n4HPQ!Y9s{C2{x;bzZ7M?S0R4I8_eG?sKz*fe%5Aaq{o(opr`2zB5MZ26sbeS92b<3rflt{_YKQhR;8Yu9;+&5dhtKy!yqjtlSihP%BgaV8 zJKiozc3zQSAB?1nj*)abWc}DmgORk@(WX^OU9?{zY&+urS`IIm>F}sv9h^;-wf?mn z9#Y3C$8JCpMP}ioo&pj~3@^ybt)9Ol7k(n6#3rh%BKv`ZD@sSe=wY;m%;-N*)U$PX zt`{AN6A7f(qsa|mbtA^INIp@^Pjd)M#lFlPDA7o@UwsfO+ty&gX5v(FJRVt%PS5c7 z5!ToOint>Gr^ezZ1hez59FwX^uyDn-M$fz4-*By6JyJqJ7Wrr|f(~AtuQPn*Qg%b* zToCAO={J;GBe;p#7C$APeMwJ%0ONH0-AUcwY%L;89Ony0=Ll6;0-Inl5YozyGGrLE zVtDMG`B7+`g&PWFj`*teH#X@x*@u?t7q!Qp`99vw?0qi}Aqvc>Zx;aobXW%bEdb*r z%UdC;DG7;caypAM?%Cw$TI*||AQ%0L?nc#xDa~<(_K9r2S{MO z;yollcPyJ_WIAsRiXWqY~1+W_%|Ce}0_rVXI-Bt$6RGwMGCdY6iVcJ2ttkbb)Ih+OZ zGwv_aYebPm@K9aP-8A{C*D#UPMo96^>+!XVqoo34t!hP@7NPj;3HTi>NjO&=#6wimLVS$WwC;Eq9$l8)#ny z3%3ssL-;$}y_19%-@FvBp+<$yjU?S$ly&FR*CX+g!^ut_zb`m6fg1mW7)r=1`~`Qr zPjA)1?C$|G`*U#&a)5Q8~uUI>XJHzlem|LOn6v{uEKGAG$W*_N?jtNlOC z>3Tu`W(=l{RG~?uRb%3C+@rV)xfUazGOr<8*!xZKWwE#sp~$kc!-5ADe=@t}*cA5; zUVdt`zUPzm7$--`f=YmjXqH`Y8^f7q#@Rd`Ts!(?si*Ie4^F6S#G|Dmqn8E;$=@ky z8_?Gc%M&#}>(?xiA$L1?55}O^i|Z*HN9Lxz5tA~Ut#lXZ{cB(;5cBqq8YnBACs_XN zfzd8=L0RY0zm8W3^SxC4xG9KOyop+wVz@9CkXjrF!Jg7V56}qi&$gYU*VY}WB}HZ} z!8ThhLwxyolug3=PlEot*6Bmap9`*gc~T{bRLU+wirG0v6 zisC?QpG3Npl9I)QV5Z?ym0rQ_;-;P~;dEM(rb@n44h;Y0kv^Y3og&=(u3~}M;nLwG zjaiv5*ow6bR}Ojs&12`KQhA2T8fK4W(jDCYV$y~oczll$rJQ;~X63BC8JgPDa%5fRO{s%CVt&jwo`9lO6k^{=x<9}CvDJO;60o$J-XAFc6pY^al7sI-OG@l; zsaA7Bo1tycpuE@yY;Hrx%uK=mVGBNPz@nJ`58?7>RogkJVqj+Ki#o8@FXBiMCQynO z8xoVXgdo#KKRn59YYXW@2Trlqo$0hpPZWGMg&nmW31&XaTD;-6Z=i*$bgzZO@o>Q! zLuyLCN(-mES08q8x-AV+_NJQ=-98bffATN|;uGM}kWu*8a~Iw#qFl4Et562She;y;L;UMGn88!0yBBY< z`T-WJr^){A*EDnvo>VL*=xQ$z@)T!$cY`fU(L?NPj#BE?mGs;$ExCq$rwX1OjHeUm z_Of!tM0BkmJI%Ft7-eRbek+%6WNlW&1jLK$?@tY4nPpo~f_$G0UV2sHMxz!#0 z>o5OD!C(iSVJxDy1b!cnxf2@Y;$jXm3L)4Rf6lyOr|P_>Xt`eVWF+1`2uCLO%mWG$><aPB=f=b@&K-oF&td|gov|$#}~Guw6q)q zOj-f%z!7;=3Xz#sQ1fFC*3O|zjz&+8)96*{CQMC03GliWu$WJty;w$+%Y>6WBko)4 zbx@wmK#S}_ftf{G46#F6O9raS!8c)kjrOFNf(u2U$mW#(VC5l+Gnj|B;|y9m6&|H( zl-lQ>!Q^#T_C#qI;CtSezPjmUevFS2$>M{acXOwK1hy&v)j@@m4+_utSOwkKYiRq) z#3m$>Bv-6M@HX3M2z$JR%2^SyH& z!;PD9U&l$xcspg&0l`|oJ+DvX6RD#)xlD1td7(s^bIVcxxeB+KS!F!!ET3rdCGIdH z99g=@gwau{c23>&=!~mM_~f=XW|buXlY&I4$TYV~Ta5O4rk0U*jYvF6#iytSolxP3 zA|K;nGQ&HkDs#O}z6pm69m)i#%`9q@bG16~V?Irgy$gGc9PVRyeKZc@05y;P7 z`tRoZX-Mmhum0J06E403y%g&iIq zW{L*-s1qDdunGUvv#QpbJiTZ;2m^a%F1)%4QDYuTXoOVeO9^m?-u(L670o@^ldtqN z+?R)N|aB&Iv!onINupStQ%kWG$I~RG<>wU9Xh2!UoIm5 zlk-@op>bPvj*a$XIl$ll^Qn|~>~#aL(wgnkjLJ%6y%L*VL(ddRh=lGhaea7d zxqu{*I5>zWSg5gf+Y;dNUAo(?BIY09Ach?4r+Zsff1U_WOfnZDh?({*ab;Zl+`v@P z%ddZvsWw52HB!=>TGswF-=V4o4y{g-RoOP(KAIZ-&eDa_yw_E7GX8q^I+|>_ha8RL zXMOG^POJUg`zTd5@WHX+`iU6Zmj<82i&LMP+msxCsFkA`IH6g+7U%PR5Km?lHO+yT z7D1Ho$=Gj0jK-;rFBoIr&fT_3Hah-lHJvwtL7po#ctY>|{JQ~9&!`1~Q zd}bR^ZbEc4TK#Rwn!h}Z<%pCm^L*r*7v>HWAS6<&W$O5v=d~FS7=104T~U6Nog+{x zZHSKYe^nZ9Y;#9pKqs!K-XJ}Xve|T)na4P(Nh;-Sc!N>?5H2oCqC?e{cd#{;BkJ0? z>i;6OeDC-pNBj>$UW!Id|1CKHLtk+|G$8)=e#8JQ4pNBhICiqdx?`|#l3DTCW38-R z;2yC3>Nni(#O7_3?LK#c(EIq;CnVpG#Ss(*%V@`3+J^vncivWPDc)0;+l$q&5!yr?~M9&BKqP_YxeXdxcQe} zhs@2ZVCdwvc$!jCMAYvLL1M*d4h)@8%H-U)p5L_6oo?h3xn+0^E`1-S@rFZ%w7l+E1Q@MLPY)T0CRpRGXJRC|j8Q`MM|+N3*|d&TzgrTmDrSii?fwxW+cbF1V81gT{XdZW}t!JtF`|Y!~?AI zPueLik5DOaH^n?zKC>jw%6K(YDdtIn0Ab$YV;djF?|I*VA(%-_+zq z2o)@bA$-(ScX*1Rm2VFta40;WnU737yfCI{#48T;gTK(7|Nf8M4lu|3Q$zWmeQry% z85RQwd>k;TTzJ3%vz;-8TsrWX)~$nGaCt178WoExReN-G;3F?r>?A6xIF5*y}`&5~+-!RakGUqm%`c)%t zAFRcGO1@0OQ!5`+ek3D#cSZG=bB9j>jE|A#g@!{o(R;Wugb{&1@9i8p5uee%?{#0I z0t}lkq|^~=eUCt@Ib=$?<0TbL`L+Pz;E|J6#$y40&v7km2bdzs)pX>IU83l{Z|f-} zzu1PiL!){(WCP=cqaToh?!X=MnZ^)Wp+l%vf|VOVC8&|J^)#71#NdUWCOSPR`tW?m zjGw1s;^Ec|kMUvpuh1|n=hl8;qdfxACf!OW6>Zw8M#LfwC5KoKI!iVlx+AwD{;nk7 zxGz27)#>|3bkcJIKN-!cx5-*Sw8^}wPCHtvNdMUxz?O~u#Z&1nwocD?%Td2lm5g1aV?;PHQI3#dAeG=qCL&Q zj*GIyQ(jR{=G#-G-_BQ1_C2yoW#05tXHoPur=y5#0O#xf*aeZ~KyzehfMHg7}FQx$!!;QqbTxYw zGo;&zT<`Q950&96kqjY3=-`dw-MNEWAiWE z{ZEM7&9q{?^cd`RKeooE+Yv?}l25|Coljz4I@d_*X4tS_GEmrwAEl&@KXdL^Nu`i# zlBlP@X>|+ska<1$aesT}o3d<^jMo~LY=!So@5j)0GB;}yk;(tmA%RnN!!E8qYHhbA zUbnr-VKcn@?T6Z`bZ-d?6UqZX2l9>`^QTL3DOpYk zCJcKT$5%;eba2NCr`MyIG3Qn(TkWMW7n!rrY_BG%FLnE0>oex33hn-~pzC+UkynY8 zV4rJ+&e%tH8v}-8YJf>k5(`cm*>*1GE=`9WwE*;o|>y<=M0G6tx(jsBql z)s$L)*X#7@>qIAa&^!p3eMUChyWI|Uas7W;wk>X7timf)JxlYYA%9bWUznYMCz=#)TQz= z1p{V7Kp>6v57)iQ4;bQ$+r_f--oWQHLbm9=XYt&@%{5XE6tJ=hyJFm3l^Cp{Z4H*g z=K6#Ro1a^{&@Aesfsvr)`Ny#J6C{(Sxar`!Z>>tZRu14XqvBq{v!82*Esnk1z6?}Sv~=euoLS^zht1-L2u zmsP)TQ}~dopG%GyzCb>%j)D;DPLj){0(R1NllvUl>Voa=E54YbcswXNb-v?fzL^}0 zE^lkbpkpEBH|64>_00Dr3);*Z2!9Jq!Ec-@R)^1?Tk^9>GyB?@oZq)!l(TQ>r-(e( zXZTd!;nkOOaq%h-7;`zPYMw;Zo_P`w!^}l=3*k+<$bAKL%fZ$R6Eg53`LuZ9C+J=@VI ztEsKA-5%Cf-+qYyD0N7M4t*xBTu|-rBOu|59xMvhQjb0DGB*5+%E;a+yy$~itl#;i z&RwwPsUMn-}08)}F35IBO_%9Gs!eCy3gPSO(oJq%tBF z9keY9(R<=h4o-lS`^6s2Q!_-apj5hFcrH~7Dc{+k|IXavHWBzba!y$dyw~)xn&V5i zzU`y9F8(HUD9N+R``pt+hlXcV(BIWbQI)@T{1|1_ch7@Ai)&@NiMaLstZnS}nrADM zQa_1Fn?0Bpwt73IxtsOzU=VG{qf#@Sn9~y!53?`4-oh%mKruBw0tc~MV%TyTSt-Z`+(V) z)H)M}6XG9xlRw-k7Nd>2W8*~sQJa0*Sn5@bjahQJ$*luFnZ49E3 zcW4kjR45{$n0N*UX8s|(vw!oKv%jH(lrALRmLTZ-pD3fNAD$bF`Ja^d9)pdIm79AQY{Lm z;{&xtPXb!c#1XQreAW1&P;HN`K78Lo>gw4n#3U`f7*#6uzX`mpGJFc zm{QVfjmN2TZk6Y<`(M0ytzCih(q8>TPyIIjJ?Nub8M|2rky;{LL;F02CzkH{IVOG&Y@?FB)*?@Nv3uK|j>oU0 zoE0(ifSAV0a};Ud!r3HoT?mVXWkLEdJ2vigm1R&?_`p0VEnSqX?BDmUd0GyqPuBVh zZbQUHTtz1pcd?&&C2~K&=)T;HJ~AX%N?`oZxrJy38qD@{(cI-hI`p6VLzCG;Ur?wyod2KUckge`_#3U+*t z**Uvx13i^pHSs!%KdWAq6vnX|M@E>lk_O{%vlX@Efxhx`x81f1Y5O&%%BWjPg8LDT z{eVE7th$W#fa$Q_~G2>-=Up4o|2QtiQ zq5~;br*7Y;TAq*n@HH*UGR(g7x}J7~&$+!AgD%#7*Mau@J5w*qj=O2|a|)L3*N)?; zv|X<~%NL#@ zbrgZ8m|iYZRc!Mwf%P~A>mek#!|`eX1S4m#i10wzodY1~EHI`ZHgR?~g$UH`cnoBe z@Eg-PS^ruDGLCS{1m-O4Cup*eQ9|&iR(!_M{8AL`n(L!?CG)vHj5mg0OAlqF6JEWC z5ie$A0Cif%<)ESrfh6mZsR6O1{qLA^Z*b?Lr`2}xjR=CS-b>!0-u>#^eXE=%E-iwL z2HfJT@A;qj>1Hk!$5o2sz%9+Lgb7(CrHj3I!@KJ4;mr!hICbeUTARSzI{Pcioah7T zPG3xLS$}-vG0b|8fp&Z}c~Pe~ejwN~v6G@JALW=S++pu1ddO?w`5d99FB!b^2Y8-= z-TcgT_bsM(WvT>jo4d2lCtQH_`cKR`j96@NZXi?wm68Pp<6sJg5K}v6#B}tB+O&bj zY+!Ku1p_V3GtS6p2{moVqkgaw?g(<6B7KN>`|i06S+QuQ%GT5*G=EPr>9Z;8(aIun zhU$n#^g8+(F-cQNM0uH{r%u5Lt6T@{Coc4J8mhJd}?m^E^mb{(AD1c9wT1-DuR3vbF(P&d1o>Qu2`vLPnTS$pLRuN&Y=i z7RvxNh4zvwwygZ@#^7MLeao4^Xi3{9+6T1@ac3xbraYeU+`H0Z6!m@p5& zf%)F{%&TdwQ6jc}-NFmI@E}NNuFOZ<>O>ob-VW0U&C8BZ%Z!h+w|G1&*L83@DP}MB z(%1C@(#9jIB3Y6av}^(PmVHW4KHV*>yioSjyy5y$ z0=KUu&vxBX`j1P`5m?0ESM@)dbO1oNhf5?v55a-LNdpZl$1j7N;lDLblhIRmUl>g9 z&e%Mo!&dt2A79%AuQtveb zsiqMU`(aQhW2PEC<7 z;#v6KFa5l|)deTOW#EwpZfCeb5B_J>h?J!}9*B@eXl?@*aW=4<8Zp=W(I{(_rx;2OE}=Wl&V>V`&CZzUy1O32r{=R@o7J3V+n)IRtrotZ5x z+*O{9b{}_BP*}t`PxY<$Dobacl(}m652Pq|QBv;1hI}kFef}kvJ>|h_YtWI5M=~Q= z_3Tjdkrj!f!?GhTyCUqhv;~b~Nx%-xdv_V^&}%`wN6xcFWzBhOR|R}Aqb0E$xw16l z?bUcWcVDCfZv{Eg_}o3{68>C00E5=%#JViR%F0e4?z78;Cw3E_dpcOR9f4D37;tNW zufrOM;uhemfiw{e{M+D7N1%OxQ8Ydyqhato!6yBXoIkCtu5YxR268f1tpp*)!6N+# zSv*0aeE>$$;f6_OLK!Gdg2~$>!5W*yBnD#+j*kULqwYxr$xolP)F}uH{CLQRmTD*wpsI;VV*?({_7dA>1~&8xy8r|o;b%{2-{Xd17#TuUJt zBES8OS^L;&?qMCWU34bS#)8T^8Bz0T31J^F_X8>j&bR{BRA5@7gItED*!3vqgS)T9 zNs6e9q@***NgSL{jLYrm{!T#3>T%gaaz%CD7G{szCW&uzlsYUAl54bMjxq97J7FNx z4Oo!~G|>DHl*1o;^S?8!ATZO@8O*})2128dLICK_7!<-W{ObYu4Kn1h!kNlDo2r@e z3GoW@igLSnxNuu}{q-dL%O~~U{Q{VCqW{$`I1PW(9X;T#z>Y%V8{kT=jeMzw!`iz@ z+u~d;bg&qAq!;rCx`MS+r0@8MWva7Q8yq9wC_OPFxXzfb1J12-J0X6cpfb zqByxK-e$IR!Bf6Huc)f3-Q{kEWhgF^LN)iC-Rh`V2Y0e=nAHlGIXfK}pLQ%Qe=_d~ zho}BR`P+RzJ8ZP&S#zp~bn~nkMRilyPutL7@UH8*mW&JW{S5L5fakAx;%oxc}i96hHDRzd@FAUMf}7{^Z|5Bk)ce; zt?yI1=m7J`Ap)hek^TXINtUg%u3QUZH<;kBpt3T!h#$F=6jJOlGrSy|?WQrWR`+7h z12KQ)AKBtfYbFZJm>Xv(aTRYPhtHaRh%SNOwNJiBaiY1wjD(yl~2`lsuO(n4z1`7~gM50s5 z`xrqO-nX@N<2uXRPX{NyADm(g1EmaJEWcwhBaOK90H^8#ofdO62JCgvME>1Tr|~=J zafi8h^9nDK-mYH>!xB>629Pie{%$-v9x55gZ+KuBc$jjT`rxw}k2h6szY(iQa{mmB zM!Qn)yNI8&%VN^yG6Uj@&N4ide6gmjlgX*v_aWy|dpbT%CNYs>E%92gPa1AL)G+00 z=;32F8T&`f2FQx&3i_c2`iCH0F!(QCd@ccbp|nCccj z6lv1e<2& zsx!ZP8)RXttPEb!IECNpzbE*kHZGzl=TmrI-29{bsy3QAy8*!f?W&B=14XOvL%xcL zXP)&9s6_&Iq6Ae`xvA!7c=DCrHQl8eNfb;Z(G3zIKrtKw(MAsgNy}#eyd$Mv?9A-FE?l_mEvqG~>^-7kRAgjlWo3qBXJk{f5TWvWzAt^gr*nS#!>My_x6>c5 zUhmiI^?W`bkNd;T-NO=4T2?3+Rv>I!6xPZL-lT3Q$dHFzQa}{#ggl`@^oViu22>FQ z^jV;+0fh@)n3|}1IJ-!KSQuE?&ZMw*4&-$li4b*@cl=P&^EOj)G#vMq^UEvCJ(={A z4KD}K8|BxoF_J}|IR0=k%^FfN-QGy8LM#nRC8_D^HaU?TWq11YR#=b9!XdeIoA+H8 z^gczaGBJo4+zBElOHVy&K7S#qqrsc>{RNdvJtxN-Zvl&PAqJ8(>(8HhRUWSu(?O-% znfuheray;cP$fgCKQ_*weF+JzxnP(0JG926@s;C9n{isl@x{0HYEz1*JyXw?E0(I| zr+V-7beDob9T%UiBP_8-<{vx2EwYyr)9bxW6VkA63EuG-60ue-DAvD9y z8TlK4y{d+PNof~MIS|rXh_9WiEZ9$7M{L%?@J0t7Qis2}?mVPkO)0F@BHD$yn`s}r zNDM>#1nzbxBxIoXm%B7j1&Dp6_>|G{m0swD`75SPKY8<$q9Zn;$wh5L2X86J66?=C zY?Iy6P-M1a#y&7mfB!-MAhXPA**)jYH`5|KY9{IZjEvr3Dw7SCoTbij1hVvrnGL#2 zOW0_KKKo>l7ayZy-V^*{1GH-Pm!kF%@I!c?;rnj~7mZN|8_m1*WQ2QY_Zn&N-#mHv zgca%da?JLZ?~T%5&W#yIx1?<+{dhs-?D=!Qd40?pmWzyJfWklLUz!$qK`DzN?Os*6#$GCr})%J#3LOUvKJ#bNAkLEz9^A z(I#d`X?@!ZtNW~S(S$=g&WiOxGir7ilREEfvYL_|H5fHcYNYodxAOhjmMp=Ci684^ zI`g7#ChO4ju5)Q#G5PGj(N0H7>LGrS0(oqnWi-6Y=5n3m`g$u{nEN@5bql{|MG$=v8{-A-gDbBt{b6cL^$kCIG?ttA@K` zHve=bM##Fontp?dzH^3TS|-^ewrQ-@2*`NXGFn6R#!EMS!w@pw5sv(OC9!F@e}9d; zCVyKxgSG(xSk`D6Z8=^(J|5G5qW_SMhA#r~N}|po(v1}wmpyvA6fK@yh+GC4ML5%S z1w-dOhFd;m-G2qika{Jx+Et+ZEYwDcxXwFe?Li;rhD)PNu_^t97c~btHh4F%_oDsu z^y`7cW3G7IUWV_Slq(0R5#6mTR7U-!Q}~eF-MM@>1u;9`;3@n zDD1ZE4s>U#gp)XQfX>JWpTA7`>>OHwFpPWTTiyUB$T}=_g>AYdt;bhn{|kE4^h>fr ziAJoqwKH4niO1fQjj2DgOVw++f1uC3eNoijy1b>pRQOptSk?4rO1OH_1@{{6Ht{l=ei)4*R(^E9>NT<|>epz0{ ziTwK-iFM22&@N*T?*{Nrr8YY+Z*V4oTubjj;#>6VUY_3bnPXE2_@za}-jSlO9x6Hi z0)Vl`u3T|T#4vN^(#EiOv3o6lxvjyJ<6|nW&AzzCOVp&Ss9c@(!>-)Bb}C*f#c$x) z$J%E$hIy&k&H4NuS*1d02|lkYn}1zzUJEZxe)-7xTSqj%9*HI?V&{5!xu6`?QEnLI;Qyn7#&;?Hai{!IW!ZT{W}by5GMTkR16>OX zsd~`vK{Pr$;$3{A_swg7;2{u`TPtOfQ{~TT=uqk&U##+O)wi&{R89CTj!9I% zjaAbfKs*~=bXI3t4-oJV8h+(mseSog^58@IpC>ygeKxO$Fw0%$F*`X`hxul-%ori@ zb&vR|uLlC>MYk{gGWF;JqH14+DQrn((!Xoi$R#%$5b$Eq3Z59%9f`_6nZXusB-Aa_ z?i=%YplyQ7_KG}X>LCTo+ybG`+p{)@E$u{|zY~Zr`+g>4JR!JG;(KgSd+4j4JtQWh zZ|26_Z#`P5HawWF`?mP8a-O)4dRc4Vjaw92=ApDtg|RCdHh02N9<_otC0AuiLPA5O zikeDB(9>5s*itf5-<;Hr;#GN>6834a(yF&;(%ARZSgH0{`OEybg1Vt*6tJs6Rq@;Op!#p13MC84cSKe z(9>4h7<612V@AR3(UO)mnk>d{?8qaUU{%nge0)4oTkc_bvWudTjLcUlvV+g$@vC{pzSad1WW$t^;(jIJqv_fA~RCB$~dAavgp2^0} z+#~lRIS@pakoW@(YhUgdNADz%h@vBkbBZaGZD^zTEB#QsV2VKOknLNUwaly>pM9U|Q83vw zPMF091R=Q7qyWaBl6sR#i8h3C-Hsn`f2l@qCie(V)@^_DFHuxUS^xw#l5+OfBgCSnykb zy&az%t+wXnoW;y1G&x#Z<{xCfdO!b`70s~}%6sG@20gemIOBh$H|r}~(YQ$U0Uwt9p@~XEU zN|XB{Jak_cG%YeDiXOXd|}}#A>)HLb&e$6*3b)ubeQMzqwtEn8!~x zeLH-GH3a(7_~UKYcdi=LoKVMO?3JOVtSM`%BBx<0C#j;KtRRbMZlbzv11=tIexJKk z+B|GLD6>8wz30YW#onqg&%S*bQxP02GgO=B{c9X;4xbZ(*O`^O;ToRrrH7%AJIh17f6CCHDa%M zJ*SRqgAu#(ZqewCUJM{*zFO32RtAc5t%QL{UIw| z1=mPYz|A5N94U5EuM3mi6(U9Fzsyx~Rz`k_k3-#2m|eKSzfn$CQCQ`Pfqv-+WFxWi zeoa|+c%j;hVzhxc4X3?lrK)nV+fyILE7S7QBw4l+9`kCb8=tf*{KWTies&=7#2n8~ zBgyEFY52OJs@GX)Bt_;cKbWVG6|P)J{@qAILhX3Hsp%TWHn#^s=lfU?LVv#G-a#%( zWIDcURc`O$iZ!>!p$&B1!B6h?5AzycSN|7KJbhB>*PX6o?Cunn`Z4<=u%P-q^ju2y zjkL}Klx6a#xq6_Rq>gk91_pu#I?0lvC*{ob6%b!OX*@KCN~!n0axLR=LP^j}TW}!N z($Y$+rFNluben*Dgs8oj9%GIHaa?~bW!~e|PgLrstyrTUqGpXDGr|~c3rDrJp2e#= zUk6UkTQ=_2EtMzeKc_w`_N6;oYUA4l23WV6t=TyJyChHOF7kRq2Do+v6y+ZtKW4io zge6W8*Aj>gO^@S;WV^l(50Dn*H3AXoFRv%I6QC#`p&R-}TO1i40t>_d3$*KfW@Qe6 z#Q1VZ!4cv;k>VSSi3ql8=LHJ#RIrZ~=2`GWHO=kUcOUc@ENh`|+rM{O3n`0u#4t!% zB(#I_M&vnCt^?jyL&}uJlcVQT%4-=K+86Nzqia zHwvjQF{P&2c!v8pWU9B@Xe$kSYDgT$++tT|y$^*=&Vml)zXo3a{8z+k?ueGvkQV0S{rwBK8xkY&M|D5|hS5Jz z`2Y5!#e9z^xjgC&b_;G4q# zRR3W?92A1>`7+w(TU_x8HO&@*Lu&`Vg*-mSA3O8~Zj};r2kPFdgt@492ZseLQ=XOO zWO?4LtXzD0UT=hKNa##tj$_~b*H*XRl^L1}bzoS8^y?1#gLPyR?Q8UjHOa>MTK3t% zP*au6uDhMdnBw0 zm_#I}4=A3lSPWh&qXHy=qe!ha66HcCddOsrxUeAT%B8SnHead0qq++UKCLvR@p^A& z**a4<#{B@+@{-PPo(&`0T`hC?wfCn%>G3Y+>K^xop37Bb>x~x)J?Re0M~4=JQ$#S? zxi!v)C4Tf$vHFO=x?Y~cljTaCteyh!f8&itxU~w>MzPBkO;=UzWU^Q|PFrWXcw}|7 zt1&-uOnKOohf8yFPzlacqBKp|s_6q&47Da0zG}|55veC*&{rJ(u zNzwNC*M!^*#Bf?}9JnY?LbiWfVgs@(<^b%&^Ef>CFOQSIUM(1V>=kRAIVAp~AtB5T z3t@Nvy;`OORQ*WTT2l%8{Xya>3pROK8_1JWAQI{B zwd!D^tADlnsagedLm*lAtZ58;QyW{il&%L8X%I(-;DBqetL~&`mOHbc5Vn_-W8^n0 z5db$W4(SEnyml#4znIrNt`+*q*hRn1{~RQpSD9VpARl9Z&7-M$CLa%a#sYtDbFi(& z9-w~Eoj5f(?R{q@{7z`&%w?@?p5@d~0rgdTx_GAZJh{Qh46vJlj(7Cw_O&tSlo$k5 z_z@Yihc3cKx&%LB7yfr9{QKJkV}P3}I4E6V3~p+Pg@idM3S8X~!+@KqEyfcAdIMK@ z`u_Q?Bi)2uH>b3QnyV8`w15A~{~Pc2um2n0e%_S?4EYZ3HaLtM%t@@FZ-bbiOd)p? z8ie>3v=VaJ1-;iU$(xQ|swi`hSn3pE5UWG)Ty>XwiIA&qnCn^+5|{yQpQpSPg$J20 ztR#?JKfJX5;6efatJWo=?BwI}6ne^|(+&Q_8SG6gnRN{Ufz;6)Y+6Mc2cD4ToO|-N z@69Fcv;9IvKJogkOA2AtkNoV`ag~cm*f$-s`y;vBJo_gO?vGj?oF|n(eIfT*mZXwO zXHX~DbFtyd+N2cL0QJIRC!TX~fuNj#_^RKRQHr`_ zjYKb3O0TJYM~i{wM~y6c20ngd7P;#iNd;@EAgzqj(MBs{k!TjMLm?aaAKL(AFcA8t zYG=9k-*cF)Gvhk;zsapE+3(5iPF(^az$^9o!u$XxV6i1wv@u$johSPe^0U!1@kMPN zX~xP5g+aoz3?x3_?|>=8^_wo-a0 zx%|EHLY3jK;n5FIYV~_$QRZmo*}o;`R2P2uX+0!I7yA(6e^;{tIq0I4Y0Q$u(~3>Q z%LSEn&$7B_h?*b!dGRyrL`Z$~GrjL-^<1QTe>gj*UeG&JVSv}?I+pJD{I&nVR4^i9 z5{93D8d3k=&bnAfBr((xh!_qaBLs@98!|QfWB2|Ydkzp<0G|W$$sFw>qbY^b+yQF* zbF7}3en()UQ$x6vt6)xk=X!t`AyX5T&uxYhIR};mt&~puxe8k^K5)VJnpwgT8O2pHJ>_)@V=t#CQSDodGGb*(te&ZiGj9y1I)7Q)?m?-~ z?5Hck2z6_(8u6z4ym4KVMoPRcIDq1#;qf&YO z+H(x4uYA(sQ^m0Fa=-Gne3jg1A^JTQ65ZaM%%C+|y;YMdwM58<$NQ=>_{e{G4e9St z%?!{CeWd;NFbFZY{?(g^bA}*S1gejQ;RYy@Q0{Q&AQ-tn-1%nBtQWtWD=J_$D`crl z*m38dc+c{Gc)icoh&2(xm#f%dlBM4e6kAl?%dIb%korQKTEB;)_k*HPi6S@?l_CNY za*i@TYb%jrQHsmbVAHC6FE|iyn{}SS^q8P|kcQB(W^LLU_W5GCRFU}BL0dc74}La_ zWhWH|(&`&UFn0t=`y#|VH3!WugKo~by8~6McLZPOIlcTGS1!g)=EhDsV5}Vfqv%aw;&f>Jo6x~Oa?h`fbtr1bR=g4E;dVWxQY|mk{(h9%KuIrQWpL%NM zKhFJZ@^da0<$1}~Lqk=!)SCUE?tr`$jU3X-3-6PDjlQre6n9o zOZ1s`kox=xu+47zVKK8v)^IDUiS~J`L9D#3D zZ)X?CkGA`ya`+gdeK(9qeWjDgPHEy(gzq-EVsA(qdI?}xDQe7k2KLH}q`nD0>Fj~e zgRj8|?bU7BezB7WpC&=Dbu|b^WQ3r-*?r{d!vj~Xq?{%S?=uJNF3Jms1V(}G(h+YZ z_y-Vj#zHhtrP`u#pIKycXsjMyt$G%2-@u9;wW?o=_dBS3`49JyUBv`J?g<}3z$2Gp zZCb1E6PO$GiG$=P?y9yuT26|brL!tmGv6oreJ1oI6Y&7k^QWhuKUK=_FsN2=^bC)# z_R;Wt(8LS|GG)bEC0Rd4oO<0~9?r*+=m+DlTdMjdoHul;l^`}TP@{_`0;X)vA%|(h+h*qls@Op-81f_J3DKK5KVl5IS<(< zhCTb`oi%5!&O@)si@+?=$=o4^`Y$gQMC&CY{J)F&kCr-^QDCHqv4rO`cy-)O5rU(o zDaZ&Ql^H)>GghKSLyd75)LoJRNrzxFUT=pslY zt`5%5WZ&-jvB6LjZ`foV3G|UI37RkVTIF6HCTbQ`oY&UE0}$vL6jjU z)e8yRuzIc>N=%A9)yrK=1{vyYWjkMAH4IIZ}#9Wd%GfskBZ`1X|SUz1Ab zRj;OfP_uZH=OIOm^0jkF<=E6i`+Hj=XZTJkr`>Yd%Z>K^77~FOxo@W4(`Td_k=)A2 zO@Q!VLgaQVmI#ZOP@LEf2f^1_NaZ|YdgL}D?}e@uft>6vdrAdoZ--Ha$)UTsJ;nnq zCy7$fF#YX#`d8(rAuB1PBFpXY?|w3Y68yot#>HXvFc3y;k5zEA0z;`e+QQz$9Fg?@ zE5Hk)rr~|}pQ{+Da{jr=|JeU9J@N+G9$eqf2>r{#1F~vAZn)}q)UZH^(!`YEK%C3GVFpzhS}#Dd%}JzZ<;VR}i@d<4a_*s;U_i8|4jcRVHz z63LY{eu(n_VWu7utCA4-o|yNvX0x5XF6|THBH!Gz83q|2iUz|+Kj;DxCK(=r*%w6f zy2e(HNgMc?Fjv+lN@qNYp)d0ljBwy*&wJ8i*xx4*9;-oP7tc|93vmPh=4x)oiR3Yy z*CY1bjPAJaL%xmBZfe{Eo0HaO{qGV61R}`}Q$#!zjR6fSY1hvD(4}wsNPA2fKTksF zc)J$F-dEf_(8}%_+_v~m>pyg{*efcahs>I|}1MWHqd$IVs+X6ZjSFmyR_G*UQhECMWWwT+_K;ibbW~*EMWWPoAkS8?C_3R7jbf=T=uPdcUg<=Fh7>II|9LsT`?O!~8U{t`C zt>e0>H`sdan3J;zt2I@)PcFeS&FrkI;M#eM;iF?BW>j|;Rg)1RQ zJGuey-|IOxs{eofy`OsnF*l*_j4z3I-7JPS7VWMGceXXc9Rpxl&ELl3&&znmQF&_K0_ov34xB=# zhW99PryhCdxcAyIsq&hHB1e(=aGx271G%q!$&S0+F)iCJ-KEU}%HR*E9Frr`)^XZ8 z)PbskAI(NJgH;uKJc;8MiDel(N()H1Ce03Y5Lj;dJ-Lc9859^xGUKxld3#=7{&Uij zjpqG_>Mb*bi3pUiqJ_SirDFKkc98`6Cy)K$zrnVK%I*H9`H%Gd-_@BN&eh!86X)iP z_O!J11l9V#ml>HZ?P@S|ck}VKgp5`>Cuz`Cu7u|3@HA@9!H%#|Cr$Ew9D{(jRZkUc?Xa8m1WO$EPIlbJQ>_k;Jnw*h60fRlQ{;i|FF7hsxXF0yHGI-YwYw|%${fO0Tpb}K z1#ts-{g1-*k9^yVq+-8c#3#;x7WkMu7|3PS^P`kjAMQj+Azry{M>Gf-Gg8NzDGfuU zGzf|_u#CBFjAlPpsmonu1CFWYuHV>^dVZ)hlcTUInmmGj&TjX$E08>ZT&2ICllg+f&oM@kGl*(=8d#0=;%mkxA2dSW*#qK66Vq)cn9 zL8D(yUe8=1=;Kp*kP>&-B~|pW$6-M6XalM~CwI61-K>8o z;(4h3ZYhq*jOh}cEMz8U_PvDgEzmu85F|r~J94#dZ~1Lj9%LMyiD)?3Q7uEnbFR$p z9@#Gf^y1PNs^NwK--wN?$CE{XGMc1lZ62%~%7xW;uq@~3OXUi;S2mstf0|q7p8z22 z`>m~Z$}gtRzmu9emT2SOeoU-%>~)k5R#@X5``Bl-+y^I1ZKG|CNyKtg`yt>GCQ!$- zr!W3b7EklpF6pxt5$c}qSEiRdzY z8j6hO5QF@#!(B@x3)mI3dP309`|Xd%_nh#;o`x$VfYo0Rr>Ub>+8ZU7`2^gnV0qrD z`%^Ec2(Q#2uP{Bp@gCcu@_`rJ%lk#TS>cH?Ne=xRkyl+#Q_Kq#@<8I}GnJ$mVzQ!h zW-CE&TASm}qwTc&Ugj$6v$FDiW$67>(Y+#uE_sPlvEFMtyr!Wod#6+Fv>^SOfmcpr zQ5kb-B`4hE#T6C zlj;99jG4Vf$X9$nB#fo5l>zU{3YdvQ{HlD$;rKYz&*dZdfm||%q6gZPD*_)M<&Yho zuo|qFJHgAcaN5i~-I&#=tX&-ZzViaaEG8N#yYb+3>tMhu*%^nyaN^o6E8107xrw z{gn|+zS~+D=GkZ=3aYzI5tDg2%(JM6;=EztLeaeggc5YsOJrT-7j{@*x-V}Se;%d^ zh3gQn2HSNS=LSVS=uxn4R)}W}{7n|_D0ulpVes$wm|0kW#WaLe9^tvp67n17nD~Al ze%loLy{W$EgtUuJ%SdbhCfsT~hA8Pmx3BIuOdzF>`3nsiY3frBLvGJcqlb2k0Y=6r z%r9o&y`LGA)rWPs{H>njBWX@ivp|l2{2PU2mC)#Qd`8iy%`l+FbsbBP6Kz!$4j!QM z9TRNPDGw-p6BZKAn_`gBs^R&y#q)EAQgJH9OlAfs*uc)k07j(q<6d)k#)q=dnN@dmBwjKx}d z%Om#PKQFW}=@c{5>#~X1$m}jKZs9L9FmC-1S_G*&KoNi#lZW6zm0-`6i$14+y}qMj zN%6#5A=3-0n;&0|rkUP6+tvV3{oN?8S%8qmevsjyK0fcGc%5FCA8q zWScr_tsaV+sgpVqPaUelYf~648Ydd+bR?&YQ&TDKC(nh`XPc0~6Sv{gMFHb)%1Ud` zhC+^2soJ6rY`GVfy|c@#`P5@(5o*bK?@7OdgeRD|W>a)YQByWa?SyCg0Mt?DZQASJ z8}g<;xU=t^fw(%~cxK&zpE<74hf|N&n~4k|=jFRWnaA{tbWy;;TSti}qUr;w1 zsL%%aYO7REDav2U@|k7lv5R+B%tdT8T&|lhyQ*BJAJ>%_yTbnNq;DV2naH2LAbc&M z@3H2*yxN_8qyQYmkY=z}B;%c2pS1VGv^)3ltqE8c5OM83DS>I9iw6eWH!2WpWsgu( z@vWdg-i=|Jht1#JIAR{j8fsyvIBgbr2T_+3lOkM~qNO>d6{}x(-(S+r!?`>S{7%CJ z6?L^FAF0WX2K~@s*S~Vx#rWpR4=x%Pl5>STmB%QaX$U9!m_uTD8Ccj1RkGqjdZS{bg^S z3a5EEAuS&Lr43J`&#zMv^cS{rXMzgzW=Mx5c$W80oHUTY`}5Mp^SP# z@a0og-J%P@^=JXN0*ags$<_cZwx0Ld{fhn4qL1@^XLpJmU+(hWTCVl#sRVznYkC&G z0xi=vw>~ZjRMdw&_#9YYMyrj^YJ#jQRnSy?9M|2S;OwbYc8t%zM*GEaCtm{wWuB9a z-1%KD551JTswfXy>$V6GIU*gkFJYpuA|^0f+a1VMjv59; z_?G5Qu0GKGSB5Dmgdd{hF&@@f6od^!#4zfz6$+v?AvMw)r7W+7S3IEMH;A)K8Jr{- zR|v7UN38d*|0y{C`tWzNgEt3^N71ehURZdn{;oJ}n|Mg%-fLGo&X%qI){65tO59NQt_61X7yr(ckL|I zx`4w5DMe{)&6`odE{|}VCXut@8akz#7Guy@cRcFTO6k7@{8bF;9Tw2WB7ltkE22|= zj{wZm(3Sukxn`TFr0Mjsv+B?8{V1e5odgo*qg+b?~*D<_zZYU>LE@o0i=mjvAJ~5J2~e zaqN`#=D7cs||c*&lf`8^I3epB;<1QaQe#G`46w2 z&uO@6_Z?tK-SqChz#APkkmOk9LAQQkn*6}MaUS2Nw`@2~Ts1ia8rt6u&4mP)X?Q$> zG{>h@?J_;{MeAAq9LfniM=oZA%vN{G;mCB_!TtOdoyU8;R&Ugk3W+p~9~WP}ekTCw zOv%vuDE$-N1z4q~f?#NX|GjVL;SLxA3pX^*$qi*`iTnd%qb+{-!2fsfek2TgHVMhb z*Pn&Y>^FlnLs$7JUPA?k;iDOK8%|sE z5L;Se?H<4`D=rb%us7TFMmz=P8`C&Ne75CntNP4Ew`I7TOZpD`QeLxo0e+ULsVM z1`PhlrxrYU3*jKVUyhyY6h_QY++1fQHsU#iC|*7lxp$#+#4e*T^u@4cMrXPN%c|Ya zyF6XrcyF;3-%j&iYtl014Y2IEdE7uf;PE~>j24RhoRG1wVpJpn@qM1;s4^p~u`(`S zBuD49s-blqU6|u6q1qiu#f7lNFcA`;1>Z-QLRIqz3(TMVXJ3=6CKS?LS2oV?FX-$@ zNT6j__F1zYnSoT^*ffXbTd@vODI#*+`?e>)8%s>YvRq|$xwgpIw$0f+{EgP>YotCb z5+i(AySBzvWmUDMa85h6L@ZicO3e+W2G~Soi6jv{jVz+4`4~OZJ74$xoru&`4v8~P@K@wPIV&0NX%g%QU42jZS?$$mEP*8 zz-f>=N*4xUmzEwfXUjH|#Hduv#LP?0ng2X1WN^0A`@xKy;eDD<*=H^}-jz1=PPXR6dN}fmbwvD*Q-G?RJqndb&BGWwf+^ zTgJ@ooj`1ehcLsX4WcARQ@Cj@O#$)EEBs&T4MLyT^&Zf7a`Se(q>hFdC<_~iGJTsdTn88QBC56hG&GHzHV|cSHjT^tEvO6a`dL}&Nvd$FTR^^98BJruqx+!X|HIg zMNq8d$$0i7gfvcbh`%7%_r_CjwrsynVFm(jxl%8u$eOW7*7zaUxmxoA8m))9g|iwH z%nzkrsVT>;B{@MZXufwA%HZBM&x|ozmf2uMK){ElsyRwVI%DSNnP$(6Pwd9}k`*6; zw~=kmnJ09a!2AOkEX2mw3r8!|UJuZ#Q2mS(XQ#HisU(ov9(>ZqtqwYMTT2+BLykMQ zfN$On<66SK=R=Va7{I**Qj)u@IGD_wY(Z+{fR?_jqacIgJ%@7l!Y4h$pb!NVRSP>5 znB$S92VFQ4J1eG&#F4+$9wl#)*wAl531|s~96x6wx)#$EEjD~GZ;hBPs&|6wl*1vZ z6sydQ@#^()CvjDF8Hw*F6=nz$&Kx>!4w;v23dU$ckd0LwfDvVr?g7H7fDiu%8Jl(OFux zOX*!Ny&4h)BXl{m`??ral5Ym8$WAM--ruhoByzO$N<-@j`Cns)Pm)CCnqBi+mX3LM zT?#K#m>eIT0YS&}IEe%yuGi}c3E^FYPYut#`Joss%LG2!<8o9F z**ufAANLWa5_vim2@O$?#Vabsp}Rx&^YVv=LazludcaOo0jyQchmq_L9(H*-g63mw zIkNhKI*Ljn3zCe%q#oJdtH(B%o9T|X#2r8~LBDb?*S2h^20cvs(H74^@`BmeT;KuBP_b{4F$PIw2m8~XR}Y(W|d zuMaGec=!hmV7o38p3*%{C+n!{6U5FS`H#?{vg?y;FB_4)88oMVSiwWvP5lp4MWIQB`#~c zv`Uxh>Ct_0hip1$J@w|j7f*zE69)#Ldw*Z{+^VAJ75*GA16r~Bi<{DDgXuc<{^amL z@#$>cy9sl*_P!+G@vYIm=CWp&!gyNJHCDnRw(;VLOCp%%z05*P;dBq@W45QhYhGq2 znj&LL*R27ac%z%=7NMqVq|fiYv@6iV0U~+5dHK(w;hxa+aJ5FH^!Q`z|E`PaX#1DX z+=DMZ7)yM{pRmu9YX>nN+2e*c8+3hYtWMrW(u~bRcXFkG?Jd$h{Wkt5U*nSI`P{d4 zN|}R#m104sh+UT3iVH`i=%P3>;-B`(<%GvL6+NpgDm|MeWFXl6J6uY|+e_~egiBGs zshHuHnsOZJR}m9v%zyv(g2!b8+FngTYT5l{m9n<{FXx@N5;+k_8utz)t!0bm5t4Gh z0D4*M1R0=+R>LCNSi|i63y?HyaOgTL4T88G162m&KYjho6^q%${p{SkY3nFI&IwTCzb~Vd;bGrt@wvrc5 zM#SdoysgNY95}X99MadT%`Xk~swz=lNl3fcTYmTRpcy)=N5ox4qD-DYOhYL7VR_;D ztjjsB6dtmR+0e!unZVvzN!!=lQ+bCwR(SQcx*tR$-}i$8FXM>{^O$IN~ucEVpLJWgf3 zUj$~#JFzsJt0ZE!;dmE1!u~|EKy-P%2cN4`_waYu6Szlgy|ik1zjt~o#-ugq309e6OTHLV5tW#Owny} z)B;#3ugh*Ep{T^8c32=MGbO4lFh5t$AoWCEK99x^-U9y~bPxrv93kZ#M5xMFqpnua z-uyLG#gEPXwNLEZ484W?@Qi5PFjC<#B$Lwfjy~Uhu>|81;eZN%qA;gmL;?L*p$$ln zz{3iR5jXT7zs0{PE`hWLZRrG$dO+eK6l+jgW4!)T{Qnha<%qGxxH&^biRVB5}RjfmSNGQV=aU#Mz$J#nnp?XAm%a0a=DYkg?Zd3E|j@&otlDHpZoMf;9(+NE$6 z6gGW;2C^LM!M5X54lh)YSkfSuP$#XPewKtjK`mQN9$Wh(R7r0AZYHU>UF|phwdEAW z2kcBosT-pMrwAY-QQ@kriLk_2Ym}3`*Ay35O?B*ibPWZKh~W={lJl>@^eLJolbb*3*B&x?#zqpCYHFyHul1GB*^ zLZ@Md$>IX%7{4ASP!u@M5jIcl?OBu@Y*{)vAIiQ`GoM1u0z-pTfj?>cX$Ty{kV-SPY9{rJzHvAa z_kKgFCsKTWc737}-Sq*a+xJg0DZW3|^zrbO_TYfuYqIM|(+VJ=4p zNJL#7w#DHzcycp2JK0~}`C&o=mMZ>9aimEcC$A>`AvTQ+ZZaD(Sr4wTT%|`Hqgvrl zzR>qDv9c4lG4wjUxZFE!Vspo$-7UxAX}z4GrQ*4L24zRSrSW z$(tG^MI;Bl0c$V#Lj=Xwc*LHl*~nj;yO3Ywr!f&7aDqm}w2`31n;lX!5@#@GHy-WG zh_!KQLFZXe6f7|D=S;W))PWR<~^y_eFy@7A!~^(dB-tf$?l;-eIA=e$qR^!3}P zMsV_W+}9*86@!E-lhVlI4sjA0|MR2|B{@`r_ZviAx!4=&fIjd2x`yzm!y|rK#$Q*_ zFH&FbQpb0|o)c2^sS*AqNMtk9>5BWdjski?qYC4A$mTr38C0w&H zzf6vlG_enJ-~`%t0H0r%IqZiUe|7C1nW^v7eVKRd z+`hpJFJ0x_Kc*|A@AZ84m8Re1OFvVoW1BJdnq%4jGl0-aOt#~LSmLDgi0>ByMn3mz zZPtd|zSz`;=nr%Ue$~lPWYvji;DaF4;QaAlhKx3qt?d_}JutC5n&{-{9|jU4!c(v= zyP8)dogromYYD|R4rGvs))5D%yav`vNfOyV{|I$Ehw=}Hdtp0BpUG25kSi(J?|9bk z1`#1OUuK7C%I9mZ5DDr_x}ZWNIZ{iY?x>B?)3t1LRHojUgjMT5C@ z{Nk%=6NCP$v~OGEn|Vv3!8sJ8-%&J%9GjOCtKXK~9zHg(r*#zu5MADeEAE1aicdE7 zl(P8vv@FRb5=~CLO*&9ViNuSSixGm_xpZo6NP_%V6~oX5ZO_H%Pq0pGuui-@yA*zM z2wTe1?J~^s?67DH_=mHNGs?ovb4L#Jr>E87=Xdl#*=q-3cxQ?g$tL5cY+U^rS=}EQmsH6ZB5}wDH|qo%0o7HaD(1!&(`ho4}A1{8M0|-##U~~g4 z7t`OC%gY-~Q4ZkrgH|@sexS7Dmr=8Ea(99;0Al+4M@TQg@2Mz0-ax2VC@(!Fl5kg~ zhz&ulPh6lQgLpBPu}<2~txo7A=-*m+YIy571(Jf~alwHR%V;H%9^&meUR7A<7BZ8Y zILP5#7;U8<5`g$H0;xk(2Gkxq2Y0G!I~5Js7OZ>dKVcj8hy)#<8##p&$GeEsb=~UM zDMeP}C9d~fsHwvHQXgod2DA-1nzFkr(4krZD%Z3pdwsrj=<_EXK_E8wfcX4i9krRN zLAljNubkY_DvyWQ=xYYq+|W5de)Wp;XWBZNAXq5cZQjWjKaoM`s&DiHEEF>E+jXYy z1_zQG#vPjO8mh9ATK_qduG~#gsIj;)^z7mobBhaw67IKCbo6=^^3@R~WF$kZ=6Y&m zYzPO*Gg!tBPn_jQ*Jjllb$U$W(V*Dm zW|#WA6Sv>ZDVaRc4XUv=B-X>I&VLx0#47a02Sc`nyy<+!G(OwHY$w}-j$jf<^qrV5 zB)Xm3jNd>g7A|>Lf>8HkKrt-KVOW-3m2*BeNFp5!G4xdMn5aKH0?JcAHVKx7w77WPbq>hCSr|P)Mw@IN`a|2HU+0O@dvp5Nu5(*XFCqdP*%#Nc~5P3a}O1$u@Z>5L;06_}sws}(1t=On_ z_55tTcq;n7WGF|9PK{Ink8TIL?YujU{;sp%{gezlTH}Ij=iNYd2zMV_E zSD%k@^GkjlIQ-p#JXxe_LC>a1d9{hvC6Mcj$|wIOIg1C_cutwX&RoDc?v-UVv3N^A zfRiATXu`CFJDN@n4(#9Je+H6F%J6>cO zx-O_pcAE11Vizk!qK4mm#wwECGb0{$@ea`~@cgR<_wZbh+1@UIC=MXCZtXd!8ibVL zTJWQAX_wViQP5OFL#x6WxR~~sE8c(ng@OG58M(RP1F$geNPdu$D_DLJ>ert>K;+Ng zRa}pCcd$UaJ1DD5%lzjUQ8ph8twAXX#aGG)xWL^_v2%AXRiOX$n~$~Vzv5k;WPeJX zlxRt~S=YE}RO!8^JC-P&VZG{rdWhzw8-e|LAt%i5Mbp?lulEetyxy*85r3A}UL^|& zpARCZDByVKiGb7pBn!*5F>&!u|!V$YGunGON3ABsI>P(+L;x-H#EAfN(H)0XXmSY!CNZ99lp zTQoSDczN~9b-)hcZ=hX4CXVi~oDjX|h*rWl!=u;^2Z9qPURj zRB{W|&T`MR?9_A@1@0NT2Orh?hx$7sQVa~0#x9m%OLs+2KrsLB;qoMN(-9S$gcTgS;9V^aR4!HkAMptvT*V2fyV3cyVQw{ z2%>-uj@W9kYEu~yC&hx!1dZrSu8Q?ZU)a;R7C3Q~;*R27X;QAR{H9CxhwrPsKe9$U zyfiytgHszY46HiN2nrJnA4S3QD~~+Voi=4alGJpNj<)}8wmgl(5gObhtW&Vd!rO>G zf%mn&#_P!q1s!<>B%D(=&yOlYR&oFO<|DyQFEaKc;&&a=rklFso+1hp!I3&{`r%jl6uSSG!#aXC9(6yV!&%BBUy z0`90Z&Y<8$_T;ll-^IRfo`}n3-2B{2~=i+?o&LcJz zu4}m3ZC->Uef%Myu1?N{Bv=x#UUvMLBTk2D945oMidcBv=zQV3C3(!37*rffv20c; zh;UT&5;uZ4OCc)gtQC~fHdxAh`@$8Uikc}nJK~ZyUnM>nW7I07Y{(R!z+6`C_)K=> zqd~qXs>-VHwjE2V0Y_-^!w%Q(4wYgNg9amlF;+`R6Bmg*c9BbN*Ebws?Ntx~_U~W+^BE0A zr0#up!&pC~9%O#3AoE9Le$37~bdp)02s+JI>L^mnhxNWCuo3(MH)%o=NPCkoYKbZ6a+h@3=84HUAa!e9DRxR?5I_ z!(&mz(WikWIJxYO>p*EX?-60uBdzPM1)E(o{f11bwVHW!GP%UEpF8Gsm#JJzZm|&X z`9If@U;!7!I8{A9hO0ZGE6Ki)gWSikyN!tZ*j_)kFTNfiYvJD5&T~BTqm;LB@r3C428J6|%|R>&A_n`*zzaMD~pA z)e?p5nY|)JMj06?Gdn^>`Mp24^ZlN4zCZn;b54)P>5r@T=RL0L^?JT+kswtA%E*(% zd|@XpM}sCd5aRa1xJ7}DYJIrNjXd@#2+ttOQY9n76Wk;^Vd9eE;|_+^+^$Q7CAx;0rV=BU<*t~X zh#_BYj3YkmM5oDi?n68< ztB;3?zvkI#DEOijai6Ypa$IWvNUQgr9Bx7TIG~oPV}V)EbTMhDL;WOX*0YJ3^-PRL zCxP_?jdI0APuYP{z}yH!x_}!mxL#qN;kF=39@uArkI>x(BV7L{Q81o~%Vi!gCNy&@ zY`H1`^&<={C>b}A|9EDzkH7x@<_71&XSxD*BGcGU)A(7nk82R0)XOQ}kbQnF60xFN zTW|ezH!jgO>Veogx!Ps>g%%jgg)wEd=Pn%Iv|cAuDluq0Gvj-@xlnaMDT?sonGb5s zUQ@Cky~1iVqBp;hU4K;BOk17O*P!L>sr86jj6OkuRDkxdxtOxK#HW?EJ_eV-3CnuP z-?_ysw@&t&XejcTYkMuLa#^BV2mLwOTCUq9&U5$gxE)g{&-_rUFdx1r<>tShhk-- zeAZ&m)v+}pY3-n&oNj&Q)6s>aUQsFj@1@CZ?0)aYM1@R-8Y8KKJYB~r+GxYF$qt_V z=K9n+9|E3~=eiSGK!qUN!>Mm4n2m?9vyd>DP%GSmnmB;70RTJDaAPVlQ}WJ4V+>9? z%ql)U+4wB^Bn5X^z5Mu97ckpc|G@uH^kdUO^R+6-#J=vDRDLU8CQ`xS9i7A(dy$IAWV3t>UgG>_%E_W1B9TG12l7!=UEKWD zhmHxIDYNICH40aixbn>3_l{+C)RwR!r(1;LJ4!vsQQ4d=Zw^|j`NH#T)o@f+fbD_G z*P`S50E76PO!C#9gu%|zeHDnL5~5HVD_c;&K*C@ z=!>IZQiJk7UIzVupB3GmFflqJnC7KRU`om_Wz%4AQRedGUC}1Lsjo1OpFaGe%MGbD z_jL(=o1R(5RSQee`@yhViTn}9Jn*1a9c{)^LDyZ%(A8oc6{#eiyAV;VqbgdI-pox| z0$by8Vx#ASkzFq+t6QBqX6ac(;3$bSR+S_(eYrb4C@$U6J0_w5XE;==-^NV$Sq`KQN$K#i?^d zM^EsYbOX>b0r|gT;{Tybe#6>c*c?hMW4NA_Em@H-s{Bq?62I(5iq& z^;D*kP+-Dvm6;Fx(*9gM(JripjFEBcp(1kvnA4k5J&{0)=)-Q(JtZz&j`37z=+G^q zO>zMk4SRArOTAg}oNHK^3{9JXq7}e?XnJF80qm#WI1Nq)MJlp^ z^oU}{dbK;3i$_W$Li~1{w>G}~;SIyH!sRRzE77zS2Lq7YDyPh(@_ADA{0Z7Gv0Io>*T!j+k;<<>JM!ESUk&Ur&MIT zpO@&f=F=)qjg(GF6}vgA9&kn$Xa|^V`u*0MU&)BUBJgN70Dvzrfalh()RSzK?n$!Z zdfA)bwqU`Kj1Nir0pM$7j-fMj@7NZKvo^a>M+)x?1$8+-z8PavW$vD=H5ps`^Nz*a zLQovY|9I-_9}Wq9;a(^KKvx29+rPBeVA6Uswb+-gV}ktxG?C~Y#DcVuDv?J zi$js5x)1Y38^l(PA9$8zxjd{&p;3!#$`1h0jBl?YXa;zcHhVn3$DvwVHE`3bWP`TI z00j@v9Z`GPQUZs_RZM|?RwN=qOW2%mHY|_S0HD>MydC&O@`b}6d=vsEn>d_QW|H7y zd|co$dPIUFB&&IRXPk+kj61ac4vuP5h0QDNDeyd%_L-TaMZ9?ge794>M(kX8s1;)C3~oX|6^Q^%E3jt3;g$IJ7SrpyUQiSW9;9886+jO;h|xoGd`kQ; zMiXEYfD%ZBP0NkU@a$5PX)u@jM8se|#lkQr_0qzu`J?)3H<*+3gMl40QH@2c7w6Wx)y$P^H* zk-Vjw8SH)*^o$tqfFPz%gD2q)PLR|4{DasDP#k$soKrPtDloJO&~pnyj1Xg31b#S;v~Pvj{@^nM*vx{WN@bMyt+@1|)SMjLtdt5hKQjk4RPs9*BNO|2ia`4dleHGCOz2jzW6Ubov5aI@p&l3wWF}a-K_t%b*FU< zp;h{BD|t&W!y_*2^TR!Qdo$a9nxi+>lh>*DhZa`0t;ypO&(M-;YCOnd)7EPB{PwX? zn^Z+0=GWEUd-s2PUr^Ixs@iz~U9`a<@brteniIG>`T&iYKR5z0)dJ-M(1rhkssD}c zj^}W*+)v;m5-(S#7?0OXmIkAVN2?d>DI_|9N1Fh8*8X7Hl3;RJ3ZhJc$LT0~*K29b*l)3WmP+5YHkFdZf2ZZ@~gS%4^sig86BENAE3CAs2Un`(u+b0+Gjvux#c|_Li z=yRq>T<4o~SrC^ic`~yGc4FZ!+o8)|Q`pxcD9dknVR(zPMi@dOWlTqg%&+^lw@Utg z$oJhk%F63U;}6o_JB`>5APM3i6*7vbfuB>OL!5vG(#{0}Hrl!)Tmfto!|wh|WC~!8 zKw5;b1;{d(-T%lz&VWDh^dl4czd*{YgAHZ``%l}K$?x}4l>)YAK%HfG!pHMzMgzx} zVZH^)n}gW{5GU)>f$uN_aE2mi9USAH-Pz?(D`t`^Gp^u`yw_dtZ}!NmXGSMYus&@& zbIbT)xwLKJ#nsAQxDgmKu{d>sUhu5#Pr%@k+9X1`?Qb11W{6>$<8)C7v1_X;Jh z?R>63C9x-2qnkRD)PbHFPJcI2L0iCjUH;?dWZiK;V_GCJ<^3P;7lv;gskOghmt=Q# zXW|`Nx(*RQ%J7`28DAV*?Sd)C_}%(DXDABp>VowMwmz&=dz7FG2qzbBX8{eEy8~Pg zg3;XcMkBrczgaN8Ct>Qs)?`KKRoa|rTb0kRGtWK-Lo&%6%#h4N4$*QT@=(G`nM`jR zMsE1+&8|)Uw~)iVd{4I&mV#|*6nY0c`I&Mg&J5bkSgK2v(uPRg8fL(3YS%|aE4qGt zOH>vCA85~Nc(K+U+=5$gGmv^2XZ3S>?eH1&NnD>Xv1vx0*so*4p4GNk___)ErtONQ7 zAJ`BfrSd=EDZs3&3g9XCy6v#kgA1>+LJ7ry?dA-I@WK`Y_m)O)qg!TM&>i^WjV9DW zf&#->-x4CgLsbb=`3aWWddcE?i zp+);!V17nHN#hs)rPi#+E!)BT^PFFvLCt7t?P3e^xywfBdP79H&cqi}9y90LYIM1r zE&SP*?wJwM)X6}|<^47&kA7TG?p6B`$)?85Zwyv~L)Y7d_VvFH;qa5F`MPfJ;OM^H zrkp#Dxx?B-OW*YifB$Zr6IfRCpfaaEk-@UE1=;7Ijfoa;aP@)?Q!sY@&!W%0vsaCf=dvGNG|(rlD+rR1_Y%1zVc!!j7-Es1+Pu| z!9ffC5g|`E^JTqh*@^PT_+BNuWr-8`ole6A+x-P1HSWN~XQAtZYP9b&YzRP&;u#e$XjHC_3h&iuT6ABlgTGwDDe(iNF4uUb+a+TAe7egzd!p2n z34P8Ya=sgM;mw|Ch(v!7l#0lnITnBvku8_`A9}qAp8|DI1wV3EPw78_P6}XhfDxUCqpHHc z_IF1wz^4ZLl8Y|f!wdPp7%b@1JXP22Y-{8IGLXG>o9WiWOH>})pSnb|qjiU|0s??YS(mho4SI{c>TGdSQe!^~FpNv)A4yDxkG%NLm`{ve{wB=T3@C4@z z*02JXx!3hei&}V9nVplhY|{RnNQZ13gJv=M<5a_l%zB>SX3Hj%t7h{S+)$1(2U{YbUz#iqjIU}u(v$HR|JhfA$Bk2`Q zt{Sdh@4$5~7m```e*Vz4+P<9DS!9(eQdu$9_z^Fw8Q|@hrA-Aq5qn#4q%+#GHC6Zp z@AK61|crFdT5vQLS5W`#2B=G7JCRPFl*2Z>I{^DThh-jq9xRkVC zewWCz%L^|PdT~myMZu7+QOcL9-ud_O^Z)qg%Rhw+fpSQJa>&Y@vhwsD0WjY~9$djd z!mV8-fYcgqwy z4jpp{l!0dC973FB;Bfc&PU{cJstdp7NHs*1ML7n3%=Pl2Cc~q*EHVVIS|6;ss@|H| zNl*R~=sr-Tx*+p0dx?y|q#d*$Z2Zm28GG%d2y~Ixk*w>uj;YHnXH5*WKb~<6e#0tn zs@qc2FIjqsy}Tfw2CJFjnncA{5|)&#yA*T7;te`JTM%89NAWP*6|A5vkKt2(ndf1x z(j&|2Uvo=LKKrJU+Abdrr*Iid!$b!aZ%JkdcY)j4%Q-9hV`Y{A!LG6^zh*kY?aT}p z+|HO8vcT{TTL2VLTI!SndN|sG5xW2y67HU`;{g^Oh4O-ecK`C-fwut^8)XK4dirG@ zXw+_bJjLMfam<|9WhuP&&)g7w?!_~1K!bUR%Kw%5esKLw6KDads+@F3Z4FaQ@0I3A zQw)mA%&R?tMOLp;|FWNIkV_w!&(Fnfe?&@$c)eMr{6tZ~K3@CPS&AmOUjN}GX8qit zb#~^3wg)tNaRypS@oeK$`7hQO(9-soiq6x-=ltjjPs%yKa;Ntv@Ry=SBqW zi+EYNO}=$p^o4LRxPf*^(E>f-Q#pQrEYee&X4z-O-FRD*!!=RXiO51FZFL1e7iE8L z$vww?N*eNeb1qW#j@enZx?5YgS>1oQPe9}IQ@dNPphp21AnpP-aMWMcf&V%=JK{8h z=OF0E1b}|S--l7I>@V?}J8IUCb{|Sb%ZNm6Nk$FI_l`K?9G|N!D zQ*sXbcGcE?ft-pH{5`Bh+NpbxwvX~on6edPdZ$z74i@=OcfJLcL+WB@rH(02bCCu|lB1?h7Ulz|jQs zTe?8C@V^{_iG*bF*-D#8!rP*n1-1F2(d(+%kUw$mDds(HLj_LCU5po4L#SSmkVkem zhZg{?(=8@7@u@q$Gb)Yy(SCv*em|2^5o2jG4mE^V4_`hDCnEBV*mG1^TIl z23`UnM@A$%DLuvKpkL5@C(2_-PmZPhy*)=o$(VDVh5|7vOKUH}laHsb>Pw;1A#@wH z`n`Bpiatu{(skIa<wNge?yZuWFLS*O8| z<2yhyX^6P6j5cJ5yaY-Eu2@*6CZ~LJ>M(aSQUT0t;DEdAFQ9lsQ_l}{w@|`-3P82w z0!Q+BqWS*&yCQs=Ky?7-rF=>VBpev$UBI`Bu~ye4HiE66GJ;Ksv6IIwKs8HxR{H|c z>RB20k>m+oTIGG!V%@!qzV_yBI?B}4Z_9SSv^IU5eh@|K;z}{xb!AXTL|n33J=-U! z=Yhty?YuM*{&;gm7(c=N>TNw==l3li)GnXLE(t+~(&72tu7s&dl5M)oD=sYEZUX zlwR{W-_&P{+|GHd=!IgLDMGQ`OofthIo66hHG$!gDtNX(^VolXJApQUL#|ss`;J$ZfS~0Wi<^n^2N*ubPxEV_m6igvsXB?()~q_*TdUB!h z)`M_^f(uuL;njDG0 zTL;Gp)i~2VqjTW?6*YMETj}VdUb~oUWBsk>LXnx&9w0ZslOH$|Yb7mSTFWK`Jrl<5 z*`h`~U7^9}R-k7swdDTM**mqqn7CZ@>-giZq#Z+NuECGo6wyndG@M{Do;o|cp#XCS zjeHDB1*pH_NDqk8;OYVQvO|Ms5u|nb=j^ceds0ewV+^1AzGGG#&=UfgGn71$6P@?z zTw=r7G?6Uy@1Db%o|~9dV`#<-AU(VFjK%ypbK6mFJCqN%&joEF)bbzpvkiZU46z4# z7YCqsDJC3d(fgg3pO!D0k9_!k)(MeXs8OX?(~>@-mYLc0+PM!x{Gou{#dZ!guVJMw zTB6_{RbQs5thPlP-SD`fJ#q{Y4#|R${*!1%t*#8>N9Dj*$FK^IAJr6$n^}3QpFj8g zB1=4A7+r^`#UKOf68>@F89bZ%-W{K>OyogR*dT|Zd@2<}JVV2=Q{6vsPY(pb3yBmE zaYfkul@fdJZ%grf(u>#P(#rdAr0RY!1+Wv7%aUodv*|(XFW>iBwFDZMY)pEEfh>Gpl=qm-RS?FHs4QTa;elQy~of&Ib zYp`quXaint`vt!AkJ=-Q2L$7?J@;9TE)UmJg)(Wh!LH%04{#2@4EokiT&nZo`50x4 zz*-()9`+22q+xTe?-~9w4L~CNgxf_2J@>s5Laqt@nB=g@2>`eMdU8&1CB5jAA=^2+ zf1C~=6h$Hc&d+o!_6w*85va(iO?pqrsS0un#W5bxq!j_+XeSR`4QjOmjER5yX{**{S{* z6X@ZxI?$tB=jI>yxIyq)M$#XF=MI8F=!m4LDs$&$h?2=M_T$C+{5m%d2mXz zdF)5xEp_@$Z*9*&Nw-XP0f(4rkMr^j>3ZwIOIT0{R-5~Z@!Ve_Sf0Yq*CaMBM~(mC zdO!opbZXk9pO=r1m%s_ifX^1;Zp-J21ep*2i5Z558eqa|W{o_9cL;Pawzh5L1g{zK z0l{#n>RZ%_%>n{G0O9w(m#y=5@->kG`zL%(!M&q78t`!n^U)R#YI-U4{Jr+Hsk>i+ zR;WDo-h+%ZT)IT_C1=as`~8)M%FJN0xMc85|6|kvz2sXB)~A%a>ltZ_NpI_4`3*Q{ zkh#C0LlS#)>I;y_dv`O&rPvD4O%{%28j4vItTX_ATCP+MJxmVt@MmJJGVf8QaWJ3v z-oIq@%IhgZXtu?6SaR{0Ahrue2Gd7m?SA$aNm=^uq-`r7>jSjZ)%Y>6j!6Erj_kp; z9e|WU7$VdxyrGGCR72|ss>@Vj1p|lmmgIG#4Mt*cWhVqO??;4gxPWB~t)u&x{N+1! z>19N*TydXXpWjvV5XZ8;+`7Id$JiFWZjn>qTT3uohkjx63=l;}pT`iZ)Vkj9w|}*# zR?XBTHqfnN;4xWZ_O5(fiD~98Gs2n&0bNpu#=px;lW~j!?@fqrT&`Jqo_hUp z18nP?QY!CL(<%Ak<$x`oLp4dvWaFiIu~qcU{Q`V4jb>5~v*neQxL*J4GoKvSGL+h+ zf*}DF5PXu$YG5WlZ@F zug}UJ4pj?8Y=S{YZlVr!K?i-AZL=PtdM%s5;XBSKs?fXfxRN>-7YVN#-!gat`Z7~N z3DszCd(a34nUNAN!@2A|ER)Y5(rJgLtF=hBZB)$I9F0xR36AbrEV+$y*nb8wthXruCIfHnpzdj@v0zzz$u8 z3h2)H$xye9<4ZF>ol~Av8>W%+_J606Lzwz;?68%*Au3Zi4MYD8lvt}!&W@`gj^aJDR&X&6wiW%+ZPE#2A(m!k}U}e2lWt6J@fSq zARRJ**q}7vKr5>VNBtK#N2&Mp9vd8&M zsf zm1_!Api%}Y19_(`gEBhpRF-h=M`NHtX7j#zSMnH-Cc&f6KYjQ-yZ`;c6lUqfSbT^P z#z8uGVHz+vr=j#4Xi1$ZjtTp#GN&G4S|$d3@=C@6jtE;m8@P+D+!JT3jvzld_h3WU1FLirOtI@Dwtz%=mlHS7zm#Kxyj zi-j$XcOouW%nYyGecr{q^t$6wX;(I!`?4C+Il5-#9wBjQd^etv^K_vZz>DMdA^mz6 zp>2CGFV`cZkLj@wgTH?6huV_;mOU+*l3e90Y>Q4+!%x{tY5`X6$20(2m2=d>gD)=m z76E*3x~O@^#Dcu6b~4A?EpeSkl#~1KGz#%$lp!;d*ya#L=PS%n=2{{WH7ZdT839Ci ze?H3mK=c3{G|&ZZfD_FZdiVZI&?-52i)E`G=eB5c4-hB-dD$bdToT$${vS%rT;sf5RL9eXClbL0)#zi~RM* zpHdE7BFWT#>OaO@b_4o}N_)ePl(da%G`*1c&Jm@IsrN0=>(_CLY-MU4UkHx@LZMt) zU5l(hBn32a{s!2x(oiJmZkwcZt6t{aMW46@PXF%cz6%^}?-jtrS=O3_LTLJA;(QNG zoP%)k8=@bAlKj#42GV<796TJoq=W!c&ISSE9WkuEf9hJNgfnCmSDzd@X$W3s>%Plu z0BtuQcn54ZfV2m2^v4Gvo)9L4d&#gMlLL<|)vg%_wfo5tZ^E3KLB9NHtAVmq+9( zkaSf$O6xK&WhJ9CIWtez`bPtE!1FetX1+8d_t-fYqL6-36lO!oYYAWoZa zf3wS}^ZeC$!gR||!Fl)HQ^w8*Mn?1(>VB)3`MKIb3m7vb5T$6zXujm>d@dpTJZQv8 z@qbHzHs;4wC=C)WFFc5hUhRne0 zj1mUv?zUiZ`p+(~UAH)J-nIievxE?94g4g;ooF)}!tAM30kAk=1wikKCBg7e0Q%H= zg{={?j2b6#3m;c>{t%F$eN@z~*qQCvC6vyj50|t7Xc1DkMf|5z_DaOzF#uDepY( zZ#AumcrOAtd(*(?=HG<$d+GjI_RXi2De{9Ot;x4?_yxUnLbtAS1%p{Lpj`EenAaR1 zH(nzLQABRP@r->VcTd0hV4=ZIYzI;nJY7Ac1qA+~8UhEpixWz~4&>+90q3@rD*_Y7 z2Uu<}9{~qPK3CA|1;+5de%HzY=4lK1y+CF!pooGat>hJzwZ-|BZ|E!VgRUpP!~fvO zi2rn-K%qerDb$q)*<6ZV02ukNzn%d}7Y6#W+gPjC0iXbd{3a-MWdI`}iF5TT*~gR7FIn3+T~o^LvI4DO*IG7^jn{~_834`Fa9POzh{fZ0Xm z{8!SaZgwYd$a7klnjn>aKJ^p%co^_qx+*2_9*-XH1=LOK?<$Y8g0z%A73?60w(%a(!mmrO)D&rGzS+ZppqH>Oj|VuzpJf-3pQ4_cVGq3fG4ji_>0Q%w z#)@pDZ$}HO@|Lu18@=l7b-n#Zb>|>xJMgTLPp2Fo*8ymIDgi zYys9yJT5k-gTx`NF!6LixI8O z^+p%yjUe*lk_x`apvuKd5x3su_4EEV_i3dU4+U)kED$fq6GYdHcJp_v!H3YjD?f1l ze)g_Dj*uaJFpG_6CTLvp?igXC?{sO4cAD_Lz9S*cdpivBL4u%MKFU;iq zpB?oC3i^jQ6*rRwHF?=8{@U&E5>CFQF|7){5GL0% zO+KGgd+FgmFSpyyWR+_JIr_D}Byoe(_1inC+@`a?bW*y%-g69LhEet?T3$mQ;+-t_?PdJTU zv-uB0o`%d5sqBnp3qLaEF)4?A21iyh_6p9AMdRAhbGMz}ZoyuCi%{O4U%A3FK?|Ui zPCA0I3&LD4_326_Du2xNJumsxsH@igGH7wy^}%CB74^Ql8R7E>0Uc65xQ<~4RjF!^ ziyBIuc%K*=nik>`TJU_zqNW^#yC?UX*I+ zDLDv5sqT%z=LhGhOlqsL8%HUNczF5aj11Hg;L0W^;UH1zb8Z)j+n z{Lj9tkt0bkTVXSbt29d{Qb+`SeJwc4_1gpjqUh(JRH-FvLHLBsyrmsj;n*nl- zE=6`ro1a&C=j&&cv*zdY;#QcP^omQJ*lH};hV_dC4hkfUQ^-}--uFFm8l(K3G2S97 zz>o}sWW+!V;|2Z_eHU*$rkn0v7-ssOdzVJTRby;H&iK*f4S-vlwk8=9G(WjZ)nPVY zm%^}jpfZ2NRe`M+hOh&03WT$u6ei~p?Q(L*{Cm|*1pk2G>3GOCK5M&sdIsS6@3D|^ z0#64j4Zp3!pqCFacfBj=^pNZyLeFII4ph9wuj{xyJ+yl9(qX#{wNT=ecv?j=aau-^ zNI{Z@4UuIEz9oZVAa6Z+33p@D#)pCnP=o;m#W}7BPnmA`%VpvN!NzY9!~@#3xT|Wf zO;@$lEZs`7TgHIxyx4M$%uI(Zre*)O)rX7@r*orNcB=usK{scgSTWiSkyc&Jl&w#a z8_u5eRhP`1{z2ec2VJKe_F%$iNl&uXt7hByCElEiBRl=xfLTbVPJuyl1cax7HaQA_ zDLia}6u}e9W`gwS;3LB@_^sn?FM}i`Bb>N9tl$eH$CT?^1}5)e=D)dwP$Z@+xsvY^ z9}U)}i7f@OQlh79Zx>e(TmsDTbK;ilI#C|Ln7(Hr4u$FFce@1E6q6rGQ4?tkahxEI#z$FCZQypWgd@oCik1t3>NZ1k;KkGN2}Z9KXRm9N|b{ z`4)iLfxNx{*>ooOwe%rHM%VmFN*qMOyfjw{CQX~c{{_om(M`}Z55E}V$3tZuL%;u+ zVf2jtnF0QiU>DU`$1bU~*Bwri=OQ&yM|6ur_3=v{kYy`> z_9w)DdsPyqe|&DanMU-->~jhliT#$f9xean*QDPU+!8|%@`@;1QFSd1r4ixOYv8&drUjgRfWpB!p998kG zw?#`)@Xqtz2k;L`!LQZ$^`#;Q8Y>k#sFHb5S}&}#Z#~E$dt&Xku*27z&*?|{Np;yRDBz&gNK`}n#(d(|;8g=u4^6kgQ zw9NO5EcA>1aOMY<=wF<4NS@WWqMjChxQ@75fkl3i;@4LncXb;@b9nW*ughDZC+j+Z zxDcP1uycJ-{NB`SH03%n)HVy*5>D`122%BueHJu{i2Qtf3&T0P?D9}GF$|pHf3^W> zXefhpC>Ml)wyKs6RF3`=5?-03S&zv5k3U0n&pi&`j9rgIg;=S~hmr3PSsZxqj8auP zH#T+BZ(iNMSB*{mCW)*LJ;=YBZ`u%QPm_4Ac#R z^x0p%C@jv=PJ`#gTw$i+3^J>2Dd!7Inh3*ITIm=232ZPXe0SP^dc%+KQ*Q@c(N^xT&fQ@WdV-Uf_)? zBntTu|7oy?#5<=7R}~ag`801SfW{k!W6%suZ^NNYyLZoPMD%M`GP|MJMS7HAN!J+Q}FO^_1>#;`bTSwJ3Y@x*ssNE z=I{I<9@@oG$A{0XP6H;i;7Zt4bCVpU{4V6(yosw|?NB&1ZF3U>mYIoleA_#1;`*>&tfD{3b1T}u! z#nwv(BMge&> zTmVRTfB^BY&Lq^=23KlG^L2E|BLPIU;f31a6LptD%x7Z%aftcs+!!ztfSx@7#zkA^ zpS@O@HVKZJgw{I?;ua*ZkU+mzj@I80H)-j4CN`5f3Z8OKMA-?mzkH$9bX7IzllfPr z_Xb?qwOftLEW78~-N5WNl4H^fb8mBUobIv6Ua?|M=Y#ARA~o`PpL2zF_it zH6HPh46RLaHvFyxy~JUq=eLAe`OC4ROKXxc4e!>defVv&i4rp83~8K`knY^R%%n}y zDqp$lsgwr{EYqG<=|e^k3QFiAIrYkIdN0nII&Gozsr2I&{a%zUA&8^3$`exoU$4%sUT>4Y zKlaD_mNo}_^KtYJ*u*Jw?Qh{2%O|_BH;*IjP>uT3lQ5W0f=0Qt#0iTNmCi`4NGkj zTOwDR8&C8oZhKG<65}fhp;GQAh=ZCj%Va(sCoFL#JSdG=3V!bn*oZRQMi5bl?(2Nxi?3> zyq;$Cyj0}Uw5UT(>N*^BQ*X-pG<}t+Y(@{vu4y*1EApo_^{n-eRGi1B;$~2Jxtss< zqMyZxCyPQMQ)l#=?ABsD`MsRGLv4FGc#o*Bcaq{CtaP@Wk(fC?29(F!xi>6Yu=e<3 zAjusED9fpW0UH+&Xd2}O{E(iGXcsF0{jvfEWRM+bi*^N}1{lG>-+rWyiD1>QQh*yG zG(ip$*>s_Krp%r_CVtWzLhbwlTCwSjdoJE|CvZ@EWp5_64I}DW%J@m|b4HaxYaC36 z@4u`h@3*t5y|BKNPA_EOgG-mB5v2dMNbc)qSN6CXC$m1Ax`~*B$u*Bs$+y~9o?cy? zLRh6KpsDZgc@E_+2_#EOlu&Eia^Q2B3dP4W%JAc}gd<wV&oN@`#$g{_u-1PQj0&X% zfpYz^O3Guhw<$NM3by1Sf58D+=MFNfPI-olmj0N6O^uMPa2@zGZD4{ufQ>vj0FcFEn3 z*s{6G$M9VWB13GtNAh3!SJ$mvG7{|am>a67$SGSJO3K|94Wc4k{7`5}T5feiSJO%# zgnW2_%-a*qVWNyn7<{ ze~$zl+s-n{mA>(lnFvRSlb77UIiJ>DqA4TMX?{4^US+~x#j=*R^kpw!%HO;rtkIr8 zX39DmkqVTWigfMalG{Jq`dQvqbJlR{>KIhMU;@%#W%dgrRG&8x*##B?^%282>C(7n zvx}~$vZPgVei)u0J=D(_XZ;jeGZl5EA_nC9VnO1nLzq`z`6HEA&;QG>=h12O<6k7jk_(?4n0#{5yBNFS<9f$V8e{b z_wggolAbw6Ll0s%>pHY=Ks?1Ct`blNz^UjFJ1@}W_6PbpuqlD6VN5XqA^}bU==KHH z79^X-?Br@SM^Q_qGYyygU$M^_M<}wX+%CG|(;j0Lf&f6@eo;H~Pl5 zT{wCA7q#_0O@V+KL9_qBlj~7t1JD=b+@oKXVP)6|!?$`?o|l&pv_qzPuZ6_zZ1a_K z--p|6Zv_uiOzX%p9zF{@9)YI8b9WQh!J-0`7Ay(zQ@%qjMML1q_5k=E1Tfly|FQxc zFfT9`K2c740KwD0?-=n6%kZYnEMn-A!5*D2O0r)MJ*S_2wyu|b${(^QFmP~fX=CYE zS595BMW(&TFqP(W);q#3#DmH44~uTMEk_l%yO5s6<|>b4K$UlSW>#RI zOHQWL-BfH&-m}$NEgHI+9IIv^KTG&HTt(yZBkGY2>h#4YMiU+D_la?E2RIKK@CgCYIX*3CCMBfCg&h{G0JuVdrB|NYyUBMQz$6P_Se0!$5yZ(e#9b1qGZ? zCt|0++u*?Y!Hx)wuT-uT(TcmzUdaJLh>I9$0EqD7-9?q-fjh@PJ!N90BIM}9sc?4h zi(Y5)HAa0|A|TOaZqn?Ik zl&T7SXNJi!KMt8mZY#eJXg+dy%(b-JR{MQDipa`f zvY!E5EsIv;l5*z-0V7Kho^`b~TLtG;TfsB;gsb4IgKqHI=dTLMN*=c^fG&dzuG^2Z z;-R5>pajC;=WmV0A7iy3+tB~SYC(_f%S@*f5RhhF;4U`q0?Icrh>3sIFznrI#dU}% z4h%G0URO?VT4trOV021ij0wG&-I5!$i4OX9gIY+atQVk?KqB+uA{GbJU1E!{PCxB!!-qH|do6T}2L$uY!xd!6on((!3p-3u*y*bWi;Xy=)PD zwl=m%0VNC@2?$jG<)eZ6HvfKP{_&7-L8H6?BNOHU16)5r0b6%_6h_4fX4C&A*{@}) zxaX|U&qP-6sh^qpi!Ffvg{ydos!{_c&a^b}@BpxUFM^Rh6)eobvI^TZP$0)$=Z?g|?3BRO%>H2%Fkogl6V}g)=K{zH^Be&)18?m#NrAzNYnW z)@zP}^iq4KewrJdH@w3_4GqPJ_fe6RbYwasN&5}x8mEMJ8$_?t_hyYA@WN^}7O}~f z{ELG$wal^@k(_%NMAvJkzY$$ySTaDUh`8gL_UXplar_-tSss$l`^313aYmrNir_~| z@{~2_?&al;5WxJ(4{|s#FHL?xyaU(Slivjfa)*CcnE?D*r~7bF-oQ=mY6}A(J|7rn z8m9T*cj^ZAkzNM`CJ?t~l~EG(hVpW64RuCvmZVrrEhaBzAGCmESR%7LpWncn6eT}` zH8@(m_r9EeF3n=>0^a$28Cx6Nslc7CYRo&8?Ik4}TJ3RyxnsN6ltCrJ)Xx`W!ac}T zR59zT^UhwrEeBH>&H5-v-&ohcc=3|WdcITR*nWYO=NLB^>TO9ypUpM97&+jFE9-{| zh513uCaRBwTV^>AMFmNcKWe5LsyoEl3xo7iy^>*UJkm3a^hf+8gBN7^j_?x z?`CX#>_|;;j6e_fQ$J`MM|(KhLqJLY<}J((w;$@A-F#K%tDg0h=bk^Q8I|X~?m`MP zHOLV6xTv-5{uTsA#^)^)b=Z6fdPNRYdKn!_G)vKGNUQ>ICE2?`YW-`)CZcDib5x{w zf@|UWo7XWLN#-|L(``DZ;`L>QX=Z4K#~YHKY;1vc^d)wRk*xlW`oUc>CbwR>G>JHg zC&~3QW_&Lz>Ag4%&~!YFCFSM>VsRf5N_Z%n`NmtG7|8kKQhodpa^^0dX$&}XvGlIlA1)-&`cqaq zW&HsLnhPHQYXe&Z&{zX|lBbs^TmS|xCXiJL#s_I0fL#u#A)vb*W)AI}Qv&jefJzfXcDx6%iT{J zAuIDMS}s?gXfw)T@iR)grIAC!W>+xsR@>x*D?TZS&o^_vdVDu>2eqz_m1N`K3f8(J zRB=Iyze_bjX7qW(cmvJ5JYA*2&#LEh^fGJwrd&RBCXH#^gwduj9u{V=kE}fN%b_%G zgVFwk>EL)Zo(5|~D%H8W^ zhz4zM;TUvE(owQ8B*MOO4<3U0E7OT?p;X6_XtKsaE(6K}uyVOX_5@01j(g)Qj5WAh zpQ}b^uc8s;_Bisj$265rR24Wtbc!Gq|BL&n)4lc_Q?>?lCfQ1Lea4&+1dR{$ON#AR$~S4>@0&URA-}TK1EB1P^TQ(a z03RR;+AQaRtl)>?hlLujIiJ5WIxaBGtwO@|)@+c`@oe$94C>Z)UoQ1A@s6wp1%X5i zr&__0aI~M7BN~Lo{|EZT=Z)|HPLh*gFd$(7T{MHTj@nxM(AJ#1Lnox|^()k28t>_~=Dh0Z#lI(#bq}BlWE=H$ z&&gHXg|9~y)>;$YAEdfVq_a~>`HL=HO}vPK+mdvCsL36?#IqDtl>6$|KK;r?*K^QS z5+~Oz+0ee4KG2gm;OX<)q?&!5;8T;Pz&@{AhB6$6w=ORhW#)*&_(3Wd`qkEh1D2nW zn$!+!gAHo1eA2iNt1IW?OXWp_8$)S>JhvAvwK?96q&icbNFJ^rRteC_|N1ACC;>gj|Dk20p|=Irwm`Ak=o{|mL2%NBtvtk(_=s6S=cieptd)DiDdYbECGu4QaV@S6d8`{B9?bs>)s+0~BS ze`)jqoSnI$w*(|1II0s`yNhlW&~}^=jlB76*pJCjQkF@buswTwkkH$`mrSqHx-1LX z?YvOg)uQOvW%yn`x^%Syk#r5jF27+NseUFavpH+ETT~|+*%U#8ke>Z=!{^pj5~bRS zn1a%q0#3Hyhs4O^%LCp}1|r8>0_;KFXfElHMy|lCvi^R?U-9=6I@h(JjB^Qw4P+k6 z)k*Fyf^SQ~p!??&fW@xvn&*0FnTg=o03T?jX6)jjuFW6YSy~8SQPTzrS#1w@ki4wz zCw%e+h5lt<08FN{+ss#}W`KdAob4_E5JAX<$Q4ygzWF~AaifBEi1gk-k7S2itE(5uy0y$@6Obue7!;*k;4@Yqa0DAR7ykB8>(wE@!HS zRp$rD0GaGtj7&Dd%aoqAwxK4pqTx$9qglz?W0!my4TG&k$MuOX8pH7BgT)Z399|eB zW^*Zfbx5_0PZ8Jsql!G!34)NT~Vt@|v zRD9JREgwL(?;wG)bA_77022U=eNcNE1v}vqK^yW-Xsm&1lZ=Ll3lmN&Q)BK8pT4otN7$d?j1 z0LN913P1>7)#+jxG{LS94w`(YFn!};-Mz2h-*pBaA6@5mKPPorr}9dkk{-1ZO`~() z_N5P8%okD4Tt?N&bt&9xM9Dhy_xnpN3clPw@E?>*6S2Dr+WO{w5#G6ue$LbqIG5rl zqQuboCPM7gRpA|VKX%GId0-CO>1VQ@kG_Paj3b_Gk4NV^+d;Is?5bUpDt3*Cmj=l< z`bQz)Q9X+1aG})y_;?uHr1x;$f1kD63+V<05d}Yg%U=HYs0N2P0zh;A{Vah#&r?r0 zCvN~Ka!12Kz>>f%cRM}MA^hJH4(WO(edWrG(OFxCMG%~t01eT&{D;V);M8;sodQ6x zFfN)dQQ)E(gIqN9PIULSTkO~LJe$?G@Bh3?V{_3AsI^7QjM1+zR<`3waEdqiEvnf* zdsVumTg&+XVD@0{H`2{rBH>vsKkSy)ZT&jEpCFLqwUxbXYxFR0K z3V0HjIfeSSCu|t4UaKhj_Z^5oyp(4)9Zge99&U$4rR>bOI?M#?q!;?p)V`owj_$3D z$+d9ric7qhorq*+OSkE1WI&SwIA@_x&49Sli69CFa8vE~E>#A_n0Vs5Rei{k&a>L2Bo{Z8DMCb zp-U`Ux=U0*kd_81B@~eo5$TXtLIng-kayjKd+UDw_qrtg9CYA|t9M*CxlQPb~-pjCESjm;Wyc$h|rC!Bu)aL7wb-_Ya zTV-%5hE3JNf}iE0s+!vNk<_Tb-e_(%>D=J$FDhvLYK2<$z@ZY^Q|)h?1vB+q+>9OZ zh70B1PgG5*eq!Oh7_yB;>@Q|NlhG&`ec{Qal{Q*+x}dnn#JyIxWh+>A+jNX*7Pq7F z4jpcr1aDt{HG?{g->xUu09}XJIo@oE0HUiq8U+C9A z{O66uk^omp zOHDdJb3tI*gLH`9%?Q3_Qd)~&jU=Cd_<(RuADOB}8T_(RMB(%Iv=38*PXkNXD(+Xj z2iPiX;6ffXQU)C~++$rH;oawPcUexRcNE@DD!|RBd9ZR`4*!HaYZKv)QzLhvrLjrp z7)dJj%k6cGgE^WJfh(5~d{QoT-GhU;EUT_~AP?9&6VG<`_Td5ucLaIBKe4`CTeVC) zyl@X3G=9{vR$zreQql2~6AzR%$c_P5A-FBl4z#`0KCB(GhI}((jNEEdojsgh4WMM;{EQ8G=s%MHY}d*qT0L zx%kr~cBAB~dc?_-%ywFWnVj2vEmr)Q2AN3vW!62(rvsMonF~NA>qc&()@%9}>)n@{ z-RH9x&~DVvXvhv^_N0S=fAUGSX&E5y0A1QqpJF`+UM;CY3_cFc4(yvAAnS@!5 zf%0!7y2-{q(+zDj3~Z@Cwu?&A6wRfNX>U!iQI!qVpD5Iovf6y)AJ`8l_$fe)oxy6DP8vqRe>{?i9_umvuge-|G7xF-0>$sW^e zu}7io0WQG(_4JcDw}o0PD>DuDTz=ZoSMyh6c=o!K?Y&&opy!1h zc6B~rj{6>dGMW5ZI}&BYKBm5q?e?gMQn&YH4wBi^vxCH*s; zatsi*eGCGTf4rC`p)f9HK9TuCppeQu(AOz5m!*K_Kuup4#j-T@%vt-#9 zmaLc%4xNg>15OjDayhn%ttSnlAb>;68FaVc(l%b;nfCJ61)CGnQbk8s8PXv_qom&v z5i#pbZ zF{Pz@txYI*3I*SV=0Xntfpj}A>GRmj>26b#yM~g_$eMWfh(DYxG@&XSL9VB}*H%67 zL&|rAmSzQOFPexXut}(oH0y~*=E*Cax(O-6r)%%a-l#rz`@_EVmWbH3P%Ck22^r@t z-WoC%P0@q7mBUtP)2MB`zM1(OO5Z1d0cr2#X>euQF(~;^z4i!bJddR8liXoj3 z6rBDdJ7wcQK;O#*rXc!}J^zf)se!Nlf= znk0$sW!ye%AvIB~vq4Q4p1J~3g`{AvNc2zZVlu89S^{~I_qsDUOAklN$b*z3K zi4?zskKpbJBbxhp?hHB(w6`#!?#CXLAd3ZzL11onK5+0y7%)LYR1^^L2qYGmWcS}& z=Ou=HrUjr2XQ7OEMVw4o>?B~g;vdiq#H@B>V|&WiKkaP+-=tfkF30H^eV@x{Hy(ZNYC7M;cqI3$<~55Do)jK+!@@>g z9SK&L+@{=#agY$SixmBeXL>j24niwdKmXL+&0GqySnIuaPG%{ zqwSi1enaD@<@J8IndOV|9blOt$L6u6E;#GJvxI^|#vK8`Pg-3EQ|cb|;~|Te+MBmN z>3;#DN0t*+-e!&;a)z^3E^E=4WpTjE^Y-_?UpySHjKQv4eoui`(Ntq}|h#egW6d zhl=bi7^fE#D_W0Us5+QNvZIf0^SAT+yru! zwI&6+{HYOjr0H@1tju7AEGM<*io-*-FI@raJvg4}w7LL;dl&RBo=;Fbq9(lqtZ#+| zueZeIg&S8gP8)`aV3pBvfbkEvFC|S!-?zj11CW#|5YZ!SLK^`%VgH|`RB$Zd8vg^J_}V4CcP0#AOV0(WM-e7 z={pE0N7P3d7oS&+_!@Q5sxU2`1~aEnE4gDVTaG>ANsdxG2iyq{U$`T+M&!F zKaK5!fp5Rz#4HzM-xB2dy#hjxFD&I%yCE~xMZ+Irm-Yyg z(E2Rc*fS+uv@J=NWxEeNI6(bUv~$w0gNJH}#HUick7sE34nwC!)RTY;j?2Khg}5 zy%pnh4pPZV^+Gvz&t!jgqC)l10v-80j?g~FTxm5m+c{Xl#Jo`Kh{O=pRN{0*`IsP5 zzaDA2>sr04HKMaCS2nC-8hS@X@6J6Ej5AeT4N;}$2>9=CFI@+)$Ai-1KT_8i$y!~SPgh+Uu^y&MW{PRJ2*O4GWf1{P!O7I8 z0yBr{yCrdq)Yc9%1E6y@Z!3a2vFD#Qku#qa{^-ZodAp#d24~!+dOF#GdRm3?fSMog zDx+!J$s#VkGzwiv9MHLGannf2FRYyn`Bohu{3sS{K?|)@Ekr=`L*2cJozg%W)+ol*YRkI8_lql@@wB{MPzd`U4@a#qvl27U)6IxLyEBYJSC7bc@|5f76cs* zK_U&;sGI_xuQW&9s&L;UrjE6leFg!R4L`p)|8#V7_cMLs;STQIY$@#f$OI>_aAaWN zj(w_B!F}%t;G_echyWnw@-ONFj9@XYUbjTC))S@Bj!)B-NUgYo5gB&P-u;(-4m|px zWzFlqd;tVi)YzSymFSz$%JK_&R3QpN#%^BF#@f|}h9z^8}|inEAMES$|Qv+;lg)WCA*NCaF# z|2$MUAV>7GVASNVP5;4$Ghc8u~#D6V*@tv6K(%F0W2?Ns|GL$lJ zwWz!uJH`5HDbh(fsQ zYa7t%0nQHKAd^#+w}hzgo(@N}{NFoK-)LkKi$!n))(@w<4Rym1?t&ALzGy=EAqdDr z00G&(zO?mWwOwZW-GCaeVhz(NE6FGNX|f`>hj8S#tM9{_n_D*e8R!!6@i|AkSvA$V zZ^tOH%#%MZK6in7@M%=pT&+`Mm7~1w?f|eO@+X~fRNwOfW+3%}2HPf=j)#;ceSM=V zmGZ{=*0Va>xh2T5-Iz5s*1CXXG2QLT$xV+wuGlc13s(z3rk!t(Tg=cDU3Bmkt8PKX zBYuT>we661$qXjXoR?LO@>J^?e5|8`tOWbx&>t_+NXe596!c9jxZzAf>HTzogP> zhKOAS`5Y=g8uA?O0Y-}4t9Phs3MLXc0d=HWBFDdu4o@e7m&9O*9ZHz z{eKR)-v;`#Ha3@Lk^QgBOU{nIz_YR?!d#5GTM!^W$XMvjgoNWXO6q=Vc`typ_mTl` zVj)&(9zqgWe=n)^EGeuJTWIFhhbP@@LbRh17bncTI+KYnt|oZ#vH39&UhBetoUF#S zX`lxfa-xQJS+$uS3Kp1F$IkadQ0*ilXBGBJn0$Cj5tYq%n>cw-2~XV-?;>?LwS_^I z632I=on}3O1ZC2K6EEL_%DmhkSQP8e$O<>8FAq`&bUnbOrO6q7LPUXfzcj0e8@G>D z%F;0L*>B}O$bhCDU8jb_V<2V-H10UoVW#Y62YQB3x79#eSN6XTR9{2Vcv(w{5kn>9 zqVEJAAmv}SXe^CCg>NV+4eL8O_<0q%_V2(6P`1Olns&tMT2gx%L#YJBW^iAp= zgv8Yg6yoemIfJR{b?WG{Q>SgAs9uN@+Qz$x&9wHx>>715mwQ9igVF8xE3X9nujj7> zT)@#?e#P!{)$^txK=_sjd&RzY`gWKFcB2xWh%Hv3tl+!BaSS+Vw=VIAAESoGbQiS?m_4#p2JE18+>rV<8ooyi{EEh{u~|Jqxa z@am=dz;O2K{Zt(Q1AW?h|~t6XsCxwUar`nK>nyF+YzcfAq|%Psg*n zuT{kJ!xY||wr$(53;|Q?!Ie*7d*-^LSTvV=XJh&t@ymUrH!S^nr*(G;YhHnR!*tQ>S?4Y7xfrHv6#s>8_AtF zN$h4cS=&H$8qjlHz27wblZ#pf>3Vs@^;uui$%PY@7tP;yoUvn$2laQyPad%_@mq&* zz=Yo0QH~%exrKp|;d&lA9 zH|Bh0C(Sz`dP4+X5vrq=(?D&wzXipDXY{du7vb$YF_4x zL;QDL;!3jB?n;1b-I9`86a0S|+H&U04TC?pbPSiTRwnh=m@N_#a6hAK;(oXhHack4nuLv9S>OJFNqnjB?zeKS$Q%&FzbE4kUPN#pVF$=+)u5~rM+ zc+_UFHk-(3_lkT12O;(7VWOFVS#g<8fs@)8x&DZNt1lyn6|mQ_sDCIi0iXAKN4F$! zQcQgGm4)nto{7z4y>o&eogQnaH4d5QiJ1$Rm3rQ>Sl()hPI(@3au3^T$ocbmre&LHcl@Q6 z$<5P+q=fZ!W{OkC%L~U}vDt7K6RCK|bbqv1GG-K~V$<1w-`{M+Pl(U^;so3F*VO&S z=8fyEThcS?I}Ze0aGvHpifl&GuutuuS>dt;Z~i>}%>>E~f}ix@=AE~6$!(_8Gj{SX zWd|g8KKAm%%frJ9IJIDu0$R=<2ra;yKDzw;?o?cS=5w-X@gs77wTXbjC#!PJSw`Nm2PlO zT&I$poNa-yDZA5f(wV!4tZY1rZjV1#p;V1qug=W((YjaTV(7ay8_AC3HC!D#2@}rJ zgttRCgYQObGpaM53z}j0qM=`Wt+f6|t@XKTm>xdyP@41B>X*9wp=^9+44_P7F+1_A zP^e6Q9k1vEL9B34!q(v0<4+DB9zH$F#5y=_c;gNb=VPQg$F7Vo($OCPG`!)OXfH=s z#{i_d0Y;e%O8l!3_1I38DsWR#%=apnNu|6-I$&U_3(VJW=s$kOE5b^Ot&@hd`Cxo~ zb2mupn)%y>C-cF4#TPPyu~&?~=POS*h9oU+seI+ngQ`TG`zZ8#XQzl(M#s)*wz~?li3@=y-*jDSM^6vAd_)&0N2}(eY!I*IT_& z9a-f0aK?6}mAb*it>&Lm=NY37*iK(Ua*r(3cTLQNXK^n~+=~-Bmy|^6KVdo`ldaA= zgWX3!bzP#NTGGp4mgVQ+;Xwu(<*GG;8rF%C2MYsv501SPdU=4+5MbGP!9Cp&&cB2J z|K16s&z+p}^rAoag4f`z!cIIyx1WCCTlS08e(>mU=EqQ%ddN!l{ZI-lM>Q8}j;8rm$Tk=Y&QrbfU^eMvf#m zJ@U@1lNd^T@r)Ywqc|_qi9a2OL%_g#$SYf{=G{xY-Qb#(1oeV%jV^(E7KHXa$wcw5 z>jS^6qs`{GXQc;0U%iJ+V_#t~Rw#hQzK59h^_(=6B9s+eO+Ykoc;ze5Dg!gt73{2o zup3}KAZz=vMpvLY_C~r`J0cx`9S`N=h#~m?aS{B+>PljH6Td<+X*3cBKxs!%FkH7I zxDzJwGe|1P+$iGu4)sv)`E_NS-g{tYg>^V`XHNZUc!sMS5B*Gpn z=+O9i5+87;HGc5QbIKXtcq`#qE(!<6&AYGGlhYPcg(s8n7m0Kwi<5X6KfTpHS7msO z#7~Y|efG;*5P#6FMpt}E>v zK|>&fF=E`{F}>Zieh7yEfA#?IS0Na_4OZn6Se0YT3=^=EfJ-1s1ss9)a4kh(;Pe7` zTFjj94>e%lsD=!QC03;Vh0@Unk@KUeZvdHA9&%k%#6|$a$`P5ic@Ki75x0E(rFk>b zbfZ?GCH%b9YpqFXA;C?tGw_4h5wDu4)JhR7(^tKU^WC-<9>(i_Pf5R)#234zxTs9m zq{PSlx+$yr`;0H`WnM5#!uLNn-<&m0j&h}hwBdRZyPVxyd}R7RIpNG z@dIFcX$T-}ubAZ>%Zy!EzpxBt?Utf$0nZ@r@0;>4#>eT84Q|Tw=o=rvRK^=p1d>(9 z0zob2={Z(%w)sMl~x8O9PocY zn1dnEj?huw23k-AVYP_76O?4D? zR9)MSLrO83@kG&`kL9kb59i{LO=*b*OSb7cAS42m^6t|kJW~wXD~i5C^4qC!vR;SV zHdJcwJFp^8GwrX;=CD4qqTIMd_w+k3AGQZK1mWZn$AJ~X;s&4JTQS&qtpULr>BWum zu!np5dZE#-E^rrrHz2hAcL@KNIsj6>9lMDsYG|sg#JAZNC<{V%&_%#N7^gLJGT%5~>-f;$fuqRl(Sqc17(|dT(fvvL&cAgM z*X#|MhX$czmVu~qcB&bH%*cBGj>)!}6^y!M__b)+xY}Ts$A=!Atx8|WnBysS;VH#q z=ev*9c*eWw%~RU1exjUO0AL$)^0_k**ruZmw!tHIyX0seGqAT>M&Sm91%O3S2cKip zA2k87Pg+YBL+(SMZGf!=6WI$eHvp`KV1$5v;GZFb82}#sk0eG9e5tb1F}DLxC(w}x z=LUG^ZIIvx9ii^3n)?T~2gQpg5)HoA0|QL?L#U4(P1}KSSe<^@S!acbekFRN3na7S$VM8+>BB=1 zLOnT36^xJ1-%IM+jn!~fis);D1y&mImcU*_zrB)=D#LSND-wKowKw99x>DVCeaq@a!R5pGotr*qibeU_f^EpQnd2u-u5ioyNL^w#$(K zynPVH2D=|oTG+$v!vyy}dF1H{*kL^${oM_v8_r{KmQoKnuH zP4Kt?@%|sTroT#NA+(`+7uV45veRw(2dS8E zdJC}X$NoISTPF5e-np^8%~ZHmn=g!fZ7r9PW#YEFl2`}%<4)kKr<@LrZ%$LU^dbNB z@n?!>x4+$GhWzv<7FbWJX2XlIAq7tTNUAO$l69+M>_?4RovqrV)Wa7C*@t|8RCbe@ zkGR6`_r5PYyM=AL=>xy6E~R5c=fV?RExyVZAbWuz;9c8!6Z7v@*oL@JvhFy2DT^QR zx5GL5vjcZ7YVJ*du_&kylih7tSlfBWellD3&sgz-wd= zVY2>+OKgvJC9#GF-g0YH#1VRd*LEgLP8;Z+ zXTff}6^CU{D#7J1^&NCxu4927@00HdzF@unilMVbfZ4e#(jN%0T^xa%6cQQ#d$$;6 zI=KeT%>7)rq_6H=B?l4%`0JiOQ=qzfmi2pizo99>xImQ;OX_ol5Auqpl{>yqF1|cv z@P*J(;1G{4#z!Qu>!y^x1u;->SlQL{W5iNp3{s?;m$;B&B#e*QXBxd}==Y)_hqz*m z`6Ji$3aLbKt>8D^(_+geItIRM6LyxXjqPe*kpA0?-lA%q>9`MgMohnyeXe`)MH$ED zN=VEr0rBfykVAZ>=x!da*$FlQL|T1{e>7)T6`kVT7NAswQS(g zNtaphW=t-=2N4R{AVMJujFimj(UJ3aucg=J&YCzfo9C2r)qcz>*H0;Js0kew^kEAO zkIe1y7*HKf92Hl&AOobj}4Zw72%a^er%TGln^5`{IhX zJ_IdfigjiGr(VH9n}B-7_ohIWCIp7r4Uda@$sFoMq9M0bD|Rzp#7*|f5@V;>cVrRI zL!M+VF?S8YN^~75X(cLNCjWj#-%n$waPjbPy4tF7iOXW0FZ5WZohGtDj^rX~nc$L- zg4(O}mYOZUt0g-Yj*{$}Z$8qVbBIZ$iqU=}0DxceIQ$wbjRvpnP1R z!$1Cy{Uy2$>Rni#tHV~sW(<4)S+7_>F#NV&Z7qQ_ibG~Q&oYF9bSqQf*^2-8_ctLI zPk#(bXMa)QSMi=ih;(Nx5|dHT$PvQqlTlugi0eX>640TN_f{nZbftCy^ zuLk?Y&8RQ?4`c_`uKObKaBgUMB}Lr>aHI&v*B_vPKQvsk%kYEOB;Ozo3r@^ARKtYC3Q~D^W8>Z~$1g0q{ z&`~Xm?}MJJkmn`eR7eh)97$)0Nl1W+uSWS@WLpelpst>N(5dOf-Dl*NzMdiJO4!ND z&VbY;TWzsEcA4sOs&DVcr54n320Wc7lA42^T%OOq&$#pKzJajK&3-Y}aGE8Crwq4$ z3O>DWS;0zg^kPTI#M-17k`b1sXkAX@FdWXCbkP5(LJ{L6uWyMDASYT(51?_SnD0A& zF89suaEA2ppS`k5k&uotl~7kKeB^LHm3^X{(+` z1XdL)#E)$tBCKse&=tag2f&JT)TH%(S&9EpD*@h#mhg0?eQ2X1X;&mH{~wa`xpGX* z0oQ%I*4-_jE1}>GWv2-}f6u@Bv=!dkNNe2}thYp~5=n|{vtJke>F4jtGixy5QYaLw z5OkJT;H6AHm}IL>E2dE!8a9oYFt~D=s`CVr#lmxsHfm9y-mzPXg8scil$_?fjT)zT zG12(TWp(OJo*pafN+uH;P)^8VrlA;xJ6bIt(kkgam7NCm{^%UrAk5o1!w%lYUxBwL zp%GN}7Q%x-`0L?)=yqQ%ZPtpNkaUL?EQ<;FXlndZ75(?#>goXa`CbnH2+W}Y2^tP; z=^jW(w(F1fwsvvET&uvX3U*YhLKHgC9QpT-3l<-`&mOz@az|is4>yD*CZpy5O2q)G zU!%=Al?xaL?MX}*H2+7_qrBA@h>1_+VkdN`FgUp+XnK>uW7KRt?d(0?rw@V!=Gz9> zIzm3E>&;Pm+9K4M#cVkWQ%yc4PZk}N1k}XT6qlJ!Wh%@-abs#w+}Ol*>MqrL?tKV$ zRLGd_ato`y*qGmxIG?5k)mqC|f2~6CX?ne7d>|Cd2bN{TZ}NlgKo6p*5M3T`?B#%{WFRDF~B*2iF_3o`)BF@ zI3<*2VG5d>a=LJD8%qTg%2v+9+g=$c@;v}J1q?|wLEGi1Fa<*Mf6I5*1Eb4zb!Tst zHaqM}Cy*J>8Yu**oMkXv=?*C5s0rno;KrR6MqyHs*g)D(G!qqeDif~{-HU4uULxg^yv;BDOI@Y?CdNz9c|xCL{I=FuV5uKt|5pqwoNZkAwGJliJTu>(%7N&?Q%^M-Y}gx4H9F zWdBgQqWUI1)q!L?QRI!LlM3~k_o>rv9bboFfm#hqSLc{TML2*=DOW@Q2(=Di%6ZWP}J>GE-mo36_?HjQ(N4w&hwYNjZ;A?@8MGb!^7WdJGap(K&_jZcbd|S zDlN!y|Gd-+kIrXKdB3`)MVeC{&++OSS4f7Mt!SCEjBlL)5pp4Z^5yw&64Q*%YDXzT zB&BpV81LaukAUkgxHx>L>4HGMDah*5*)898= zT%&Q#mAIMSY8GXgP|vI;%tzL{VqmH;^T-#^r>2dW%H=}`mVh{PZvf}}=tlB~8XsJvROCH- zyRa|4{UZn0m#2I2Tc{`^MaHa*9Wo-qMms|Y_|0-mkelWzEvaHXy0x%6YN%;ANSf}4 z=U$&3_P!uaAZ+?MWhFO(U~#g_u)OYrWc`dh4ii1{B^JXh9YJqXp~2_NUwq&~F>_Yw z*4Q9gR{gsBP7?h_Jexs1YtreFO?l_E#Rch;)9_@=j z-7a$CA$QNF>fr}W9uYZjIQ!jD@_VA>_&ld}R&yj@oO(n3qd|+8SVisL9nzSF5E7%& z^7E|LhHV-?4M_p4;TPT-x*^UxWPjX2H$vXSbO@O7{vlwd`UZJDWq>!zn7<^)m) zFhx5BD9IBV8vnk0PJ~>{!fL~HFdXx?JjZkMUu!;~KK-r_51~0uGLaj)y2^>nIQJCg z-Du~e+Mj2_`NO|a?KeJai(0juoOIWs_VyTztP02zBhjdk&Z{?ask}B6I(4yRtF*Va z2yn`x($m?PoYm?N#MJ90hDa3ePqR(xM|f+Tqgpd{DlGLwsSWs>Z}gNGZOL5FA7N4H z?t7VW~qZNJYpAnEl_GPH{sk zmdtJPwZTqz8r34&Ujb;5S?^^M8c!PJB>;8J;nSIGG{$Pvd(QyFDYX>)?jy%_XVTrz zWQqaLZ9n1Ur(bY>+4cFRq{R2xj%-$g%HRQ{69Wez8eC?2LPpIkROO#4BV3l6yOd3A zHt7M4zgQj$rcJnKFK^MIyLGw5)C7x@NKB=Kt~vb37BC%C;%C#tGaW(=_s7}@liit4 zt-k-`yAA4W56;W&FM8%WEv1gFDq-DfKdn};MGB7n(#-;z{tV6jBYbq~EJx2w9{}mP z;y%;3?|MH4IPQ#q+|u3)B0u1#pV8T0uiTFTtpvvh%bUW@vxgC;bNS~ zj>?4ehZYw-)`=o&W`uCfLsRoNh>zO!n^!~U>et^HE~TI^V zEe*ZSp1}0PCm;y70g8AO@ITojAnef}Zo2MhbsCyajqgvcC9I0Y=D!7sT`146zvBxU zpk&_w@L1p#s2aCp$ECkKO2&d{wzydgAG z5sa`GrDd^y(ypztFz=A9bC86N2#jh5d2AM2P9q$#v1%}@bScZs8EKTMg_}Z#3-Gd- zE0eZlt=XmFT>R9SESZkcJ8T5u_$i{R=YP%u$!-(<=puFMQXKDJ>KZs$z28|h|E<%} zo*te6pbhtT_2m`%??VU)HGluuA2p^8fOLTdvZIYZTt&y;8=Be0sX}^kbM%?*Y)vZ^ir*^>BBvKLdNg)r%WTE#=JSB|rkzQlzi6$o znQP?QGgxTt-SaJh8*0My{W1MD{I805^VWXkxJJ8g38*#o7io~+(VX~dS!CEg&6f() zH=qR}*1X&>>1QQyE4EIfWS%HmXjCbB`knLkP3l{i=rCN=MYrO^!x7-mz@*o}5fPQ( zNd#BI<8LIMz+?_&U3Q*8Pagox%YaP&>xlUWWHOKKgq{ViJKgAWiL_jlO-^}cIbAEF zs&5Tm@nN_)hagU~pKDR%IcYT)aoO_|Y6)z1qJj9hE44bhw7nG$DjQ2gLO z!;Wmw8>cLao8ozC!I=~4shM3u*7A6yRje}XIq5s>muq`(Fp+ny?rm!_lR>6$lu+qG zpG8SDTGjnem@%FKwla;tbcwdU{li_zRwh)@BV6zz5=iZ^X~D7_tEQI&A=!4&S)web zZYl2vMhT7(J^^Ff{Eaio{1PS6%#F`9TKSm9KGEov>dwQP#XnEnZx=pN({uJ^h}%PX zfG%fSclr5UE}7(EuSlLVO=?2kmh2yuoH%`N2Oi$FOCEe5UGS36XM2#8U;c%OYC0OQ zofWPOwf4orOMuu?naI^=>bM`0~Oo)&6 z(G0v~!JCYl6t7S8yVOM`Stb=NYcX~G#}%Y&6{@xBA1CS*dw%X?Bscn@E7_LG+DR`n z32tujxdw@b@b|WTN!;bWg}+sZ=l!KSichaF;kn(Nc!%(H!YrF%Dw(_@m#?jEpD#jA z#IrrAn}FoJ{G<7EWm&D(Rh>W%gD_X6H!B-Veumo~BZ z-X+u%+h8ex6NIzWiCl3&)3DsAMB=gfFFLSNw^URkBMAMzYC){FsNm`oM16NAdk_?Wx;VIbTZ0KHsP+Lw9R`vSF`LZqUY^xy@vvE_muK}GqqMsMJF3<0 z4mR%{1_4f+JkZ>f02dgl8>{!;)U=_jNZS}|{YKCVqNxJk*?%+dLg>rDfp#|ih=fr) zqg*{^p?sHVCaNjQVS}<%KP`YGgQQ5i$*4y`#v5BIrXCQ*PP%^R(jy-&DrO(XuZ7}e z3i_s{9|~PZ=$hQ>=Z(7N^qkp=Tkg@!H;E)}2&#_KF zg8%hWP8i5L{u(~UC$W8aV;0Pt0w6^IkzbTQSQ@C5J@$11QN7lHhX?>%_JAh|MtSZo zFefmghK@RE4TyyJT)!&6Y^Wyt&D~&04_(%C9|f4*adQUEZsnmB_)a`D=Iep)?fux> zdcVpA6#_C*pb($}g}`nl-}^I!X@jhUoFaIR9pCDbAJ^ovb#?=-mT+i#Jo~&#abt3j zanEw&VD_7`(#D*m@b}svG{cUB@e}*Nco4`cd=Belczj;=%0=gMMLF6B0ySl{S=#E1 z-h#$9CaMnOkAnNiZX6-F6s>!4AG`*n*wxLduEiI)c^Ls#goy@Wl z^14cLaPVJCB{@qupl`s)8^DzaI>-S}v!={OIh9L%{1na0)~T#W$+}Y-E)U&h&OVWe z8}u|NfZ1kn+s$?~Upc(ddTnZS!TeKyZ`Caa!qlzl!3vroJFzcQKi9R&nxyo+Po2b$ zWL}%Q&1i>gH@Ot;z`|E~-YKcbm!XtL$gt?V!mIVwVtLyW*f54^pB|C&?*yKd1r8lG ziBEX^2S4V*kQws^sV0s|c^}AVDhCU_bxXOoc^@BIcd}_H5#wN0*9UG+SSow;cI==Swd8x-9+0oxhsjsY1>CrXiJym6;A-l9+UMYO!j%XhJix>t9Gmb0 z#P#+_9h8mpkwC{4W`lBd#RR-y00&1w2wIqPLGS6D^DT2~r4agfT)h@Ud=V>ho~krG z?PU12?tzi86X)XSPF?kQ1{hO-u^`?c(RFQum|mdp2wrw~j^ge`pK(WP<^+aTG=?sj zrEb;vNiCP}u65+{d2e=CS5ff~lZtEeIlQ9DqWM}qJyBd)!2YUW5R3^;8na*qqme7;EA0Nt`+&o7BI2{7M=RhJ(Z4bo(VBTBE}}{ppQEL_10q%lqJ@s!NI~ zSh7pt`TdJB)6w7CO%@1_;5r)8mcZU%g9L6F%&hPiDFgF%ck@-btjp}?Czm&#WkhnF zrRh=fP&or!{cjiza$vS&TvvLtuArcjlt~VCpho5{3Rl*43i2r_R~LEhyR5mozkn`y zJTNh(N^n|tvznP)^!oz8wz>RHeC6l6f>UAVs9B~(3}dv+-d&3?O^J(Ue}R|8w;-4@ z=AK>Pd=H__oh&GK$zdg-HhabNro|_B7jeqyLAHj3++iJOZu}I@p4!Kg#_ibr$v8I) zK~hV~Sebj{`P&Y6uUKcdr(e00+p)rLR4Q9TMxjaly<;%Y;wsE80He%QK3vu4UZj?J z6%SRWkgny}4bTaCwCn->23Yy^-M}17^OxxiK!Gp@*opYc`4Fb*9V~O;$5WHBsmlrt^{gvj6t9Y9vq=rqx7P56;_wz=v-K`!9^ARcg=k1p zp2)tUL-&aGK2Ln!>_ymOHvxL*0SKJL{n|#-F@-7HKNkvtCgOhk9UQPYV3ctDKviEs z0jX=_2<05Wp_u`w+|?X0YO44)aGDZ;FvSHGL-BPs65mTM0P|UWaNM4@aq0pIW)S-Q)aWj__W+>2!|#u?{EAD zT_D^26)~_#)6%RxcTbo2V?%TiPUZ;*P_u91UhO(zfBNt-7&+^BYIUI=7{}=@GRg5cU(HEV%C~k3Q#uG~3h_=OVG;xNCEe(Y%BuDJPNB zmz!bov)xW}r&9-ul*+A+2#Pq}D@B8Zk{UW^{F4jU9Ae@hJwg<61Wcg(Yu!(ACKjC{ zfZP_6tN9Pn!hK9l-}rq*`?sA)xYH#rhMTJq_HIZ&OC$X0ds>94!E|f)8$I*>&MB)A z&E-$2HGio<{yMZrBfMP@Fu-7hti-x#M`xrz&;?_L_Nvf37zUIu6HGQ;{XHW$g9Mp0 ztObdC=yVx6$paGUNEB;GR2xBJ7yZn&rw3+ZAJ=U))v7aIo;t^4oH?JfGQubljlUEU zBGukMrs+8rXk+xEU&zS1v48ZYZDP|xZohvTe(MD@)f|2O8l9H~{>y+EFTrr?dAwU! z16P(*4*piAb^80SmsD?6m6}E(^G@qC)Y0Cl(lLKbn6lQT;o9frZfa?g(K$$502B?t z4V!yS?y1E42cPT%h2)JCzssQP=3(z`?+k0+?7w|8oXFw^mRi_sS{CC&SPx-)J0;E`mUP}$c9 z>8L3Cgvbv0uJ)H|1f4>+@KSo23qFNK3F8a2-p;+GTKckbf zq2Qz+p829+qbRNF?P3H@aic=fu(GK*)151IO_q4+ozMC5v6vuuPo3XH$L8x0nJY~l z6oYZVS=}!rQqWByrFJz7=PXb%4%4#qd?x;R=mt9Zrc|Bl)b2MU;=!Uoi0We(#n!_S zv{h_@vewoEg$6As43zoLyRs7d0!`3;v&5_a-j!w79(sbivdylY>CZLLc-m_$PH(5A zK)&s4w!`5$eXmYew3C07jGNoo##vOC}hd@`wti z)AV(_w?S?LUQ&i%Pt|I@f9o!O_Y*dIvhTVxqBX4}b&OLRC2dU~UwhH+7$ptMPS`wd z(b}(^K2*Ny*dMEWBAyRu$ee+ZX|FBh4Nv6NjJ#!E6*mudo|y7z$CD;FdSEoB3k1wz z;hzr=cNE*J-#l=eF^Px-ZD`C+ee7@pQYhP?tUVBLb!lB)7$igZzf9&)4Hc44B3LCp zaW7)Cja24=r%(%cFwS!y2@xQG=PwV&!ODe#QMnqHQ#=8gLgpFhkc#o=Cuz@H+}r)J zN!(uAA$|lah^({1N$jgx&`CB+&J|b6vCaEdPT~!F;3wJ>asrym%wgJkXV+yphMdrcE+D1n4*QEIK_;8P!d>o z9;-1tYXYPvf+?=$KUG{CC4AzBcbXkprWC0u4R2STIQTj=g-2oul4Utzd5jU zHSDlE>kMT8Iz!>N%~B3l_ktQm>pDX%cui5Bm%Qp?#;^;pPJl2KbDF!%eh&YpCa#mV z@rKKR7f2d>e%I80z1$!%igX(oXzBt$0XXBqko&(^mcNQQZS1UF5J)h}Kzbcfo8wH6 zE{_J=rd^EPs4~Vwq|53lAKQoSDrd)JF2oNj_=|ud4xPEla zc?apgKL^=O!{ikpQ-L>&!<05%;cFyKf3f^-CiOiu@(p>{!H zl63zV{9s)H-+nBL7G!imHL8ajxTm|rLCzWUHwAA`a09{^7XJGyV+3DwtX^3c;Q_T@ zQ66w`&4gUYNEncSAz@%wg}w@EBmE91v~oFP^@Hj|E1w^h>dgd6ty=B!*Q8(3P<(h5 zWKG)T#~`eNsi$6n8DCu?wu0i&G9yQrE11rB7(ua6yzFu@chta zn|H)4X^CkP%ijH(@Z5LE{F1u(sv@edy-$`TcoPWLfsr1Bh;KX9iR{^SJamAiW$V9s z)hMpdCdM4{{zO(zy&x(@E=q>^&UC%0g;Q4NZCf2|z(meG&=4E*=!8OnrNx9<7D8%+DVI6!j{eKD!?_>$#+*jE1?n7D#riH)Z{ z90isPWJde`ci~Thg+Dd`1_d^04}?G5)zJfUX8mCRi=N*Xk-`lW@SZ%P%@u;FRGrxc zU;v}70I{uGnAlcwhVO%5Y8MHnc2?UG>s@ccPPIG9KE&sea?2%#W_EcMz);zv7Mdbv zb>rEF{5ypm1@HXSMuR^7Tp775I3zW&+x#ZAApS6-oVK)pu99|KVS@3MkAtmz z43+KF`47c)#l!jq->Rm5Y}V0GbU|rO;KTxHPF;J_w{x@i$5Pnuev8S?GTT$Ok)n+t zz3{UmsNeN1cM{%EsowU8TR@8UfJmZrpf-w><~ut}W?p!>_midzDI#GsMwG^p7kc#A z{vTcE9nSUshH+A4?~#l|BxGiUlv!5xOnmV*viB+}gskk5R92bUGs=kUWRsB{B7{ou zdp=*Cb2`7v`NPq5&UO0Zx!#}mc%J)y-GPesjuq-Z)Nk}YQ7>L$TXgpcwIU;o8D39r z&o^#ai?*zpVTl)MV6k>B)-)e;f3-9c@lrt{e2|S%xO9xG|AsCkGM~MLHo~VY;=#$| zd*-s4|D`l*O|g+(B!IPF%-gpt1XuBn-BmYb$)WuKGTNY2x`Ozjj>ne^(VzlN>nfbFqMiDvY!Mngv%b zYs_ropBwbljPK(<6D_^-a!oU|w$SR8k-10#_j z71w;{c>+dnrJbwMbC%7WsciB8WX|MCYZxpk|=@B2v1J3I(py{M=we!-y7 zu7&&HU4qRf*IIR^BZsuL^RDd@gQah}`6HE7>moO0LIAPd0pqw@&>MjdDtotK&=(1V zK40v5sh}{pP8@=Yu$(wx^}+}XI(5AJzy7aD7Zy$#+zkOKxH$1xtUegu)po62crua+ z5?&{Cb?;T=v+mrZ(0lrwtOLEL67+EkJ)@LHXQ?zYT4ebCP};TWtPFv++v^{GorF5_ zBJw(DH?8&OkI9Km?@>@XaPDo53Vrr^dM^BqN3v@9fKgC>vtm_Nxu<;YTHcTNnJa@D zZndW%mCbm&PU39PFh8(qIyQAa5MQqlmDE$SRk-bhCXjrk?wSs|Vn=i^An7hOh~T~Q zYMi8Fa^HKk0L$yg-O5|ob=LQw2$<&FflpvBYXJ7}SUEu0EtoeUTGQe zeL9Rb4x=3Tn>DcJXU))o_|oW(XEXss5)BTItNs4exd4$IoejgNM!=Y+2l8}5FsAVg zV}U$62#) zi!QrCq8Mz+5=Mq0k2s4L%p0)+IE&eOV&T$~PFS(Je>{Dp!dTm-@_fpH+SC1R@6$Aj z)k=A4jK5bB(JJUMU9_;@UKAf#O70@<;nQa>DCly^5K-=`p*qT;+aiR07Ml%dKCc8! zoDY*)@7nLlG&mSyx$hm8ua5Z$CHv3Oxtf)&7m62Uegj4mT4+zqn@$CJ>>{?0(Fe!G zS)?w=(IUDUQ{6kVTLE{W-9+u(zu#{9D;g~~(VzL3y4A--FuTfk+kbWzWS;C)p*^%X zG5A*8$08~uCNecpCov=yB(P7<2oKqt*ETBVN}I(b@C?Z90A`aCvAKS7*L(IUHL_Gz z=og*U(klnYTC@xHai7N*g;2zi9F$ z?)5)@I}E_8QjNGU*{CCw<`{)O+?o)Nt2a*05nmHZ*bWV&b9)D9S=zx}^6x)iQ9!Z}|4?v*?`d$XkrX{2D#7Tf!E1lj0C7w6rI6|R zX}PKhj%jit`i^aJp{!Tp$ibM9Ct0bdBMw^Eq=X{jRDmGes%&pD+ zy}SI^n`$zodD_<$d$!(RMj6C#-ZXyU_5B^UMd$-kthDZgsb`%4XMHG77QUfin8i1V zp9b+M0X^|75mY(=K%QGLl6sOZ=dC%l+h-iTTOn~OfmfqHQt05z2!{pgS+46UBP=> ztWPepjuQRp`Qv3V=a2eGWkCfYj?V9stvp87S%OlNiCaUr2B$-v(w}*l$y|njY4G10 zQ7;FW3-VsnI<9t{g%qngo?ezSQzPHAi~fXQ$b?bCks9$ z)8vgvk11l;Tt6+O$dX!USDuts;kDWBVc}FBix;45?C{1Rahjsh=mccWg!1(E^>WEC z*?tZ1=XoaTm?HV&8)*x(7dA8ZS`AU;!`-N#uZG2HZed?~6ai-+MtpGK!m_r8@Hk!> zH_%!ES^ zTzV^Vov;qWQY@Uc#E48o%M+&ezr@Z7eQa18W_>nk_FXtcI^{G4p$$O~%MQbnjFDCI zTFuVT*z$q$XmsvFf1OBur6kEH7wL|QTge5-oa%WZY$C{22aQG<%Y<_5s+lsKZ=Uu$vy+%8j+C~3yz*O1ElhPTs(pqH2&SLA(bg45^kjQ-5GPBhmi z*wA5#+$He6={QJg@`Xr*Y`bDt?BlC1;Kkn#h(!PX`VJf^t60Mqhql!Ki8zwB=b&N- zI(!J8-d{yv&O`ceh7Ch657k}N8P}mO7w>&q1ogF zWVydkraP{57*78{@E0k3I)>3CKv0p_#@=J3A$PP#{MAt&_PKM`So2;Z_ZF$T$Yj<- zlO$Is8Hwke$Hba?f0M_H$hvH996b|O}^5|$Vj{19J3`Vw|N?wDI?lc&x z-%GWA%qdo7aA>5|{rT)2`0v@{L-gbu&yOuXm+#li@-$xc6|yTbgYryOv_rU4 z&;@q9E!rH7!k?&7+zh>9F= zBZ5f^+TI!M3U6NrJ4Z`nFnwSMny$w1(uHO9S6=z3Dw0<|o%>ZMp6S7%TO_NkSTfmn z;!7LpONMp76x}>qqaIVM#k*%-x5u|i%1ld>R?;QdNRuXctSdi#madK8_U9?tujKf; zOyy%+!PatT=DXnSST^R zGk~m|dFnhi6|Sw8Ut*y3Xch=ey3R6nw})JhTc(u3z2(&2)HkVD^*j`>9&T*^M4+_9 zDMnJ*pWY?qLCM(u>&TOqft{O2TUCWuRpp92*w$R(q4SsE|BoZDb?C~5CU0m!fNag( zbIGDHO&ZCvzZKgb0}#dbr}gkTaI)MSzxC+NjIE`@Q@L&lAL4iVD+%ZMd&%8NyY*UL z1W7V%U5&J-ACBJ0)`|;mYV~fH&b=)3mWm`pAg-umN~`v{@_d|*mqwcw^=vIj>n7L2 zUVbYaJ=<8gNNzcC^`$Df;oCZedCv&M-8gzY=D3-GrY+a0_LS$ZRT!`R%3{@-Ac*daTnil zDRsk>p;qKjs{@}PmzA_NwPbX7<)OU^JS#r<>i)3Gnlv}RnY7D1qY{9D<^UkwsRk4%B0gOhD zO{JIhEZ@p2KbhpD$v*9__vvNd%MLZkSm|-rMF;i=PtU=w^e9W|LgJ|~mz$0B+O8VT z<0k$39mhK|qhChnQ?#Wmu8+sQlxE`eHa>gH={k{*5<3Z>^5d@77mZ7c&$vjZM|&9U zXy#JK!z$7@BRWh@*uJ7wTkzgD@jF~B_8T61~r2&OMyaLDGC;IfM=l8X~jjmR=zqyde^B@_{2Ps?$)X0~=wNpb2Jg?I0$0 z4w`8wQ`8Ly9I=u;;2T`aaNiLO73YgqJ;Wt5Wb`$p4d`6|9So|jz3gKt6?Geea~(#ZK10csUg|Ww(*IKc$#gQ zi-pjT4UV^CS>H6@*GYrnPGmr$y^8Xol^r;)wz>(Zf2L59;-w<|G&fD_dKjk^3g}5YaX~ z`=;nHDK1sX$Yc`PQyVWyQH)WkC>s?rRCWx!q&elFbY02?RVk2@Yo0!$Bjs!_ z+O2#te2k?3m>DiE*hopZ%M&YT`0DgL}8PmoRDnHhK00DXTvA)nWw zqos%3msbE9zg+9Z&EMs5)1jTE%}Mk%o|>Ij^5X*JH0;BWSNel`$UiLa-Tny5UvB{3 zV>4|AgZ><2K0Z)LV2-+hK(K7B-9YilwygS875%1o!yjut=yqK|&dflTN0JfTc z0c_=|&s0fQMgKTbrhPoutxC@9#!cB$jQKd@<|t32V<&58_T%;A>Pel+^xC@NCtVrs zM+)pSguQLfROnEY=@gGYgj|W6pW7*2M9*hQ_ytG;04uWM6p6@Ls)rX+t|etfOq5(& zH8oi;NJ+y>#<`3U0Dp=2XtBGuvkTLg<4v>2;6Uq^se`!##MvYLZ_ssu z*}>S{0g^XRpe%qQsbO^EnYumY1q!?U(Opru=!TrQc+#tBct;^>c(-{9GY zMfup&;YzXRlaLD}iOnwa<=vFms1*Cp3dzD95gqTyX{e4V{OIZ&aD8%xae#xfg{^}J z?;n4}5Q3cf&+-fW`EU1U3(zwC4Jb#MqwM~Fh=7TOKCpGAWD5!f6haZWB=`S-W##2Q z>)KqhT>BaY%|QU!sqF7@h4bZb%~-~~c_Sj_M_esK*ecj(z&;}4u)xqbwz5ZyiL@oFxN4yo05coR z)#5YzPg0((zKG9=?1q;6p7j2)6AQ6(2Y7~4mA7!ka)pho~ zdU;3Rod8Nh(`=W#_4g7_i`>j|bKc>G+cS~#n3|?N$m9RTG}vq1j=;x?d#de--IakT zk=I5=vPZ<|DK27CO<%lb z5#@|P2o~^)`T;Jyn??bo1s)1d-@KMkAbuy#6wfZ=d$((6ZTS{q_t!H!ohrwVJ?-RwgSB1c!za6^MIDdZs?U5OS7(L$$QnhjLq*tha->j8{f2jT&vI;Z zv2>^ z*AP};R6qZ=j{dcY65EM?ddhz%Fzms2+TJt-ML>%Rw7ed3B=OKrKG)7H5#y+P7YBo#?!x$Q7QC_UhAP-V{ueL&ifqv!hNA=OvA;RD4@Od8Rep zcN#BN^ljVo98Ifgn(Tpm89Vv>M=L*+*G8+iXPv}SFhNA)E66tqQNf|NoGPmYcvxR) zxQ@Rm%#Bq_@f*ORfa!GSarTxkg*$t4h@*?P=@+GsZ~(9aFrVasO9U>iuF?v;vRap5 zU&DBP|Gh*uhe%k8JmfB4djb(K3@sW4;9x~2W`6|>AV=x#J&?C|NUwJ>C!z$^JHY3b zJeW#~BIP&t_VOPbi*KO}6za$`Rd_7PpUx$Ga`@1hr;Fk5jt{TI3Md}Wu6nOB=pHFe z$3hT;Z{0cTZjdmWOiH)G&~|*weAJ2Jm#AytyO@H^8$nmLa(s1)TbP!}-2{aL(==ah zQP)#!#TOnuAvmwJSu=G@rM;7ayJ6SLi-hu)NpXov>N|a=Xq+%6HjuUtx`XFXes|9f zMBL)^AY<_J4?(7f7vSf>8><=0&e6pS_HJmcn;{Wa2uu!`o6dFz41oU=4-R$;OHbsw z0#%Ru#^AUAzk*QS7*%Ajs?-M6uc|> z8qCDmbEwnN!)9PB#lK->aD9@E!gv z$D(?{izeqx)5SbNUiB%0rMSGUq33St=DKNlSM2OTwmkt_pF`uG9Cx z6GC2)sLyn?b@NVGk5S2RAC5h`=BeE|p+3v|-5dG56G@Hgf*|dC-d^@C&dX@ZlS8)1 zV>hFnis+SQpQ=K)x!bX{BgvA8q=uCjEBDw60~7Br){n1NP83X>FN$n{C5Y7MK$?uZ zHG)8K1X0>0P{%{wxv>l!oCx>zk8V%H5FNoev zJ97%7FVf4EL-@O0gr_p1Rj%vqRC8j&8rY+=&Se?ZocHMEsBLO}CYTqM6%E>(KKJKb zVqe{cY0=t41yMdWuE`$puh{Nih<|9TnABu@`59JnHk=Rn?+fb3`g@)}b`Qq6KAoJ`S(6<@5q6A^HxFu zj!mWn;nkny1Ct6vQVDNRkskxQPEzKEMg;N*tld+_=Q4llUi^W&Kle<&Z?WR6%2;2q zS1jZ;B))yCr2OiafQsRTj|>Xk$5n3ipLFA)59v#+%NQ`xv`Kf%FTJ9hthgw<*79;Q z(~FZKgrE2#ig$2%@MB@Xr)C@(k?Oeh1s~s*i>yalMAcRru|&nyVBIjz@sTxle%=nVwz z5b76FMtA1T6pyUgb5wh9y7k}SY`fo{m3>`oFvapT`TOMcl|t3t`0h$J+I7Z(zP8io zWSSlMB-Q7aP=l#?QIcEIpSRs~a}5AbI$!*&ozW$c2X)pDnPI$m>lEJWPXzvlH}Maf z@Gv8cbRHQ7Y+6zTFQ~!XEJ3=tx3?M@U|EI)&@p(1O2CSO=jFkVCpqaWKoZo1o!=G! z?si^iCpT+A-~VNx{6j|2SpKSE;?!3Q8p6bCX&V1b|1uwxwqH20dm*InK2(|4da9m9d3vr^`$&pNl(eOTp zl;bQanTnaGbbs>Obd_+DzmHzGt08LCzAF0c!VJT`G@5X&>g-2(Bs%RJrKiu>RppP? zPK7rc*j2TiJ>NN4Cvh&chkOqEZ3lEH)?e~ERiYZKd|p)x1jb+Bs~(rTI#ealMB=!0 z0-A)|x3JGEILbKP{`{D7SBVrXPHAnCf>0$r_)$?l;1P8hb;CxM7Zg`k4j?RX{+I5` z-P+O~?dl2@G*>m(G=}c9H3VLP-07^w{~Pb|;qO22TH=BRp`59kE0Q_@siUsW2;1?` zH5$hF1anRMayT!V4#1+P+o-*RQ-${9Ms9ZY?yWA(?&L^+^_Swzm5&nB_p5KObVV~q zJV@kNr1_5ZG%2Pj^i1C{2X9+>pfg*_CI4nyA$y_~bwk^{f_C?W-Z|=?li<^Ajb2|x zC1q`1=t*flVKY_Qi0iSOe7!>3$a=(iN5{rIDC|+La%b3CP3I)2I3KeDO&jRn!HQVs zX*~7SM7oIiS5i9_$@-Njg^24`lLAfHR$EUXZ}iy*l*yf-wLJ#_fa_bURfv}*+=qDx zrXh47*xKI91IZZ?=2f%yMB56efFb$&%>!dVtcp%Py-{iDs>KvCK&H_|sGQ z;^IUjB;_`-_T_zLL)%GH8_r8e&w~sZsa-30^P**xzD1liY#@j=&YJ04K$HYv9iOM9 z<6Jw0+#k%gCShFkYj0`yvS%NC)8=u`p8FCtDT)!uB8ggt zwSQJ~M^kcqx*1VR`o_st#sJUo6J4h6(2l)$klVWoyAx!Geo6TvEi!UV$B%5KUpj0C zu+A1vG>2K}G+H4hlJKA5y_|fFYQ<1GN5(a_AB4##Z^979N=`(%3FKfQ+RL{pb&x#skX&}#< z+~f62@$upWip@bK`7SBn!y#kg2)pG{BEviSiwzYdzSFpt$5y=eq!twLZ_gfa`gQ2# zY(Nkc0(9(HswxM3lxXn7qh0oGB%oo(FuyP#o(r#qwKLkn)(e>sA>kCiCq=f#a9&zF zBKGin44;ZDaNnyWRc8AreTPp+e#_7Hm~{$;biupEt;5<@g!<+0`oEaed2NgX5mP(h zGG#snZ6V(l;yDf?Ik_~BpjL_knqJhX!d05YhUAR&;4b3`$^ulq>tnil=9utOCyGE( z&66})XPWToN-pK!a8v7ix?-F=P}(M>qOV)6yIGbiHmbOU#2x=nx ztZd1x>G%_u+%l|*4|Qyc4aKV8%B#9-qu?wZPs*~l+fT}Q+c1Kv;;JKl?hq7~2MT+j z8dM%_tp@ykO%-ikQ#4uyjUmygqTOJSqyp1h7;pa@*kYd|d;zQG1e-~CiYH{sAo_1a zIB_|J7*vL*3V#CLdwy^GVI(DCkGPazr+nHe7*1MWt0^YMtCZREEAg>?{8Yxar=Zt- z#VH(R@Bb!*dU*llxpMV+EFHqpN_8J^u=dBOdDnoft&Pzkc`@Fl)60V5m;pEmizno| zEv}fo70#Ow(I9wA-=2HPHBCoS=XA}DXXjD$*9XfdMyuPXYIzb~{T+lScB zM<@Nq<>No4C(a;(1-d8%s;fCe{;ioS;+F$ym+NnI@88OtqDN%GSL@S2C(aR(>Bs7G zNIID7D+o{0%PHE6F?Y5)sTE!FG>_fWVRT9J=2$87PM_K?Fr1)JedIBJxaRTz1aZCL z)0_##c&Zx~r1i|(unlgMMRzJX)Wr%HpVNphHW;k_9xD*Kd?oB<#!YeV-miTMDZO;g zVGll!l;^O^Y1@@G-^<9)CFplAY_qo+C85~NYCl`o(OoAY6pF(?2Tm}+N4{T95UnLc zLEX}zZrya&84Jf4<5v`^IWjbI&B+pTma6Go|0rbG^jN%i@T5)l6_} zj)_53>&-yw36=?`-TV1@^dWAX=kt!gREUK(cZL-tB3_&KdPqwhy;*Aa)!)95#_WX; zO7w*nSGA5w?}^pz>dzxi6P!FC{zkR8OKvy{WjJHMnSuL9+sk~XItmS^!WHwj83k|M z9z9CN?AF)|s!#6XkYMIXiA42rjh~;wJZX>-UguvX^@vmA*s*q~S5UB^ zyaFHU({vMy==C+`h>3~A5W}q!y_3Ipd3OR=6k>v{Xkw87k^gjM*t*!DT)jbA3Z1Wg z*H@$x01U`vkwn%Qchi(e2*q&wN!!X+Ebeb2q&tcGfjSb)+WmELhOg1)`09l&Ta}&` zR!wd)JALeLmr;*z)Sl&=x6BaHVc3Q4xAz>D!ZXg0T;-7Nlg~x;;})vTXB)%h>+_Z2)Xj@ z6yur8$`3udOT<`ATblP)nz8-C0+rkYuCMg97I7E+!Z=0A4u6Z<+jiRaGQA+WWRW#= zxm5Ou31hg8vDIzAPWN-S94tI<`nGwDUwsjw*Bo&+t3WZe2)mW7d?xd4)myomX=&}a zGax_J=2$`sgnvJvw7Im#_}M1`%*a-p$ZO0bA1}`}Q_TJJjzPL?R1URGy}QnzF>qv}ga z?J|+mQQKoDL`Jk|YV&Dd(O2H}XUW5UklkDL-F{Q!y>mZNjQg(D2!qGh`>moe5>L5gMEfAwu)988t45f1Bt;#_lRwS(y!16+reAqW9@ zz)J{?Lmp=f9yE}Wo#B!N6+;|5$o#Eb;l54!0>fBPkx=}R2y{k~5_rKuoU4s^!D%cu zOvbl=eVRUFN^pT2u_1KgzQ#)}e_FIgbhcTHd;B94!;ol8W_hKaq8mNrh-Pu0DGFlH zODv}#NxJ|3QcfK>5f-(tq`OvSNj)#`@7AE_B;*iWk8fXv*m6eR$~jJ&V?E;G$(JK9 zEA-#s@L9aJC*UeXvTk*bI6%m?yzpD`_%Fq&LtRaJhY%9`rqNPq=J%)c?Baosk{J|y z8)Ydofy5rtic^41b#ra3COg@bV;<%Cdk0l>(Ouz4CkYjge zmL#+k>|xbxd-p_}L|E7H-DIrWT&7=^^^FapNh}J=V_u=3+=u4Amp^t4?~OKb_j2BV-CZcB@rr) zZ*phG74s!URa~==9F>?@ziX3f>cz^?c1grw+b46Yb7--MwO@3ZvAUea3>RIU#Yg}Y z-uxFt+#hAb&{zaMRXPwWW0<_E%vZ&klHIlTb~Vu|jfzgPpWo~CIX|F*F*)O1CGwcw zn*dX0`g`agi?#+|gqk4vMhD)R+^uaOd-EJ}skAWG1gn=Zgy-2Ct0LpBKYPew{A6EP z(iA^4sT5*!9eyCGhP05O$&RsD4TpWr#E1rwH*)RL7+Wb_V!{(_@0Lby<Rl{rR_qw%&P=Z4tN#ALLH2+Vk-PMBeE+#E&Duqa|bf*nY^&S#(ykmoc=AsSrX|z z5taxg0*&q;{*-@zU}m-`J8NNKP)>N*xuAFv|IfZ#?q3r{tbZfy{y=9TY;#8|Wt6pm z5I?WBhUUNiH;m-`Kt-p5rZP0vTtRHki@^kec^{skmKcuGKf2%$j;{0}C1Km4dZutA z@$%Td-#~qJQC8#>LHy^tEr|3EaV+3vC2|;)Ox+}y5gALY{PHYezJ;unzPs}!x}gek zOde=2(>b5W8oB>D%C`PUa5=8Nzo(sezyIP73qRF{6{d^WR9STI;@k5_=f0`1sZtYJiUcc8J@ z7Fbg@Fp)$dKm!E30@Gd$((a#Amd%W#53FlDHX;{KL+iXg}Uhs&#YlZ_;iHjwcbK=W|SK9b776+!==@3&DmYGRzbA_ zkD7JPaV(Q~dwguKo}Z0B2B955TeVoM zpPs8fTgTeaM&Bm)h=W=PWN&$|w(qR-i#14KB|#g*IHZCng7hTaXEJx)tjuSG@*JX>GTLF(y=wW1A9F*y=q6Y*+kyw&F)8wKfSD0tFFlS4KNW=eFM;s_=P#!#`CNd_Xu;SlkK_Cz2`*aIWttuv=UTC zt}Dh0Xnryio1IP8U$#xV=9hSZA#{Sr!G^ATOO3B<)sA}pLakRk>0aQtdAJAEYYaQCo%#s(9bP&3`<-W+; z*qz@P0&$J)AhL45k>;6_`QV>t6{UFBgosJ^g+)=u7L5z>WBBypa|8_Gf*+4Qu)sIf z*U)vV^}hY1vZ&M0Ocy8sGk2#x=p}Dm=pSvs6LsvlH%)Ds|k&J=4#pY1o2~_+^huez#b1 zmy-H&XD~*Rk9hR5!=#h613ppKLlK6Q*<*X_Yqz-@h$(Jg{NWSC%(3^GDrHOxyN~Z7 z)C~(H{a6D0e--aPg)FemIU+C#8!s@in>qZSlm*F=KJeP3c|}DVQXj5r@aQV=YF@si zfe}>ID>hyvySJvjbt5{-DA~QFV(QWnIV&zEbhLeRU@3l0%>CH9ySyJmt1ksIjZTwp z{Mxu}EHa_c<1<)U%a?{`x{GbaP+^v&{flHEvirr=_RjV*w`zhm8=c)aD4ZPYi}*A+ zOqFa|v(;L)+Nd^hGcO4#S{zxF{xnKQ#!8jwNwikZRAb-fLoYqr!ld38TAb3U*kmwI zjmji2dV4MB<+T|dU7?AtXT_J-=J|ew5}FO^XNmd+h^0nLg>UzGSe}{59bmMj`H?L3 zc67kIrEe&2oA}-2#RgtH+Ey8}V_nV0d!HW%C}mzs@x|iS5BLCIHv%*}a6n*gjU3bP zwCA-&qa1laWq`RDBB=xWUER;ezAHGnMl84MMcdsmgRZUybaf5CrXpS4tadz1Km%m4 z1uSs9BH*!@UHvSh-Gq>b0T7=_yZ6elVx^U3&l)s^9}UhzhUE87;cAWHbV0G-IY-h z#q1($!8gpKwY)FdK0+7^lIh6hD>=SZDN(5=RPw6wOkmUmXZ5gLw@?-Iimf(Iz}RGd z8L}}*_meM61|Xz4n-ZA~U@5`RMXA5m-XHfBOH*qLUg&V|rwmHNBuNbu%>axp%$N_J zY;x`}-$RIe|2|s&Q&i=Q7Ubh}!Q1?&_%aQAUeqPhnV0F*Ao`EEukM^Pka6bm8%tAN(@teoLr}Jpi zeIiT;(_BMqJd5f+_d{Y9*m0hx_x2pXR>(!X?2LnL9s+$`3@qkAi2}^&P|jk!nwqMR zD*_`x%!!MjAOn#WQ>SReh8go+mEm;j^X}Z4SAI*n4|m-MNp~^#-?_a_6#k?og{J19 ztom}N<+B<|JHF$oFx;)31lG(+OZ8wEj2c(;q-JxhR=?_ic&0HIdTBtoTIju2+0A&+ zCnPn!HEgS}f0$*M>ZCFqJrNv#O=_}Ubg*w|Np)Lg(8luJ1AB4TVA1E7bdshbHwO~t zXB|&AOJD`oLXbibL;-p*(FO6RWGjbGdxQyT5QU|-o%&%*^8|U9@FDM#-o1%Uc$c)n zy9CV=d~HWivVJtTuv@ev~RIXb@1x3DZTIu6?ZJ>ZX|jTU#K>U zc<`1aU0!?n=VpS?=28Z^N2kw&qr3EMAWpO@ju4PLwjhNq@VlxEp4k}r#)886PkuE1 ztdgje#>3uj&?@maeuI*}Tk~-xKRf*DoPEpC^d#2ZI zkjr`?9wz2r4O9<<2QoyNB~?b%k{zca$459MzgS*d4x@^{PGGge+GX|D(o4u2bwqq; zCWKR;FBiWgfr^48?fm8MCvO~iB5NeA8=6x?D~%tOB%2JWl4HW%HP^o;XI&Hzw?g-y zXJL@IID5wLsr{wVtfDgi=DtUPPx--K>9&_3Xz3*VUd+#85ll>>n}#V4ZJ(^zZ#N4Z zQFw8FDxfEZsffsF#(R_D8*R&;I39DEpckq~8{(;~8Jr2YQdwX*CO-rsh z$EXg6mjAeI)SV5*M2^J++n8QRmkvf#@c31hk>|OjC8zd(O5;{h_s`AaPYHCm^;{!A zxSAR&#=K3U(uV3musS@eK;`)!;rBvL*>GZ7Bo45!RZ5P0iE1d$9a+QT5c%fkFG6r# zgm<=g>$_befqR$>n}v4$Rs>)AT|#wdqk$WSBJqT|vSm74)=jMQ@f#f-Gge6(b`@WL zog=kRZcLmgtgW}t93*zHj$tTM7q<+%WP4n{LPN2utjDK=3fP3ksfmr)t+Ut>!yB33 zHa(3sZck5X9G@c3^;%wNJVlgvQY^ARQKFJlnivKJ)}&1M_H%o$K+@q;cE_j=yOcN> zc4|yK$AP<&r5$j+JkY$3R>1GILXx(A;VXLZh~^Lm~&eN~2jzHjJ`^Kk=#!{fc18HcI0Qg=fsAy?%@rpsaCgBj0<7IM-o_{dKk4a zIff!qGQ1prkHa24dwWZf0Sa8;pTKeZbgvsKf~5Q%h(@{O>R=};&1(ZUSLFTuFW^?XZ#+M#HZ}TQIK`A`fB1ThHsXw2=-IPDqJ(S2pe@e-yJ-@*R2xm!6O*U;6jpqTEp7DVF z@x)O>QjL+PW&SJL{n`|3PkR)l%H1L#MH{5`SHBQ=Zi(v&OvPU_xwFYe(FFaPRvn1BQ!1hn6v<7o`v+VkK*rvm}Jyc?sg1+`)i~>_h)FD8iXT9yZlDAEJRwgZazW5X6xN zJGLl=Wh3m^_}sUx>-P8&CuOw2VNnXXlT#X0nb>>%;a{BdbxTrSA z6$>8acr9rC7N6jR`b!p_^jzV)_~oeSdm*%E<6?8>?W7{4Nf3tT8BMimgT;K~`R`eg z-kki}$KMlnXRIZA{1PfI*mf_Cu{1Ob`o)&?2={{WBgQj<@DkTu(s6QpZiB7TH#>6E z!P{u#W`&X^<;R(kXo_4xWE>@s2K-nN2S`fy%Bi#M}TSP-nF_j2kM}?yZNyTV7s$F~N+2oVI`;V{o zkTf9V^3y=j0$mC|FZaBG~<&BJ8$oD&5@6wLw6VRObwI!ju(I7+-~g7eqO z@VJo`Z65J_pZx}8NK}Q^{-IWXu1ak@IQ?Y$=)l+Nw z5h~n|`zU-%Q|?2jmuC%rb$O0W-4w~R5`Ubnikk5Vn>p>>9I^=9+h^QhJ_U~DhX z4}6}AsOzpGzOCVn^ARTNiB0l{b>c`3)S*0EC9o=^XvGRxcxkWr>f;SD%M` zb)1@8I1_e}m!L`r!0f>MRV7;oM+-EsvZfA(w){`ep)>6B&(n2>^012J>;+W0|4QyF zoO%tj>;f`w1prN~)=u0^0mA%OWaoLwhQsHbI&f<^2`M~ zQa-I!*=QfsACniJqmtPs-T7G3d8G~yKYMo4%o$!CM2%Gniyr?BX`^CzHFB9! zuG>9Bpox5W>QVy$%p&ha(G}$F8E*i56QjRAbusE5hcc%GbSa8kT z*g+H#)hYi_*LqoJ^oDM-k&*Tfbww@&G!MzWD}&b6i^md#?%A@&_TkpZ4`j)W<)j+) zmJy?_Mw5+pEvqk8Yrs8vHaVf)!s$fwHAvYrzCR)7V3T66gNPzB1Og==^S!^^BY6Id zjH_QL9%3K;g$KDaf-}1_kfySOMV8=@))fuI$pf!GNCL>g4_=f#AmHG!YmZPn&8=PR zF{H15009kQETZrbGgb*Bj(`2z%Ha^@g9IG6yTX-mV{gZ|#;A$n*QEbD&)t-@|<3LkJtvqKPiuvmJ3s$2d|jX>ptEp6Nu&9HDJEwWlBlx6*3<>@Xg@@gjY~`@ye-PlF@?x{GZpI&Y|xvQ$~S9tUZ#+m5}4Ba?(d5 zAm{oW$nj-BxLo@PshzQwz|ML@DgHMMAfor%oEf+-$W#ct20-srD>uQ+V2w%kGT|xUhSYMZvHBsH(fbKdA6sM^B!Za{PdHZ zI-xWMde)-PZiZGpZ>yiPk*Mfa6i0=mRiKxYrr$dW1XzRTugYQqm)C~Qft$~2zMDOA z4i{5?3$W{>pO$6g4m% zS_j6fy1ahWIeNd=kg@8Q+Z@61_wLYIjp|BhP)XR=f9J_jeq`jD)^R}-2cLuC+?47x zNf8gd=8B5`yyx?r9M6Fft4EMYhA({M!`%0HT7&e>Phn@OUi-5fZHt<=d?0tyOc)Xg zt8S>K4}JRSy1njf3=EjZs3xvqML2aiL@37%x7 z=T9Sd!V6fGgdaF1#=k2x-0r_*^a|VYW;pV8M+{L1-U%)2;UR(c<~21%nPHCeKe`#X zR9}!zIUg>DvJod+B^@v6fF!8f)L?Fe6UG+`PRdICkfVb^E8;PJ)){|G^sZr7<+~$! zZ?02K&BR!uT#>CNINsoM#z^+!3GsA8&D>&}xDQLZoZ+!apuftzxm51uJ9A`EDDbDe zUc+_ba9gM6i&^%zn*BR>Qu=dT`b5S1NW?!2yqVwZfg7PHg&EC7R6zyY2<<{QdU3;k z&0N*Wn*7Gd3UrO>%aGnXxp)alL&<|}Ikj|e0@;@5fwFa|%>CYZC*DBTgDjChZ^)JI*PbC0m=tA-&EB4nrLH!Ax67{lXm6O3t@11>B(#W1YI7Dbk*mF`dD^|;!wO>9t;q$oltgDeu zd9nBocV>)$U`W(ptOLvqSHA}%sp;}`{%+xAx^;{6wt2BHYCFQxYON#@Y^%FWepEL7AIA^iTzr3kmoq9*v z*tou@ixuSGU5>P_Z8)-lMTC&kS_u)NNjrU4$VscSWPHijUNEa!cU$ zm3+e`HgASAx>Z|AKX7ZZ*e%V?8T9oB!_|J)t?bDr3vrlIVL$&lYjAVv_Sk#Zd2CIQ zX2CeGW8a7=!(&gHJ#oH*?HFz{*<8O`z(UseXmRh=cb1PY?BNT+N>~VZ3(EqTL>>5r z*jc-R0nD8jwCulUN&e8;LbM1b%~J4%fLVgi1piPm9->6A{_KWeXpBa|##_5xhb##_ zV2J7K@$)67fV61@xe&xKulh=*^?@H34# z-OMxb*Ip`Ur*a6Vw(!l#{OzC?6i|x;ZM?3)U3IZUTRFqv1CntpRUmi+qelPZM)!^5 zisr-}Mvx&ejA~H)mLbefChk82_TzxTto4TKnqHDcs#QDv>m%zsY7Z;1zMGVkT;QoJ znEXH(y_RP>{{V-7OvhHXI56(oeH>amPUFD7oWW~-;S$0|@V1_N$(QtC>ze(uThQEk z!?U9|&c`3YaZ1aq@Non|ko|(eOjbqsLZNmR;~O;r8hBhALNI!RDSkVa`48uOU%Bxu zp@~)#feR^ikEUf;6Z0xK$<7mPB#e~v%g>|y+Yk-q-l<)llfElIQ_4)@#)V225v|m(6K2`?-td0oR^Xae?{gG54A1X@Syw^M8Cxfy+4$wq~gfLTx{sqh}>u_v+;Z-4mB*4Vm@Ev9u*=itJ@UKSe? zV(S_C$^ApZY{6BF(2}dd4LX@xE# n%aclTvpNM%y7;aA(0%AKgUBHGS%+htt1zt z7ou{;B%p4*c;eg`Q+$vdJ5hx+_3!_p}cLoPYbe6c9`CBi%}Yyvkc z0^w$=oN^{zjs zdSUixw2Yh<#f?!eHoKzo=WgNUC5j?Bk_&a8+gYllJxp_?69UUO<%k}f%3eaWNMdqu`Vq6{ z2l{AGke=jV$LnHeC*+I)&$c7e8@V?xJe88yl&)Y! zUf(yKma&(E$}UL`8A}IAl%>#+LdX&ZV`*j#!zfE7LWG3K)?zK&pbViw%GzehTF9Ec zq9n@i{*3cIr{8($zxn%qedd0i=f1D&eH~U;!A6Kyinegr==ul=9wGHk+fH^o5n!=m za{Nl3rf;(OTKx8?orkAtD1B6qdg_S~W@n1l?dFYT5?s(`NImkK0xSAfE+TaC^Opb^ zHartdf?DuGE!I@yI8gxWM52%(kA>*r<%M=2z#0b;2!m-GL?j?WEX<#NU*E+#_JRIn zqkIg}n$H;NG*{=x))aC}tAqXs`}B=Fn22``8=XU)F4R-J4re)~CV zL9M<#biOTYu14>#z4bx~#m;Rt?tNQy9^B1JkF#$tF#cB5|9#?yQr@^50U1^N>dfP)Q$$|foKEY_jz&yf$_Xb2+uSwJXKX+J(H|NC zG4%Eg)7XX>nd%QX%ca&}p69u@yY<)g?TYa;3GVU#oJPR0DMh=FDxP^<>us*YoKMV0 z({LsA0;>oG?ii(!-XYajrT$qG$HZ2QerRL~zAm9SoJdxm6&$@wweC+U@chC4s3+04 zAlwK(p%ELaw1SwFK%b3an4?53X&}4cfJ~8T(`uwzoJ<>WvWj$I%8oDnNh0kKZ z?fXgvIM7b=w|@~F5e_q1``BlkRBH+r>|Md>??i%!1QEK8t1&~2cv_w>P=;7B-ln=bntT5+V5GS4)!{Tt0_uA$yZJ)8W6 zeL|^b#>P+*tO&^dZOd0CoAEz!&(@H{+@?2rIqhR zoNxCR(x!28zJwe@j!GWx-n)Vnb3rRSKJs<3VE%rm@Pl8iY(;L?o?)3==LmbO3+O)M zQ!@tg(1w%u`s&Z}JtqVLe`maQW7z>N_coMn3@M`X_sn}9ghw?)e*>!1KB;7y6e^+ zt@!cJwZH7Ebh9jWBo(7zUqzU7Q>)g$QagaIeAYSbv=FEK3X%hFiYuU{Y5RuR{ygz< zlk9eDf{|+@CHIVk)yYAil5B(6p8W~E7eqqv2a}{iiz=}%VrI?PKf(*(RB~wb-spVm zUL8xk-^&|QW@%Eq=M6gPdkZfIiJPGKvR)-jblUIBeqZ-AL-uKG?p#+}#ZlQZ-o|t0 z*BXNoce*pcBp;&&oR(L55qc8>ojB|_!N1XO6QUtFO~5A`O64t_kuyb?;q&tx#e14q)u!`uRZCCbrQ((n z_KeDlC0bX@pMSKiN@Nxncg?kW*D0Sj6)6@7G z%>pieh)!Jftfx>B00JZ@w)6VWzG}Ls=@^B&v#?i&h|T87OA$3MAC(G#fta0Yj=^dF z`LyR^Cc>0Dn)P`ZhAY@O|7rzu4CTL=KXQI?BMV) z<`VN}n|ovSZ0fJ4?t-HC*2BVSKz4dV zs>c?f8!zs!`h%8RfsJNvyH{Y|HRHF_!S&?yJq~-K!GSw!Cld#o$+OLY`%{pe9>PaG z`wUF{wcOzreh+X-x)9p>IPy|sH#usv3u^#;Dki-(79q@wE2VH1o^s(TC3#DP!&`}g zg;;agr-$|RF~)g<7zNbEz=*(+?ZKMiusZBpz0n|gln7!>*FnwBE9%8Tf>xMGuxQ=@ zHLbMuDvt321ko(5#*^1eiQL3qa*;yWh4Y$qM!9aWTJrGy`m9$C#ghHZIbl>~TSuV- zY=GaVVM+1+Bb(SB)nLVM*_7>VI3{TsnL4#7o~D_1LQEfj|3L-i7Io1sf!p|9Ew5lF7TfK8pPT(|qUyUF{X$q0;56+)uh9z-B57}?>ixf92Ln@1N* z#L4NBNhG2%;w=0nhZGtxhA_GQQmg4WJ%ZT1t?T;rIw^_ONb$tgNby>=(JJ6CUj10Z zIkOA=mkBQGQ1wGS!}+)0H{U{!*>HXky9bb z6Dunq)_IJGb*SG$s-Y?pP?a^G2YMFAF-HwgXd3Iw9XaQ90@wyXee*KGIWum3$cLlD z5HfYS8B6vRX$5Qx9%>M+yn~Qlwk2yh3xU9ef^b-YD+yuVkl8%g9lMA z1_IU06Iq?D9QZO{detMkm5tilCnE>Yp?ZI8P*;7$M&p#?YAd$e)`+tqOF0F;8Xi2V zm}G9*U@SXPe*Y!1%G>=%Yux9lGZRbNeFHbk#i-blz1H1Lw1Okp7U?#=yS!&Ty-Ji+ zsFuN69f@hzKD|zueU6=GnS;1AH(T&u8WbJke&SMINWlr1LC6zD7Ns@YB<5SK5D*}` z+0cgx;_ZjseaA&vS18Q171U3mNT1)_co4PInkTzGg^b6kD9gcsolLSN|NS)oWxmc= z$oWmETXzG0wR8d)Ex10_RgMI zw40;*e`B@lH2J3B%!;hx%LO~iu5h6CNPfWnJv#)<3LpT%1lGalf9modJah3J0bA6j zX%n5|910vHyYp#ee%cN;4>qYn`5frZ&FxH?(nbkgm3p{|4#i`c<(4!?8V%5%U{c|6 zd(6+qMZYbn{|fDFi8=?SGp80+glEn7>PQpGBxc5(X*cdQ_=Kxi7@_PJV&KLQ1mKfN zkuhe@@E^Q}1<)j&!`;XV@;>JIEgOi8NQMfPbHE#0UNJ>@V|-7id2TcYQ=uNeRZkL~ zkQ_piryPbtaMdD_7}4~AhH84BXDBP6VPCX}YENGFC zQ?O8WQAhC#f4D)4JzcmB9#vs9m8E6ft|(L{p)Cq+);?*rC*RSe^Rjkm&a_peK=S#l zZ%FcP@cz7bsVzIOvYmtGL+IcU!(fQodw@v)Q+*+r)e+^ZfF%Duafst|yPI9c9t0xhV@CeHrqN9wD>7q~g ztU(<^8vL3&3_asxnr`UhWM?_XFQmUSMzf3t7FNILVL#hdbU&KQPjT1k7O<5KL>fp8 zo3!3Lri#T3pTGK-%9hlMB+gAbXC;bTrjNe$CHGpj$oc*JR_S+hF58aE)7({~d)y#_ zUohGr4fE%b&3i+?9{KY$>*;+*dgCd3W|nkX*r(ECQK!6JSmCr|o_2Ag2H8UB zoX-Hz0MV@{eN5O2p6(e-ntOU!%hI7|8usjy{0S=G9cy#SrNevlPQED0QO#^h&k=mq zYbc=+5Kbt7Q4cYTO@8BM__Ct2m{Igz_rhRH>@=;2f#)J2PRS_u!<8_FO;j1Dah&P zxUA28JLtTULLH4)&++&@X@~~}^0y9ABHn8;V{!EHtmhoKOAN(M-^5*ak=nLnSNLRC z`637IWz86?ZZ)^$^E_*z<9n$?fXQfz{^{O zn>CzB3!|Oyi7(n>nVq(`y(z^VY&qa0dfxDIO{poH`%y$MxBa=;xxno4mD%s&?@e)b z3#5wI!QY`CvhZJ8leVu*AlYLHhHzc-)Px%)q_;VEQ6St2#)eqH8acSwk^xnh`b$pl z{aCN@the5Sbzw`xUWS7APv<-pzT--*#(%QRSf3%x)VSU1=kiKb|Lj58omsC#V+-x= zZb4A*hAmffD^zNR_=Ka5s~GiVyRjGu%Y_Ae+A2Bhrm{JQq9Wo4@GGf&su_A;mwUCkxmH+!FIJ;$@Tlg~NFbACAxiNY$%R?QM|-+Tu& zz8C2meC^A^=Fa{Q<-V|l;_5tpdQU>_Z|(;O(|XOB8txp#r63d|Ie6IOAtMBu8=9fn zutk79p$M=?@$+pNynhsZ%QF+92a20%t!{BPSmti2u|m%8U6(Xi=8sm*0&H?84Aq`L z3BT^sxUHI5q?7LbiDiDqMRjNjdu5!M8tu@7PUFsw-oV~#Hn=sBEicpTz@s;s6f@vz}19NX?ba!ELWiD!SZ*BnXeQR>#IJV~hy9$IOZgf?&De);$ znwpMq%66u2*-p3RteU>v6%LRDC6q~m#fNQm-inw-%mOB2e$3R5`8BhH+{vtD&H<^1 zW!aW9oigo2cV(tXf*^2ka9;Ru@NtWl#aLup{l9RZId@#@FWyj3}yVByv;pJbwT4F;pe*)vqR^AoP;*hP&t;BocDlB2`Ig=5 z1&-CVSW?F4S%z}GHmRN{v`E?R{UQ7(Jwi=<%JMMVv~$hcP)lKApQ33n#bi%XW# z5v#5OTF?U^Ez%`m5ram06QDKASmHyOkVZM%-kRM>Gk86r^4Zo_{UBq(Ldi>>b0pmK z&iobXR}8Q?1opqW4BmuzWt-qGw5hz6g$Ju>r8?Z{89 z9-iB!V2p~a%z=JedaH+*&;%wz9LTh&dizn)>JC~>S<=*vuBN-1Zp|rJv%c&K6fiAK z?`eAH%rN?f)i?B(Mhrg2eL@o&K__#3C5B$X zeG$X`d@PDlnUI|QRsv~xG9uVSH-O|jDp=S<=w|hc)@guN;$KjrwK2&jbP>+ECc}gok|99wQ)o2_@x#xAJ{$3qd08&QZ1 z2y{u4Le9%N3Q&axZpS5lW*(twIt?N$b>jBZ<*pzN2E1VRhbeBClFiJ^sJPvvd>$8B zY)gK3xTLD-9I6tL81Xy80Jlp)BkJZzD&?j3EAyW~XS`yD`>mu=Eoc^73&QhCWXn;T z`Q7CP)ucYJY6zzxtOrK$;kOp1#L2?qLjL7yPK{{;#&kGa15yL}DhBjuzR*`e4GLaF zOiAF!4{l6(LMrMl8PAIk?o{753#`=LRgs0k+o{aQV=y1VM*f$N>q2sl7?_Gz+vG?@ z5k@wrx9iLMa5vSfa`NBzg&#!@Hp>OyCcE^K2lH=CFPFYyE`DSBvI6)hfR6(BY;={@ z1>aUP?ezdY2Kv8l?@b9nllywDFk8o2R8@!7QH-W{H-#Up5nC*)`mb!dNPdWkc{H3S;?5PBUA&V>_Kr zSFxJHSSo;0vsg@FEI)_G6vk3vER_gT7|X4D*O)#NWI6T4Rv;`NjK(|)VR^bVqNg{< zae2fP#&S~_%Ylevngkjy%aEE07rU|Qxx!g~+f4UzAY3&jkC7Z0FW)91qXoutWP*lR zfW%bj%U3eeJO<|i3lMglvW(+mz8c$CF}6n{0xJbvN5T^@uPPeDY))x0zM_+AHbbqL zzeUaG*D>aA74Y^62KE_%w{zIfBT%=)GDVthlY_zV_@tWFTQgwn&71c@aO-*=#Nd9x z_so^wQ)pa;##LzC&E`xpHDgUQuGM$UR;TkwFmBIKVBF6boCWDgIM`<7ig_|}Ak*q$ z!o6v*%Acj-^TC4SI5hC9%xOfRips2m@;oEH0E*?QfKtimcIlZRh+?lwyvN8De1-rK zgkLEEk{4wFRz`~)3;q?`wzUwGLgNn+6NCn zepgjvOY24ujQIugjDdxi_IHAv$=SSb(`G9K@@F#}hE@CRHW-RYP0d!CPyBI<`Yk_c z7pR&zjat9v=>`%7ISY|U7aS12}>DNv!<8bd(^0Z}Nnjq{j7u_+Xr5@8C(wsl`@_H#jO z3+~t|AlpMhnMVQHmd5nvKrRoNg0XE1#->sQDhS(S81Cf&A&!7?RZh9rGMc-`I|S}e zVv(g|p2uofU&XM#{ds37*B2*hKwrgxD#+otH|BdY+%Szuz;ZsS7WQl0fI=d^G$i5z zjPFrs#H!dfd0Pr?FlNb!{H>(P9<&sl+q%%|hxdUK>#skE6?}iPcT9%wKb#KsmT2`p zWeFMH^iLF8@LQT)3OMn(sefF}So3gRy>E9~J?l|$LECOw9b3T#A3r*~KoKYea`cpD zj0{*5k+Vt4YB1jro;qlk>}S6OZP=sx}|KM z5lw#OG#Pm!FGSKtG7lnrSta#UNj)1yrD+bvJF%Irp&O8wPsMv;!9S z4(nN4>tZuEW}Yc@<_qG#n?}aBfXV< z^(Zi_%F5VKI4f}QMIq9AAg!xQC|K*a&3G>dY$F%+V1B?8g_(ODLn@bN*=%`Mjq0ly z)iZi65)n@n{OyIo-xe3^A9DaUG^l_-EZZh;xEP2eDE&fYrv zDpx7<6bMd%;5MB-$k2@LdLTHvZ9yK^;W1f?=lAr`%2)Uc(w43Lf35+ zU3VvBtmk_)@Lapw)y=LST6)K1j%gYl)^n^LbGFunjHz^fit+AO$OO0G z!i{a+7jFMd*x2VpuWtHq3OUxC0Kn6o=SvQ`m(gsp)SdgvDHPea&4e!poH9G#cY55C zLaDqkl*;1b*W-XH15lG0lOUCt1;8f}!J(E5)IO4C4pUkE{63&cRk;GGD3FQ*scd>p z%+ky?K`K_?vRZD>eH2Q?X&D`Lu;XWp&R{kYh&R~*Cq+Br&{B_sXgvWom#7ta=X-Uz{@W|7vzDF)wj|N^#NYh)XnQd8=+^_d6U@`UO^&` z=|2nbSX*=T0p%io0S%CbB@x+(COnU~36AO|FDBb7Yd{1}HeyAM<4Lsoa{7KQ>3Vch z^~a|vzJwi;r)Ey+*dixY`+0rvWWHG%oHp;Y7Y(72quJ5k#Z)}YhBfwel6 z%21JO4RQ_tdUy|Pj0UdBdJgURUE4MN(6aoHIX!CWjM{of-%_wK1si)B6Zui0zN6=Q)4KoPy-71v~eC&z$XQKQX)(NpSJE@!g(s> z(;5Mo%6WSX;t4~EHwU!3qfk!`Kt1hF*O;(xG+OG!?WYRi^ldZT%Rx9D2^hEe3ZV+m zzlu@)iDxSF+3Ok3VN9cl#MPOh>ZHgQ2SOa|9GyHiOI8$!ga%@p>>-|oEu0KFzj1#m zthxquAf$pBDwyGB*HBGE(;aX@*Mb@9-M((OjBamzm?6B;ZFTE66wL5Zqq9S*v)oAS zl9TT+UsBdeC*RjFJx~KDlYtnqq##kXYego#LNMD{#L}|Ts-hVK34=MX5L%Zrlqho` zp+vrsVt&}`3|!zv$M4>U`HJdsQB+8b@MGic2uUPwVC4uWO|b7w}D6} zM>lNUwG`4xA)T7BYO9b=kLEO^qZ>U1bW%Vkg%W5Oiz%R!0y-(6Q^VLxG1p54I^75& zKNZkvjlfmyygdr(RK?4dz)sVf3h&eaywlGjy9gt3MQ*2YdidzTN58DlQr|Yaz8oM1 zBK-*Ue=C9PlDrCE^BRWrElWgu5x0RgwET+adHa$B#dDt~55S=WG??;? zRSR~3<#qj@Vfle>GcRPm&U(zZOyBE-f#doM9?Dx@yemB^$nL7l3j@9D=oY2a0At?p z7z?@q^Lw@#TA{sA*>nw$yki+J(2##Vj8(vbGSw=3OM}*OJhH6F7}GATCvZ3CGg5wN ziGL32+KD$I$V)p1r+Z&(Cb|!Rh79A(F#CF^uN$pS&t3yGwCar|(9leD?wlR&ydfcv zSU!WI?CuW+XtHO5`eV#)m4ka(;bewm>d}N?&Z&reXX=Lg{Wdu+{fKd&?D#y$$te^^ zq4u?bDhu!nIfUm|;km9G{l0GLu5CD>WqCTdv7V!MZO@@yC$xRfm$Ro~@Qb@q7Ed{q z@4yOtx#(#QP+9QYgksA@wyX`!)-(n(t|=vtlk_rim9usgA8fD9$%<)}0R5^0G*6kY zA%l)W-&fqGL}Y_Sq~cX288;^xDfmwGl8tCPVPz)rNm7ipI8=bliVCxrmEMP00vu@aC?7LbZ*m8^UQ zS?O`%iEEa(v(Ue>goEqcTD>l*m{3X1=1R^*7FMjP(;wkCdvXIjRZ72ZN-7&PbEnkEJ^R@ZpFMxxPD#XI=0%L;xi>An=K?2mnvzv0ci-d z_bxe}ah~G5uN*foL!;Nz+m=F&E2Z3QrL1^WNyg1d#x&;F?bH3i zO+N;1=(^EW9*h!}%@&q3_EmhU1m$x=FKTDTa81RDiW4{KMyF*IwGG=*e5d&CtMc9W zpA*fedHZO8FjUN@%)$nng^E>`Tx>`dQ((}L(L$b)sW?$_;zoRwwkM+EW=Kc3G?@{* z%J!YB(2ioqXLgxyaHp>Eq{u{2O7>LTskrlb)%K0#<89q^ds-oQs^NJX7m7C(Z$7)^ zyfj+6(&~ygH_e+#<}Uyru@XFnMnomHyh1=?1qolW}KEwaRatK*8zS50l{ zR;FW6?=Ylr+NkKyW2&y?JwI~c)!GvxzX4%32wDD zV8FmJPgt;`!P9D1H+KH?cl=^rlxaWnLv z#F)`s=B1WbmCUEnK~P<%y#A3fAO=keta*B;8kE}{@!|N#q$+%~y{n$zNxho|R+6qN^80*(BhQj4RftGPHRB0SNM5F? z$Ow+R5Zz71ySDa4+e+a>yx2RBkDzr&?F#xVaR>Xl7iGk({2#$PdVA2 zQVm|tXf~0YwBv(m2~sASkTdFEL=`I~6Y@p`6S9X|Gy~fCWLRXSk2&>$`2>74Za6XH za2;A6(rm;uBqSq8tf1Nsh~r7lbA0-)jA;V515P8DfFdd2-vPYJuE{AVD;ni6J9$bo z_&S@>7Ye=4Ml|97l#JPIGVSgU)*#b6MvIuH+S`%`Fvs2i-+^@!kX^=7ph$9AXB;KR zmDQL_-rC2RH-X8Ey$x&g0;|Re`ZJG$W|JmfZ#wQJfmR3&{vkgBb%b^%v|-AaKPJOT z0$oLJDs!AzZ&|{!s>M6OCChS}A!|*E)@*WQYfUJeL?_@)HtJrn*(M z`)u017SiFLJ~>DVHo{2;REVckLKRud^+JdU|0X;JdSa_*O~Tf3kD>2D!JVrNfL(Ewkh5`_1T5>5GagH;pup2na(ulG3gG)EbRy|77}F$Sugk*Rt63Ty zRaT5eP&peo#Sa3uWbwQYLv4UCZgAA*XLd|Q!r?TwJta`H>Eo?MpIs37Tx!7G;~f&9 zhnBP7N|yLiNJ8!Z@4x=HG4xz3#+fKbTeFx-LOZdE!WC*w&(Ujq@&poe3cMgn8G z)rldO(9_ylsF{Ib(J{Nmay%dhsfY!mpjZm{e@?PG0>TSaGj#v}!%AbGH($%HN2j5P z=!lPKKIfQ2kuWlKy++GyQ_k7Oo3Ty$TG(b#O9JTZK@^psfanUzfzo&^LudHC*U;}b z(7|y`vkUAc-0(#VLX+0PKG|(;E+x_5n5m_NT^HJ@j{Cr~B~jdL>a3@bcUev|G|2^M z8!)r0NPLbLC6vSz+A$>}twLYmcqe(vGa3lOz*32TPkN0f!#c}0C{#b>)Z-B^Cerx0 zf??0cJbe}2n=dfzG9Qd6xgLNlEj{ToP$U645fWuDqvKf)bK^VeK|zp&-#42E)_QSp zYnSX(P>e?+VA2EK&jffI9DB>iDZRoqnS|^%er99(fVIMEF=%qKFGP`ok1cN=3ZF_2 zKS*%-tv0n2xoV_=hG00S3w-n{+>HUKc9zRdPJOMH>|m`m)34Fut2``zjTSH{dfY?P z?B3zY^K10(edYU2#hEWfH5htw75$A65rY|gBBBYJcp$=t6I5Vzz!2YrG0!U74%osc zLsry|XtT+)=HPz-n-1oB4BDE|G!xf6rau4le-Z+;IQsNQAwY5O9_*47s6c+)J3VN8 zx7P_Ml}SKxMx&RPp{-H?zvCkrC!iBhYHwI1t_Zzr?vwBKt_zmn`sh!f4nSZuT<5LH z5Yw;{1_UZB$|(z2h~OzKx(^o!AP8O<0%bkEID?6x?eYp@vKVZ8f#IVy$NR@bTw$NC z#$cWog3$b;53j)lA^5eRjA}0kze?t(jmn@JSb+oz`4_mv0*t{itbia)xPZ>qi{E+5 z{8^Z;;aX))K0(OQWvxUCA@ORHnfO@dlRg=M0Lh@k6+~nk;YzhO+9sLsV3s8;&$mfF z0nZZwIMaeS3uz*fNem`f^H=QFgS!pO?2=8e#G6R?7c%dLT*#G@xb~>}Ab9g)n@G%C zQdFeaFC`@DECa#Yd`4P_8mBP!llkI&AG|FZAXT(5V+^c~JeMJCX^!ku9%28X4t(#` z5I_PTc@(jrPj(Ul25E-p|&F*x#9*Ag%9F}W^L=~&E%gbLc#Oeryw%5 zGZ8TffI)r1lDy1sAim$tcV5E_xbwQ%bii~3>v-_3EOi;mzWGZSuiRn1&0YH|qrp7xxFqz&9wY5*IcO65JS9FOnm! zo1C%{ZbiX(`6;XSq>Oobzm!2LuOz^Y2b+*9TfhJE`@ekLqGd4_S+$|c!yRo6*vlEP zQdm6NsqtA*7 zy0lAct9~iNU8O}<%1SPwgcr@Lo!3@DwwCrAPI4aO9PP^wE%Dz$T|4n61bK1i;B;>V zBhEjuQ=!6Ea^vk%zzYS2L7{}+a`6&(X^OCB5ZTx*Q#UP5?`ei{W|)1w)7OobZgu{B zYg$@rm)ZK+18=aI9+HeV9dT(Uf!xgmN$Vvy!qLVd|JXt{(yFbJWH%G3C$9goRf%&N zV2pa9QW7$XVU)Z@4hmU=#3m3lzkna4(X?26_OU>k@=zJr>|++Cs6`njJ76_{41L** zfPt1t$bD6b$7*1i{0PFa^KIglJSvFXoR(ZbjZ%o$A6K=a&-~-g@izH)u>S(FO?HPVF5H6sv-e?l=j=~8d1DU8I9+D#%v*BC1Yl9l zJW(lba5^y1fIJ-G{wFX+ryF=-W}%vjPT6c;)VH&G^5A7P%++tne~$hil5x+k!gF0W z`hA^Q9pCa<&ky~c)9tu4=ozMA25{RnJNDLmOSt5KCP-zq!~Rw7u>TtBQ)Puv`E{C` zUuS{i6{jlc`ux&$4~CyB$*TCZQT(cSP4U|E%f?lf=Rm}1#IDbWh2UTah?o`GLltxw(vdI!sajcS(FCr#% z#$uTvql7Wpox&;4-vWOZfprn22vfI_-wniEB0O|VRxN1!{awHaOL&kb(oAB z!So_#&EB3~pn*?=x*E$~p3d+5&m{Kh(cr>O2#u~Kxr~mosf5pU`l#q^D5$f#i2UE1qW=DT(`@J!}0oq3iKbS&F7^w8T<85NYzzM-Yb zTjE$niBwFpxiL+}uS!BbvxIE(JTF<^wk%ymC{={=1w<$-zFqI6auHykKmBitF%=Li z1qM};QFWpUCRd}0n(<_kY`B+)ZO7w+u&tBEcg+QT!H)G7DHZzcCta6ttZzBIUC zwv686l4H)_xFb2{5l?tbBf=4rJU>~WUJ&~0hl7D!3KGz=XG{BNQAPtAdWLUP+tw}5 z3~ZY@y4~$`oWNB<3T1+C$^^e9)s@euVie7dQ7Cp*@^Pc`F~w653#JLb5NVmu81Dv9 zT5+Qio-aUn-o?v`ZIuvx79px)yeh`~0%E*Vc3ms3iJPHOFemcZ%K4e$|D44k&c+#| zMNZD}j3F5SJ``8lA@B^IYd4fZUsgk`1~iu^4T0U;?8xsUXtam#+=$AAAb9cITfC;# z0bH_lULAV+YI=^*r#^WvZ{RU@c+@x|=8Bpl@Vv*Jv%&kLSJUG&=8qG3geQiYv!YK9 zZpGz>73LVRgsdRPW$dwxRL8bJZPjU@C$t#L;N7dC%6oN7`Sb39P zIVp2_V(29H1gI#ocg_6r!Sf%VVL{{`QoNcak&upQ0Dr>h*gNV=^^4ug|8JL9#& zI9k=w@^Vk{K*+!lG#8<;9xJXGh~0&Q=J)G^o0}aCt7h;zQNq)m{r!Uh+4*qxZtwW) zU~q8uH?niQbNIL6!H~S!8Sd?p_s3-S;O)WL&LP=9IDWf#dUASjd`5P6&UVQAeR8~a z_VfGGAIZt-`?L3h_lM-*=;Uzk2x{9oJ9vNG@aXy4lp2=VC7Z~g@fp1qiHId^dMCJK zS&nF)M%YBaVm0Vj;*m--%W1P(R`y56S z{?KTW^ZY#JX%(kAuPOpDoeC9LRDs1O2rQb`V*`uz8+Tx>7noRzq$v|mMHcF^e{A78 zraz&H@T!$ikWqdl{0m&@^DlA|^B|xTlysN;vQ8m^zl%)#W9!*4+Fgrw&A_2yCt$2Y zySmkNdLHXC+txc>(@=p;<+yLiaeq2BqTGJv_BYqxpLlCo=ef1vqws8@;FD28fiQL89 zA5#{^G%F_U4+-C8{*AobhX;ebu>j=p|e?#3I>5EtuF&$e_O z_|NRp&~i-2bAS`gjuYyJ!$K7oQ(oAHys&kJwDP-@-_=~dYXui8j#ZL#(~?tp^2(Ec z0iJxttZR7a`!L_gf;=QM$%9y{6Fd!uSBE>Cu(SYMnH)hk$S?(BWYrPInn!c+7KYV| zM6(e@YtyAFIN0rzT?7hUim04!#Kg_YBRuD!sN5Cp7DdP|G~ksGVmLn=2*#Pq&Brhq za`1-=M({+f5pL0i_rYDH1m+J++KN-a_d`V~kN*G$%9)=rS#q{@nPvAM0rCvNojVD$ z9JKkk6_!y{SNfLv+vJ#oB|uTY)OY7M=JJ*%7a+J%pk(uRB@Gs;+x?Y>G~?T3C*s=XG3W zdX~#<>iVqfhj!TU%#P`FJ2xzucQn#=f#E<{-Hrt<)aY2Y*X;zB=}rSlLUc!i}UZ0dICfXQXhz$lbnwdT3#cq zq1DBw{{_|$`R>pheoy`rfys)9PF{Ta1J7#Wxi%~VqE)|%VUX`WoEXOU+JM0T(1;Do z^wS@}f7J~}j^%)XXsd?tltI(MOJUWR7a);-D;fE2aC|!az8=b$GcWuncW{;q2k@+jHr+)QcFiWrZ4OUF`SeERPs zqZ7FOU&wc(Pk)H`(;q0g@Yq0n{(Xy_WrE*qU@q`HJrK_PW?TDcM*muI+cwFtCNaN* zYH`s$v36U}1_cfrC+u{+PzTG+GJA$^1)kIOEz9)`+cey@mpnFPicWnjb`Vs$o+<0> zt~qyye>SGka&vYLNt=*&-~?u5Co8xgF*KqN@kmv9=3!0g6|Lm~24LcT6zW6{|gx)n+p7 z986-xZEJCxu2}5zq?zKe^?9tNxa@I)ECw#im1(9#*b_1r!W$`H?DoeR@3_0Tn)b1Z zC(TKKu(c1Thg&6eJV3eN+-z4iO=OE^$m!V$r_{GaLc%(a$q?rh#dW#^``PGyTY zROu_{<0^iof-iRmUod_*7z}qSlES5Q7we!`PC<-@%Fy;4 zBCZJbh$RY8?N=gO-Z?S9yWF6f%;!}N;WWVdfQ)$ft%V`|`|#wTH9UBGymR==UIGe} zF`0057Ipr6ItKds`7iAu!V@EjZ$60WXc6=*nPR?GK(WNyEb$DmZ;_7D@=V0y?a2Yr z$SDg6qCQDb!iawQLkm9rk$(EW1=%?{A%@{}Oc0RdyQGZC!S3#{9Q83Y4!>8Bfaj@5 zaA>~2jT)z6$Rl1alSpLE9QXZi^n#JS5sN&QVSv59y>&l;{9$l6ZiQPSrQGweBVTxm zwc53D-In80#%$krdtKWxeO)(g+j0zM2R$R~^cKwff8RTPzx)2{Zo{8?L$7aced(&^ zDR;NaQegU5Il@7q} z-p*;C?D9}{y1<+;d)H}Jonc6Jrnyl;(4V&8k}#U{Xo4umkdMj?!6oGMct^|GZv;$) zi!c&bauRpmD5FO}RX#(bDe1hHbY25M z@>#@h7!u^09{D?501zij&DrH64w8+$myR4ccw(fQp|>2^QxCzY})CuIt$> z=+SQAhh|7+*|6idj@`BA!DThOnmxOq8>5WIz>1^8mE613(whxhZi1AmpSzau$_9U? z_$1L+EEs&#)VX77@H{WsvbdbxyffxhFSFhga*`$PFm=?*&4X!IzA%8wdEW0&J)tTJ zEZg*@>*7%F5>Ym}qqp0!%=_Y~OagKjTX)0)dCvT>A}o2wXaJITl_#}C-X(>KeyixWihh5E_==_J zov#@EcKf#8a`mo?ey<#WQ&6}3L`0KB#3(&4^;V~4nB<4_FF#N+&RF=L|FJ3+;ao7b zm#S#}8nvNB3U*k#s?BJ%T6Mku*!l|!Ym#$lUHviRhc^Ao5AF0X&lepw7;`$z>$zRm z@dN5wU<;eR+p%5KG8{)mhn4htSbC}G@arG>hQ_a4y<>H|&I^bRPg~l==&)`Xo$gX} z_}+N%v@!A`H^`6gv{&vU&l^QhyPh85^mNCU%nEENxaDf3CigH=4ZK~rOs@!*+p zqvFAjOT(?$wgK_NdtKE2)YE-jJkX>01x+6j3*3I7J{YTivMbK~3wmFz=C+D3stBWs zFn+F&?r6qCM|*v4gwfK$foOGlUz{)Xl@7oU>0gcoSs)B1j@q#<<$rMX3-Ee_*=96?Kr>{iDZncrcvI0dvd6m zG6{zY?ld;I)9u-AuVcS}VCl4_O$?SAhHE5B`NE(Rjo>6qqh7n`2_ z1eOcS`UsxX9~UYe@(~5^^m0Xr^r5Kek1#ZJn;c|(;qbcp>b}6JeDA@S>ySaiw#gn; zFt5G7TB_YztX;nMaP4CTy`EgS*6r%-<_B))NB8!a}3qIZDLk(Cx;P)(Ax2`NmQp za@vZ!LUD5F?g*5*wAa;b!#B}dH^Po-hutovjvxB25wHdY%GN#k`m=cZF~j@n!$QyJ zjy>-TZ>kSw>0P&HH6x~x-yhVdoBB*ajaxB_pq}7s3%k4eqb5azC&i60!WzDG102ek zI#CYgqC@$pgYTcglVWJ4^?L1-@xjTE{dPWGuLK_o2ld=A9_L5xvR@f+Cfg>b0$V-z zS!UPwK~SczKj^spL?gzpx5*LBxLEkg?D{^>t9tptZw$s1njA%+{@5l%9xr}odU>A* zHh=vEKwy+7^%XrSOVbTZYJDH10}7aDhSS%Hc1g1d=J0$s+`+PVe(NR6WYL1*2;eE;6CfM+ z@trZeU(kZ&tnOxu{NQDOA;$M4h8N{7au~U?O03CX)>Y*d!zPx=?WTWMoDTQDui9c_TYxvu>yF=Qh@{Tq=iy7TcHk#y z2i&;T%{rBsKL!e=Q%_`@Lq%;F217Qn2K|RYLRI9l$(j@GTF|--V`QYIQ?79?;TlmKbd@Jm9Eakr6 zmMJUU_l#L6gln>Pd#>K?x_;LUJ=+iUka~vYGt+Q-x}^t;hku5Al#kI6YLWxLfa_S! zz3pdArUL{P1)ZGVggAJUw%yX(5e7q%msK#J5q?icckOoEYfX8rWg4z-Ij+^ZG@2R} zxVOzs4G%mhOdsr+nlCS5zN`!wEK2fqLk3cs-ijH_?>_k|G+MFJ6T7296{KgmLG!yC zZ_tKU=_@6#f+{a5s4@Uc<9yjRe<+UfO+yQ2#}rfah)!9~@GvHF0QM&@N(?7`NWjL+ z^4cl=YstXC16>%%xv>zmIpWX)N)AEos$&4Q)HWrxP3i0GDljzz|NBbYl+0e==(Vhl ztF|ev9DsX)5zn$eiJBJmZMPeCb?OF7LCM57>QtDseNz;hbLR3>(AUF)_05?oJG!oU6(Fxr0<)bkz z$Q8|DqCyd1%z#|+VoXL%ROeXZ0oLgZ?In+I082O6Cag7T{o@~4m*!E#vgM5DG56bV zPfkf8Vl;}Oift&4x~Fj9ODLSIRORjk^1FtKn1TuB^t|MQK!UlFKqJ2fzafhycx0%=+9eXC2b>-ubts9!kl~syGDc$RkAXZDq2--9FG(^3 zuU$ch3MT6r{Q1vSYsNr6*|#eg`WY(?CeV@!JeX!E36Q7CoK@l6uN=7b!ydTb9h+Hh z*RpM+W9nAdFnez3xjke3!0q!WjSZvjj7OpFe8=qSUDlx;uVeWR1swn$0qdB)@A`~& zw{9*OllPz{l19ZV@4?;Ol~rhzM%wf>g|=%#SOx>+R@)aX-O+WUqc<32x2mc4#5q|q zT59TEGQz;nR|s+HHxA$YScy|$S0HmDXH^M4(4$%X)=_KKcelR$Y_+E_{p%3Y3*@|GLyEw9{Sb2DtS?k1ZmymV^`dg1odE<)GA_NMG=G1RV>M;zRK$AE#5 zx)Ry)h|TXVw`>FFk?+y07?UBcU;6UNOKSX}T3cE=O1(Kt@n^FYs>OAi<;5yov#1sm zNjP3}7;l*oKg#|ZT1w?euS{y*Ik#;OI!!F(~F9PB=g z?#IBl27gS=wz|63>vd}twbn^l-}IQGnhlCE%Fn~lEuVJ(+6rcy_nf#*?LFo zn1;PDn{KA#s>WE&sjoYyj?q9EmWfhHFK`msde7Eu+cg&^k)>492B*&uoP6?gi|n?@ z8Bl%Ew3|OtGwOxRC<`VN{OY;7zA&Tg#TiwPvYJ<4cV7J)kICPJP-^MrO(Iisb<@#I z$L%gmB1fsEO9p>xEg=*;dJERC1a}@7nl=r~1tm zrtVivPUH5SZmZ)uDml$vXJ}vQEu*F5iP_|bGJ?M%{uhv;FmDTj@`GGf37r4Q0`dcH znD|Rl#_T%HerU@pnRA#$k>*V_{C8f+qtha|U1EfZ2}zzZsHfg(f*VHiAKLIi@o9$oTc*}UQ;Xj}Gy4Pb) z;Cm`1&P$n3DkaX>R--Bt>9(V8> z)E5AKFdYK?BZsMeGQXkF<&6yjTbikN4Xx`KcY@C95ni#PvW%Ug3Df>2#Kkrqj|W`F zwieIFt8XY~Xe=|Bn$zocwXWsdiJ6)YvP$Qq(m6dUos*$Il+MYN`$Y_lb=&KubGq}5 z8`1F%gj|`;Lf7@(uEW4JwoJd{cEiwf4a+bbeeICzw~)+ckq_>SGS1uJIJ)h4fk``# zZuG*yWu2~V(5~ZHJ;!$!kC6M2TqNQOMdr+CR1wGL7Yhvvi`cGZcG^4;ek*49w+{_% zM4a(^r*iz8K57a%1hKzW1byxC1e+^3D%os`~{r>e0Eo=AR^lld@|c8L-35e#wKK_CUY(V2$FdOX`Ju}^*X&w0 z^#jxDu)yejQXXu0|28Kgs1=j literal 0 HcmV?d00001 From 9f73d93e624ae08ca61288503171226477fd87fd Mon Sep 17 00:00:00 2001 From: Serafeim Chatzopoulos Date: Tue, 3 Oct 2023 14:39:08 +0300 Subject: [PATCH 030/148] Add param for limiting repo Urls --- .../swh/CollectLastVisitRepositoryData.java | 34 ++++++++----------- .../swh/CollectSoftwareRepositoryURLs.java | 16 ++++++--- .../dnetlib/dhp/swh/PrepareSWHActionsets.java | 1 - ...nput_collect_software_repository_urls.json | 6 ++++ .../eu/dnetlib/dhp/swh/job.properties | 2 ++ .../eu/dnetlib/dhp/swh/oozie_app/workflow.xml | 1 + 6 files changed, 35 insertions(+), 25 deletions(-) diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectLastVisitRepositoryData.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectLastVisitRepositoryData.java index 296a4cce1..e602266a8 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectLastVisitRepositoryData.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectLastVisitRepositoryData.java @@ -1,32 +1,26 @@ package eu.dnetlib.dhp.swh; -import static eu.dnetlib.dhp.utils.DHPUtils.getHadoopConfiguration; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.URISyntaxException; -import java.net.URL; -import java.nio.charset.StandardCharsets; - -import org.apache.commons.cli.ParseException; -import org.apache.commons.io.IOUtils; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.Text; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.common.collection.CollectorException; import eu.dnetlib.dhp.common.collection.HttpClientParams; import eu.dnetlib.dhp.swh.utils.SWHConnection; import eu.dnetlib.dhp.swh.utils.SWHConstants; import eu.dnetlib.dhp.swh.utils.SWHUtils; +import org.apache.commons.cli.ParseException; +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.SequenceFile; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.IOException; +import java.net.URL; + +import static eu.dnetlib.dhp.utils.DHPUtils.getHadoopConfiguration; /** * Given a file with software repository URLs, this class diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectSoftwareRepositoryURLs.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectSoftwareRepositoryURLs.java index 6232fa322..abd51bc5b 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectSoftwareRepositoryURLs.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectSoftwareRepositoryURLs.java @@ -51,6 +51,9 @@ public class CollectSoftwareRepositoryURLs { final String hiveMetastoreUris = parser.get("hiveMetastoreUris"); log.info("hiveMetastoreUris: {}", hiveMetastoreUris); + final Integer softwareLimit = Integer.parseInt(parser.get("softwareLimit")); + log.info("softwareLimit: {}", softwareLimit); + SparkConf conf = new SparkConf(); conf.set("hive.metastore.uris", hiveMetastoreUris); @@ -58,18 +61,23 @@ public class CollectSoftwareRepositoryURLs { conf, isSparkSessionManaged, spark -> { - doRun(spark, hiveDbName, outputPath); + doRun(spark, hiveDbName, softwareLimit, outputPath); }); } - private static void doRun(SparkSession spark, String hiveDbName, String outputPath) { + private static void doRun(SparkSession spark, String hiveDbName, Integer limit, + String outputPath) { String queryTemplate = "SELECT distinct coderepositoryurl.value " + "FROM %s.software " + "WHERE coderepositoryurl.value IS NOT NULL " + "AND datainfo.deletedbyinference = FALSE " + - "AND datainfo.invisible = FALSE " + - "LIMIT 5000"; + "AND datainfo.invisible = FALSE "; + + if (limit != null) { + queryTemplate += String.format("LIMIT %s", limit); + } + String query = String.format(queryTemplate, hiveDbName); log.info("Hive query to fetch software code URLs: {}", query); diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/PrepareSWHActionsets.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/PrepareSWHActionsets.java index c0ab11bc4..c54e10d3e 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/PrepareSWHActionsets.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/PrepareSWHActionsets.java @@ -82,7 +82,6 @@ public class PrepareSWHActionsets { softwareRDD .saveAsHadoopFile( outputPath, Text.class, Text.class, SequenceFileOutputFormat.class); -// , GzipCodec.class); }); } diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_collect_software_repository_urls.json b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_collect_software_repository_urls.json index 6e98c7673..4459fe9df 100644 --- a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_collect_software_repository_urls.json +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_collect_software_repository_urls.json @@ -22,5 +22,11 @@ "paramLongName": "hiveMetastoreUris", "paramDescription": "the hive metastore uris", "paramRequired": true + }, + { + "paramName": "slim", + "paramLongName": "softwareLimit", + "paramDescription": "limit on the number of software repo URL to fetch", + "paramRequired": false } ] \ No newline at end of file diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties index 651bae337..8dd0689a3 100644 --- a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties @@ -14,4 +14,6 @@ maxNumberOfRetry=2 retryDelay=1 requestDelay=100 +softwareLimit=500 + resume=collect-software-repository-urls diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml index 7aa667a4a..e0763414f 100644 --- a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml @@ -83,6 +83,7 @@ --softwareCodeRepositoryURLs${softwareCodeRepositoryURLs} --hiveDbName${hiveDbName} --hiveMetastoreUris${hiveMetastoreUris} + --softwareLimit${softwareLimit} From 24c43e0c602254f506e81f1e3d361645b1a9d36e Mon Sep 17 00:00:00 2001 From: Serafeim Chatzopoulos Date: Tue, 3 Oct 2023 15:11:58 +0300 Subject: [PATCH 031/148] Restructure workflow parameters --- .../main/resources/eu/dnetlib/dhp/swh/job.properties | 2 -- .../eu/dnetlib/dhp/swh/oozie_app/config-default.xml | 4 ++++ .../eu/dnetlib/dhp/swh/oozie_app/workflow.xml | 12 ++++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties index 8dd0689a3..114181944 100644 --- a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties @@ -1,8 +1,6 @@ # hive hiveDbName=openaire_prod_20230914 -sparkSqlWarehouseDir=/user/hive/warehouse - # input/output files softwareCodeRepositoryURLs=${workingDir}/1_code_repo_urls.csv lastVisitsPath=${workingDir}/2_last_visits.seq diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/config-default.xml b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/config-default.xml index 7873d595e..3e45a53fa 100644 --- a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/config-default.xml +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/config-default.xml @@ -47,4 +47,8 @@ oozie.launcher.mapreduce.user.classpath.first true + + sparkSqlWarehouseDir + /user/hive/warehouse + \ No newline at end of file diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml index e0763414f..e29e5b43d 100644 --- a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml @@ -18,6 +18,14 @@ archiveRequestsPath The path in the HDFS to save the responses of the archive requests + + actionsetsPath + The path in the HDFS to save the action sets + + + graphPath + The path in the HDFS to the base folder of the graph + maxNumberOfRetry Max number of retries for failed API calls @@ -30,6 +38,10 @@ requestDelay Delay between API requests (in ms) + + softwareLimit + Limit on the number of repo URLs to use (Optional); for debug purposes + resume Variable that indicates the step to start from From b49a3ac9b21233edb889b2e0e60dfa3ed8c02734 Mon Sep 17 00:00:00 2001 From: Serafeim Chatzopoulos Date: Tue, 3 Oct 2023 15:43:38 +0300 Subject: [PATCH 032/148] Add actionsetsPath as a global WF param --- .../swh/CollectLastVisitRepositoryData.java | 23 ++++++++++--------- .../eu/dnetlib/dhp/swh/oozie_app/workflow.xml | 4 ++++ 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectLastVisitRepositoryData.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectLastVisitRepositoryData.java index e602266a8..ce1b0bb46 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectLastVisitRepositoryData.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectLastVisitRepositoryData.java @@ -1,12 +1,12 @@ package eu.dnetlib.dhp.swh; -import eu.dnetlib.dhp.application.ArgumentApplicationParser; -import eu.dnetlib.dhp.common.collection.CollectorException; -import eu.dnetlib.dhp.common.collection.HttpClientParams; -import eu.dnetlib.dhp.swh.utils.SWHConnection; -import eu.dnetlib.dhp.swh.utils.SWHConstants; -import eu.dnetlib.dhp.swh.utils.SWHUtils; +import static eu.dnetlib.dhp.utils.DHPUtils.getHadoopConfiguration; + +import java.io.BufferedReader; +import java.io.IOException; +import java.net.URL; + import org.apache.commons.cli.ParseException; import org.apache.commons.io.IOUtils; import org.apache.hadoop.fs.FileStatus; @@ -16,11 +16,12 @@ import org.apache.hadoop.io.SequenceFile; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedReader; -import java.io.IOException; -import java.net.URL; - -import static eu.dnetlib.dhp.utils.DHPUtils.getHadoopConfiguration; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.collection.CollectorException; +import eu.dnetlib.dhp.common.collection.HttpClientParams; +import eu.dnetlib.dhp.swh.utils.SWHConnection; +import eu.dnetlib.dhp.swh.utils.SWHConstants; +import eu.dnetlib.dhp.swh.utils.SWHUtils; /** * Given a file with software repository URLs, this class diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml index e29e5b43d..c69ecd74d 100644 --- a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml @@ -57,6 +57,10 @@ oozie.action.sharelib.for.spark ${oozieActionShareLibForSpark2} + + actionsetsPath + ${actionsetsPath} + From cae75fc75da4410289e1d5107073c5b112520003 Mon Sep 17 00:00:00 2001 From: Serafeim Chatzopoulos Date: Tue, 3 Oct 2023 16:55:10 +0300 Subject: [PATCH 033/148] Add SWH in the collectedFrom field --- .../java/eu/dnetlib/dhp/swh/PrepareSWHActionsets.java | 8 ++++++++ .../main/java/eu/dnetlib/dhp/swh/utils/SWHConstants.java | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/PrepareSWHActionsets.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/PrepareSWHActionsets.java index c54e10d3e..93d111039 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/PrepareSWHActionsets.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/PrepareSWHActionsets.java @@ -165,6 +165,14 @@ public class PrepareSWHActionsets { row.getString(row.fieldIndex("swhid")), qualifier, dataInfo))); + + // add SWH in the `collectedFrom` field + KeyValue kv = new KeyValue(); + kv.setKey(SWHConstants.SWH_ID); + kv.setValue(SWHConstants.SWH_NAME); + + s.setCollectedfrom(Arrays.asList(kv)); + return s; }, Encoders.bean(Software.class)) .toJavaRDD() diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConstants.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConstants.java index 08400f28b..efd0e708b 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConstants.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConstants.java @@ -16,4 +16,8 @@ public class SWHConstants { public static final String SWHID_CLASSNAME = "Software Heritage Identifier"; + public static final String SWH_ID = "10|openaire____::dbfd07503aaa1ed31beed7dec942f3f4"; + + public static final String SWH_NAME = "Software Heritage"; + } From e9f24df21cce089aec7945b2f50f20cbe1acc062 Mon Sep 17 00:00:00 2001 From: Serafeim Chatzopoulos Date: Tue, 3 Oct 2023 20:57:57 +0300 Subject: [PATCH 034/148] Move SWH API Key from constants to workflow param --- .../java/eu/dnetlib/dhp/swh/ArchiveRepositoryURLs.java | 5 ++++- .../dhp/swh/CollectLastVisitRepositoryData.java | 5 ++++- .../java/eu/dnetlib/dhp/swh/utils/SWHConnection.java | 6 ++++-- .../java/eu/dnetlib/dhp/swh/utils/SWHConstants.java | 2 -- .../dnetlib/dhp/swh/input_archive_repository_urls.json | 6 ++++++ .../swh/input_collect_last_visit_repository_data.json | 6 ++++++ .../main/resources/eu/dnetlib/dhp/swh/job.properties | 2 ++ .../eu/dnetlib/dhp/swh/oozie_app/workflow.xml | 10 ++++++++++ .../java/eu/dnetlib/dhp/swh/SWHConnectionTest.java | 4 ++-- 9 files changed, 38 insertions(+), 8 deletions(-) diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/ArchiveRepositoryURLs.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/ArchiveRepositoryURLs.java index f02861953..baa510346 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/ArchiveRepositoryURLs.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/ArchiveRepositoryURLs.java @@ -63,9 +63,12 @@ public class ArchiveRepositoryURLs { final Integer archiveThresholdInDays = Integer.parseInt(argumentParser.get("archiveThresholdInDays")); log.info("archiveThresholdInDays: {}", archiveThresholdInDays); + final String apiAccessToken = argumentParser.get("apiAccessToken"); + log.info("apiAccessToken: {}", apiAccessToken); + final HttpClientParams clientParams = SWHUtils.getClientParams(argumentParser); - swhConnection = new SWHConnection(clientParams); + swhConnection = new SWHConnection(clientParams, apiAccessToken); final FileSystem fs = FileSystem.get(getHadoopConfiguration(hdfsuri)); diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectLastVisitRepositoryData.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectLastVisitRepositoryData.java index ce1b0bb46..ebb9176ff 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectLastVisitRepositoryData.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/CollectLastVisitRepositoryData.java @@ -55,9 +55,12 @@ public class CollectLastVisitRepositoryData { final String outputPath = argumentParser.get("lastVisitsPath"); log.info("outputPath: {}", outputPath); + final String apiAccessToken = argumentParser.get("apiAccessToken"); + log.info("apiAccessToken: {}", apiAccessToken); + final HttpClientParams clientParams = SWHUtils.getClientParams(argumentParser); - swhConnection = new SWHConnection(clientParams); + swhConnection = new SWHConnection(clientParams, apiAccessToken); final FileSystem fs = FileSystem.get(getHadoopConfiguration(hdfsuri)); diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConnection.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConnection.java index 9c145fc19..80249e816 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConnection.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConnection.java @@ -14,13 +14,15 @@ public class SWHConnection { HttpConnector2 conn; - public SWHConnection(HttpClientParams clientParams) { + public SWHConnection(HttpClientParams clientParams, String accessToken) { // set custom headers Map headers = new HashMap() { { put(HttpHeaders.ACCEPT, "application/json"); - put(HttpHeaders.AUTHORIZATION, String.format("Bearer %s", SWHConstants.ACCESS_TOKEN)); + if (accessToken != null) { + put(HttpHeaders.AUTHORIZATION, String.format("Bearer %s", accessToken)); + } } }; diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConstants.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConstants.java index efd0e708b..eae839cfd 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConstants.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConstants.java @@ -6,8 +6,6 @@ public class SWHConstants { public static final String SWH_ARCHIVE_URL = "https://archive.softwareheritage.org/api/1/origin/save/%s/url/%s/"; - public static final String ACCESS_TOKEN = "eyJhbGciOiJIUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJhMTMxYTQ1My1hM2IyLTQwMTUtODQ2Ny05MzAyZjk3MTFkOGEifQ.eyJpYXQiOjE2OTQ2MzYwMjAsImp0aSI6IjkwZjdkNTNjLTQ5YTktNGFiMy1hY2E0LTcwMTViMjEyZTNjNiIsImlzcyI6Imh0dHBzOi8vYXV0aC5zb2Z0d2FyZWhlcml0YWdlLm9yZy9hdXRoL3JlYWxtcy9Tb2Z0d2FyZUhlcml0YWdlIiwiYXVkIjoiaHR0cHM6Ly9hdXRoLnNvZnR3YXJlaGVyaXRhZ2Uub3JnL2F1dGgvcmVhbG1zL1NvZnR3YXJlSGVyaXRhZ2UiLCJzdWIiOiIzMTY5OWZkNC0xNmE0LTQxOWItYTdhMi00NjI5MDY4ZjI3OWEiLCJ0eXAiOiJPZmZsaW5lIiwiYXpwIjoic3doLXdlYiIsInNlc3Npb25fc3RhdGUiOiIzMjYzMzEwMS00ZDRkLTQwMjItODU2NC1iMzNlMTJiNTE3ZDkiLCJzY29wZSI6Im9wZW5pZCBvZmZsaW5lX2FjY2VzcyBwcm9maWxlIGVtYWlsIn0.XHj1VIZu1dZ4Ej32-oU84mFmaox9cLNjXosNxwZM0Xs"; - public static final String DEFAULT_VISIT_TYPE = "git"; public static final String VISIT_STATUS_NOT_FOUND = "not_found"; diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_archive_repository_urls.json b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_archive_repository_urls.json index ce80d6f4a..e8671f71b 100644 --- a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_archive_repository_urls.json +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_archive_repository_urls.json @@ -46,5 +46,11 @@ "paramLongName": "archiveThresholdInDays", "paramDescription": "the thershold (in days) required to issue an archive request", "paramRequired": false + }, + { + "paramName": "aat", + "paramLongName": "apiAccessToken", + "paramDescription": "the API access token of the SWH API", + "paramRequired": false } ] \ No newline at end of file diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_collect_last_visit_repository_data.json b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_collect_last_visit_repository_data.json index 8bf41f0ae..662582dfe 100644 --- a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_collect_last_visit_repository_data.json +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/input_collect_last_visit_repository_data.json @@ -40,5 +40,11 @@ "paramLongName": "requestMethod", "paramDescription": "the method of the requests to perform", "paramRequired": false + }, + { + "paramName": "aat", + "paramLongName": "apiAccessToken", + "paramDescription": "the API access token of the SWH API", + "paramRequired": false } ] \ No newline at end of file diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties index 114181944..35c068286 100644 --- a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/job.properties @@ -8,6 +8,8 @@ archiveRequestsPath=${workingDir}/3_archive_requests.seq actionsetsPath=${workingDir}/4_actionsets graphPath=/tmp/prod_provision/graph/18_graph_blacklisted +apiAccessToken=eyJhbGciOiJIUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJhMTMxYTQ1My1hM2IyLTQwMTUtODQ2Ny05MzAyZjk3MTFkOGEifQ.eyJpYXQiOjE2OTQ2MzYwMjAsImp0aSI6IjkwZjdkNTNjLTQ5YTktNGFiMy1hY2E0LTcwMTViMjEyZTNjNiIsImlzcyI6Imh0dHBzOi8vYXV0aC5zb2Z0d2FyZWhlcml0YWdlLm9yZy9hdXRoL3JlYWxtcy9Tb2Z0d2FyZUhlcml0YWdlIiwiYXVkIjoiaHR0cHM6Ly9hdXRoLnNvZnR3YXJlaGVyaXRhZ2Uub3JnL2F1dGgvcmVhbG1zL1NvZnR3YXJlSGVyaXRhZ2UiLCJzdWIiOiIzMTY5OWZkNC0xNmE0LTQxOWItYTdhMi00NjI5MDY4ZjI3OWEiLCJ0eXAiOiJPZmZsaW5lIiwiYXpwIjoic3doLXdlYiIsInNlc3Npb25fc3RhdGUiOiIzMjYzMzEwMS00ZDRkLTQwMjItODU2NC1iMzNlMTJiNTE3ZDkiLCJzY29wZSI6Im9wZW5pZCBvZmZsaW5lX2FjY2VzcyBwcm9maWxlIGVtYWlsIn0.XHj1VIZu1dZ4Ej32-oU84mFmaox9cLNjXosNxwZM0Xs + maxNumberOfRetry=2 retryDelay=1 requestDelay=100 diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml index c69ecd74d..64dc0d2aa 100644 --- a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml @@ -38,6 +38,10 @@ requestDelay Delay between API requests (in ms) + + apiAccessToken + The API Key of the SWH API + softwareLimit Limit on the number of repo URLs to use (Optional); for debug purposes @@ -61,6 +65,10 @@ actionsetsPath ${actionsetsPath} + + apiAccessToken + ${apiAccessToken} + @@ -117,6 +125,7 @@ --requestDelay${requestDelay} --retryDelay${retryDelay} --requestMethodGET + --apiAccessToken${apiAccessToken} @@ -136,6 +145,7 @@ --requestDelay${requestDelay} --retryDelay${retryDelay} --requestMethodPOST + --apiAccessToken${apiAccessToken} diff --git a/dhp-workflows/dhp-swh/src/test/java/eu/dnetlib/dhp/swh/SWHConnectionTest.java b/dhp-workflows/dhp-swh/src/test/java/eu/dnetlib/dhp/swh/SWHConnectionTest.java index 28210f1b3..b19e0e7ac 100644 --- a/dhp-workflows/dhp-swh/src/test/java/eu/dnetlib/dhp/swh/SWHConnectionTest.java +++ b/dhp-workflows/dhp-swh/src/test/java/eu/dnetlib/dhp/swh/SWHConnectionTest.java @@ -25,7 +25,7 @@ public class SWHConnectionTest { HttpClientParams clientParams = new HttpClientParams(); clientParams.setRequestMethod("GET"); - SWHConnection swhConnection = new SWHConnection(clientParams); + SWHConnection swhConnection = new SWHConnection(clientParams, null); String repoUrl = "https://github.com/stanford-futuredata/FAST"; URL url = new URL(String.format(SWHConstants.SWH_LATEST_VISIT_URL, repoUrl)); @@ -43,7 +43,7 @@ public class SWHConnectionTest { HttpClientParams clientParams = new HttpClientParams(); clientParams.setRequestMethod("POST"); - SWHConnection swhConnection = new SWHConnection(clientParams); + SWHConnection swhConnection = new SWHConnection(clientParams, null); String repoUrl = "https://github.com/stanford-futuredata/FAST"; URL url = new URL(String.format(SWHConstants.SWH_ARCHIVE_URL, SWHConstants.DEFAULT_VISIT_TYPE, repoUrl)); From ee8a39e7d2dcd121d88c1bb84eb4794024fe6a16 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Wed, 4 Oct 2023 12:32:05 +0200 Subject: [PATCH 035/148] cleanup and refinements --- .../CreateActionSetSparkJob.java | 63 ++++++++++++------- .../opencitations/GetOpenCitationsRefs.java | 3 +- .../actionmanager/opencitations/ReadCOCI.java | 9 +-- .../opencitations/as_parameters.json | 18 +++--- .../opencitations/oozie_app/workflow.xml | 3 +- 5 files changed, 55 insertions(+), 41 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java index e3a9833b3..2db756a94 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java @@ -32,18 +32,28 @@ import eu.dnetlib.dhp.schema.oaf.*; import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions; import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils; +import eu.dnetlib.dhp.schema.oaf.utils.PidType; import eu.dnetlib.dhp.utils.DHPUtils; import scala.Tuple2; public class CreateActionSetSparkJob implements Serializable { public static final String OPENCITATIONS_CLASSID = "sysimport:crosswalk:opencitations"; public static final String OPENCITATIONS_CLASSNAME = "Imported from OpenCitations"; + + // DOI-to-DOI citations + public static final String COCI = "COCI"; + + // PMID-to-PMID citations + public static final String POCI = "POCI"; + private static final String DOI_PREFIX = "50|doi_________::"; private static final String PMID_PREFIX = "50|pmid________::"; + private static final String TRUST = "0.91"; private static final Logger log = LoggerFactory.getLogger(CreateActionSetSparkJob.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); public static void main(final String[] args) throws IOException, ParseException { @@ -67,7 +77,7 @@ public class CreateActionSetSparkJob implements Serializable { log.info("isSparkSessionManaged: {}", isSparkSessionManaged); final String inputPath = parser.get("inputPath"); - log.info("inputPath {}", inputPath.toString()); + log.info("inputPath {}", inputPath); final String outputPath = parser.get("outputPath"); log.info("outputPath {}", outputPath); @@ -81,19 +91,16 @@ public class CreateActionSetSparkJob implements Serializable { runWithSparkSession( conf, isSparkSessionManaged, - spark -> { - extractContent(spark, inputPath, outputPath, shouldDuplicateRels); - }); + spark -> extractContent(spark, inputPath, outputPath, shouldDuplicateRels)); } private static void extractContent(SparkSession spark, String inputPath, String outputPath, boolean shouldDuplicateRels) { - getTextTextJavaPairRDD(spark, inputPath, shouldDuplicateRels, "COCI") - .union(getTextTextJavaPairRDD(spark, inputPath, shouldDuplicateRels, "POCI")) + getTextTextJavaPairRDD(spark, inputPath, shouldDuplicateRels, COCI) + .union(getTextTextJavaPairRDD(spark, inputPath, shouldDuplicateRels, POCI)) .saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class); - } private static JavaPairRDD getTextTextJavaPairRDD(SparkSession spark, String inputPath, @@ -109,7 +116,7 @@ public class CreateActionSetSparkJob implements Serializable { value, shouldDuplicateRels, prefix) .iterator(), Encoders.bean(Relation.class)) - .filter((FilterFunction) value -> value != null) + .filter((FilterFunction) Objects::nonNull) .toJavaRDD() .map(p -> new AtomicAction(p.getClass(), p)) .mapToPair( @@ -123,20 +130,28 @@ public class CreateActionSetSparkJob implements Serializable { String prefix; String citing; String cited; - if (p.equals("COCI")) { - prefix = DOI_PREFIX; - citing = prefix - + IdentifierFactory.md5(CleaningFunctions.normalizePidValue("doi", value.getCiting())); - cited = prefix - + IdentifierFactory.md5(CleaningFunctions.normalizePidValue("doi", value.getCited())); - - } else { - prefix = PMID_PREFIX; - citing = prefix - + IdentifierFactory.md5(CleaningFunctions.normalizePidValue("pmid", value.getCiting())); - cited = prefix - + IdentifierFactory.md5(CleaningFunctions.normalizePidValue("pmid", value.getCited())); + switch (p) { + case COCI: + prefix = DOI_PREFIX; + citing = prefix + + IdentifierFactory + .md5(CleaningFunctions.normalizePidValue(PidType.doi.toString(), value.getCiting())); + cited = prefix + + IdentifierFactory + .md5(CleaningFunctions.normalizePidValue(PidType.doi.toString(), value.getCited())); + break; + case POCI: + prefix = PMID_PREFIX; + citing = prefix + + IdentifierFactory + .md5(CleaningFunctions.normalizePidValue(PidType.pmid.toString(), value.getCiting())); + cited = prefix + + IdentifierFactory + .md5(CleaningFunctions.normalizePidValue(PidType.pmid.toString(), value.getCited())); + break; + default: + throw new IllegalStateException("Invalid prefix: " + p); } if (!citing.equals(cited)) { @@ -162,7 +177,7 @@ public class CreateActionSetSparkJob implements Serializable { public static Relation getRelation( String source, String target, - String relclass) { + String relClass) { return OafMapperUtils .getRelation( @@ -170,7 +185,7 @@ public class CreateActionSetSparkJob implements Serializable { target, ModelConstants.RESULT_RESULT, ModelConstants.CITATION, - relclass, + relClass, Arrays .asList( OafMapperUtils.keyValue(ModelConstants.OPENOCITATIONS_ID, ModelConstants.OPENOCITATIONS_NAME)), @@ -183,6 +198,6 @@ public class CreateActionSetSparkJob implements Serializable { ModelConstants.DNET_PROVENANCE_ACTIONS, ModelConstants.DNET_PROVENANCE_ACTIONS), TRUST), null); - } + } diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/GetOpenCitationsRefs.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/GetOpenCitationsRefs.java index 60dc998ef..600cf7df1 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/GetOpenCitationsRefs.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/GetOpenCitationsRefs.java @@ -3,6 +3,7 @@ package eu.dnetlib.dhp.actionmanager.opencitations; import java.io.*; import java.io.Serializable; +import java.util.Arrays; import java.util.Objects; import java.util.zip.GZIPOutputStream; import java.util.zip.ZipEntry; @@ -37,7 +38,7 @@ public class GetOpenCitationsRefs implements Serializable { parser.parseArgument(args); final String[] inputFile = parser.get("inputFile").split(";"); - log.info("inputFile {}", inputFile.toString()); + log.info("inputFile {}", Arrays.asList(inputFile)); final String workingPath = parser.get("workingPath"); log.info("workingPath {}", workingPath); diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/ReadCOCI.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/ReadCOCI.java index 3d384de9d..b9c24df3b 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/ReadCOCI.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/ReadCOCI.java @@ -7,6 +7,7 @@ import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; import java.io.IOException; import java.io.Serializable; +import java.util.Arrays; import java.util.Optional; import org.apache.commons.io.IOUtils; @@ -42,7 +43,7 @@ public class ReadCOCI implements Serializable { log.info("outputPath: {}", outputPath); final String[] inputFile = parser.get("inputFile").split(";"); - log.info("inputFile {}", inputFile.toString()); + log.info("inputFile {}", Arrays.asList(inputFile)); Boolean isSparkSessionManaged = isSparkSessionManaged(parser); log.info("isSparkSessionManaged: {}", isSparkSessionManaged); @@ -74,10 +75,10 @@ public class ReadCOCI implements Serializable { private static void doRead(SparkSession spark, String workingPath, String[] inputFiles, String outputPath, - String delimiter, String format) throws IOException { + String delimiter, String format) { for (String inputFile : inputFiles) { - String p_string = workingPath + "/" + inputFile + ".gz"; + String pString = workingPath + "/" + inputFile + ".gz"; Dataset cociData = spark .read() @@ -86,7 +87,7 @@ public class ReadCOCI implements Serializable { .option("inferSchema", "true") .option("header", "true") .option("quotes", "\"") - .load(p_string) + .load(pString) .repartition(100); cociData.map((MapFunction) row -> { diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/as_parameters.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/as_parameters.json index e25d1f4b8..5244a6fe4 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/as_parameters.json +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/as_parameters.json @@ -16,15 +16,11 @@ "paramLongName": "isSparkSessionManaged", "paramDescription": "the hdfs name node", "paramRequired": false - }, { - "paramName": "sdr", - "paramLongName": "shouldDuplicateRels", - "paramDescription": "the hdfs name node", - "paramRequired": false -},{ - "paramName": "p", - "paramLongName": "prefix", - "paramDescription": "the hdfs name node", - "paramRequired": true -} + }, + { + "paramName": "sdr", + "paramLongName": "shouldDuplicateRels", + "paramDescription": "activates/deactivates the construction of bidirectional relations Cites/IsCitedBy", + "paramRequired": false + } ] diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/oozie_app/workflow.xml index d87dfa2ba..deb32459b 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/opencitations/oozie_app/workflow.xml @@ -34,6 +34,7 @@ Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + ${jobTracker} @@ -54,6 +55,7 @@ + eu.dnetlib.dhp.actionmanager.opencitations.GetOpenCitationsRefs @@ -112,7 +114,6 @@ --inputPath${workingPath} --outputPath${outputPath} - --prefix${prefix} From 1bb83b91880933cee1a39aaa59ad0c70f5086156 Mon Sep 17 00:00:00 2001 From: Serafeim Chatzopoulos Date: Wed, 4 Oct 2023 20:31:45 +0300 Subject: [PATCH 036/148] Add prefix in SWH ID --- .../src/main/java/eu/dnetlib/dhp/swh/PrepareSWHActionsets.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/PrepareSWHActionsets.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/PrepareSWHActionsets.java index 93d111039..2239244d6 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/PrepareSWHActionsets.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/PrepareSWHActionsets.java @@ -162,7 +162,7 @@ public class PrepareSWHActionsets { .asList( OafMapperUtils .structuredProperty( - row.getString(row.fieldIndex("swhid")), + String.format("swh:1:snp:%s", row.getString(row.fieldIndex("swhid"))), qualifier, dataInfo))); From 13f332ce77e3ef7a03ba6e0b31419cf7caf3ea8e Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Fri, 6 Oct 2023 10:40:05 +0200 Subject: [PATCH 037/148] ignored jenv prop --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 73d9179fa..14cd4d345 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ spark-warehouse /**/*.log /**/.factorypath /**/.scalafmt.conf +/.java-version From 42a2dad975051211dae5bf56e6d129d82d1401f9 Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Fri, 6 Oct 2023 11:52:33 +0200 Subject: [PATCH 038/148] implemented relation to irish funder from a Json list --- .../dhp/doiboost/crossref/irish_funder.json | 946 ++++++++++++++++++ .../doiboost/crossref/Crossref2Oaf.scala | 25 + .../crossref/CrossrefMappingTest.scala | 2 +- 3 files changed, 972 insertions(+), 1 deletion(-) create mode 100644 dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/crossref/irish_funder.json diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/crossref/irish_funder.json b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/crossref/irish_funder.json new file mode 100644 index 000000000..6c6912c2e --- /dev/null +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/crossref/irish_funder.json @@ -0,0 +1,946 @@ +[ + { + "id": "100007630", + "uri": "http://dx.doi.org/10.13039/100007630", + "name": "College of Engineering and Informatics, National University of Ireland, Galway", + "synonym": [] + }, + { + "id": "100007731", + "uri": "http://dx.doi.org/10.13039/100007731", + "name": "Endo International", + "synonym": [] + }, + { + "id": "100008099", + "uri": "http://dx.doi.org/10.13039/100008099", + "name": "Food Safety Authority of Ireland", + "synonym": [] + }, + { + "id": "100008124", + "uri": "http://dx.doi.org/10.13039/100008124", + "name": "Department of Jobs, Enterprise and Innovation", + "synonym": [] + }, + { + "id": "100009098", + "uri": "http://dx.doi.org/10.13039/100009098", + "name": "Department of Foreign Affairs and Trade, Ireland", + "synonym": [] + }, + { + "id": "100009099", + "uri": "http://dx.doi.org/10.13039/100009099", + "name": "Irish Aid", + "synonym": [] + }, + { + "id": "100009770", + "uri": "http://dx.doi.org/10.13039/100009770", + "name": "National University of Ireland", + "synonym": [] + }, + { + "id": "100009985", + "uri": "http://dx.doi.org/10.13039/100009985", + "name": "Parkinson's Association of Ireland", + "synonym": [] + }, + { + "id": "100010399", + "uri": "http://dx.doi.org/10.13039/100010399", + "name": "European Society of Cataract and Refractive Surgeons", + "synonym": [] + }, + { + "id": "100010414", + "uri": "http://dx.doi.org/10.13039/100010414", + "name": "Health Research Board", + "synonym": [ + "501100001590" + ] + }, + { + "id": "100010546", + "uri": "http://dx.doi.org/10.13039/100010546", + "name": "Deparment of Children and Youth Affairs, Ireland", + "synonym": [] + }, + { + "id": "100010993", + "uri": "http://dx.doi.org/10.13039/100010993", + "name": "Irish Nephrology Society", + "synonym": [] + }, + { + "id": "100011062", + "uri": "http://dx.doi.org/10.13039/100011062", + "name": "Asian Spinal Cord Network", + "synonym": [] + }, + { + "id": "100011096", + "uri": "http://dx.doi.org/10.13039/100011096", + "name": "Jazz Pharmaceuticals", + "synonym": [] + }, + { + "id": "100011396", + "uri": "http://dx.doi.org/10.13039/100011396", + "name": "Irish College of General Practitioners", + "synonym": [] + }, + { + "id": "100012734", + "uri": "http://dx.doi.org/10.13039/100012734", + "name": "Department for Culture, Heritage and the Gaeltacht, Ireland", + "synonym": [] + }, + { + "id": "100012754", + "uri": "http://dx.doi.org/10.13039/100012754", + "name": "Horizon Pharma", + "synonym": [] + }, + { + "id": "100012891", + "uri": "http://dx.doi.org/10.13039/100012891", + "name": "Medical Research Charities Group", + "synonym": [] + }, + { + "id": "100012919", + "uri": "http://dx.doi.org/10.13039/100012919", + "name": "Epilepsy Ireland", + "synonym": [] + }, + { + "id": "100012920", + "uri": "http://dx.doi.org/10.13039/100012920", + "name": "GLEN", + "synonym": [] + }, + { + "id": "100012921", + "uri": "http://dx.doi.org/10.13039/100012921", + "name": "Royal College of Surgeons in Ireland", + "synonym": [] + }, + { + "id": "100013029", + "uri": "http://dx.doi.org/10.13039/100013029", + "name": "Iris O'Brien Foundation", + "synonym": [] + }, + { + "id": "100013206", + "uri": "http://dx.doi.org/10.13039/100013206", + "name": "Food Institutional Research Measure", + "synonym": [] + }, + { + "id": "100013381", + "uri": "http://dx.doi.org/10.13039/100013381", + "name": "Irish Phytochemical Food Network", + "synonym": [] + }, + { + "id": "100013433", + "uri": "http://dx.doi.org/10.13039/100013433", + "name": "Transport Infrastructure Ireland", + "synonym": [] + }, + { + "id": "100013461", + "uri": "http://dx.doi.org/10.13039/100013461", + "name": "Arts and Disability Ireland", + "synonym": [] + }, + { + "id": "100013548", + "uri": "http://dx.doi.org/10.13039/100013548", + "name": "Filmbase", + "synonym": [] + }, + { + "id": "100013917", + "uri": "http://dx.doi.org/10.13039/100013917", + "name": "Society for Musicology in Ireland", + "synonym": [] + }, + { + "id": "100014251", + "uri": "http://dx.doi.org/10.13039/100014251", + "name": "Humanities in the European Research Area", + "synonym": [] + }, + { + "id": "100014364", + "uri": "http://dx.doi.org/10.13039/100014364", + "name": "National Children's Research Centre", + "synonym": [] + }, + { + "id": "100014384", + "uri": "http://dx.doi.org/10.13039/100014384", + "name": "Amarin Corporation", + "synonym": [] + }, + { + "id": "100014902", + "uri": "http://dx.doi.org/10.13039/100014902", + "name": "Irish Association for Cancer Research", + "synonym": [] + }, + { + "id": "100015023", + "uri": "http://dx.doi.org/10.13039/100015023", + "name": "Ireland Funds", + "synonym": [] + }, + { + "id": "100015037", + "uri": "http://dx.doi.org/10.13039/100015037", + "name": "Simon Cumbers Media Fund", + "synonym": [] + }, + { + "id": "100015319", + "uri": "http://dx.doi.org/10.13039/100015319", + "name": "Sport Ireland Institute", + "synonym": [] + }, + { + "id": "100015320", + "uri": "http://dx.doi.org/10.13039/100015320", + "name": "Paralympics Ireland", + "synonym": [] + }, + { + "id": "100015442", + "uri": "http://dx.doi.org/10.13039/100015442", + "name": "Global Brain Health Institute", + "synonym": [] + }, + { + "id": "100015776", + "uri": "http://dx.doi.org/10.13039/100015776", + "name": "Health and Social Care Board", + "synonym": [] + }, + { + "id": "100015992", + "uri": "http://dx.doi.org/10.13039/100015992", + "name": "St. Luke's Institute of Cancer Research", + "synonym": [] + }, + { + "id": "100017897", + "uri": "http://dx.doi.org/10.13039/100017897", + "name": "Friedreich\u2019s Ataxia Research Alliance Ireland", + "synonym": [] + }, + { + "id": "100018064", + "uri": "http://dx.doi.org/10.13039/100018064", + "name": "Department of Tourism, Culture, Arts, Gaeltacht, Sport and Media", + "synonym": [] + }, + { + "id": "100018172", + "uri": "http://dx.doi.org/10.13039/100018172", + "name": "Department of the Environment, Climate and Communications", + "synonym": [] + }, + { + "id": "100018175", + "uri": "http://dx.doi.org/10.13039/100018175", + "name": "Dairy Processing Technology Centre", + "synonym": [] + }, + { + "id": "100018270", + "uri": "http://dx.doi.org/10.13039/100018270", + "name": "Health Service Executive", + "synonym": [] + }, + { + "id": "100018529", + "uri": "http://dx.doi.org/10.13039/100018529", + "name": "Alkermes", + "synonym": [] + }, + { + "id": "100018542", + "uri": "http://dx.doi.org/10.13039/100018542", + "name": "Irish Endocrine Society", + "synonym": [] + }, + { + "id": "100018754", + "uri": "http://dx.doi.org/10.13039/100018754", + "name": "An Roinn Sl\u00e1inte", + "synonym": [] + }, + { + "id": "100018998", + "uri": "http://dx.doi.org/10.13039/100018998", + "name": "Irish Research eLibrary", + "synonym": [] + }, + { + "id": "100019428", + "uri": "http://dx.doi.org/10.13039/100019428", + "name": "Nabriva Therapeutics", + "synonym": [] + }, + { + "id": "100019637", + "uri": "http://dx.doi.org/10.13039/100019637", + "name": "Horizon Therapeutics", + "synonym": [] + }, + { + "id": "100020174", + "uri": "http://dx.doi.org/10.13039/100020174", + "name": "Health Research Charities Ireland", + "synonym": [] + }, + { + "id": "100020202", + "uri": "http://dx.doi.org/10.13039/100020202", + "name": "UCD Foundation", + "synonym": [] + }, + { + "id": "100020233", + "uri": "http://dx.doi.org/10.13039/100020233", + "name": "Ireland Canada University Foundation", + "synonym": [] + }, + { + "id": "100022943", + "uri": "http://dx.doi.org/10.13039/100022943", + "name": "National Cancer Registry Ireland", + "synonym": [] + }, + { + "id": "501100001581", + "uri": "http://dx.doi.org/10.13039/501100001581", + "name": "Arts Council of Ireland", + "synonym": [] + }, + { + "id": "501100001582", + "uri": "http://dx.doi.org/10.13039/501100001582", + "name": "Centre for Ageing Research and Development in Ireland", + "synonym": [] + }, + { + "id": "501100001583", + "uri": "http://dx.doi.org/10.13039/501100001583", + "name": "Cystinosis Foundation Ireland", + "synonym": [] + }, + { + "id": "501100001584", + "uri": "http://dx.doi.org/10.13039/501100001584", + "name": "Department of Agriculture, Food and the Marine, Ireland", + "synonym": [] + }, + { + "id": "501100001586", + "uri": "http://dx.doi.org/10.13039/501100001586", + "name": "Department of Education and Skills, Ireland", + "synonym": [] + }, + { + "id": "501100001587", + "uri": "http://dx.doi.org/10.13039/501100001587", + "name": "Economic and Social Research Institute", + "synonym": [] + }, + { + "id": "501100001588", + "uri": "http://dx.doi.org/10.13039/501100001588", + "name": "Enterprise Ireland", + "synonym": [] + }, + { + "id": "501100001589", + "uri": "http://dx.doi.org/10.13039/501100001589", + "name": "Environmental Protection Agency", + "synonym": [] + }, + { + "id": "501100001591", + "uri": "http://dx.doi.org/10.13039/501100001591", + "name": "Heritage Council", + "synonym": [] + }, + { + "id": "501100001592", + "uri": "http://dx.doi.org/10.13039/501100001592", + "name": "Higher Education Authority", + "synonym": [] + }, + { + "id": "501100001593", + "uri": "http://dx.doi.org/10.13039/501100001593", + "name": "Irish Cancer Society", + "synonym": [] + }, + { + "id": "501100001594", + "uri": "http://dx.doi.org/10.13039/501100001594", + "name": "Irish Heart Foundation", + "synonym": [] + }, + { + "id": "501100001595", + "uri": "http://dx.doi.org/10.13039/501100001595", + "name": "Irish Hospice Foundation", + "synonym": [] + }, + { + "id": "501100001596", + "uri": "http://dx.doi.org/10.13039/501100001596", + "name": "Irish Research Council for Science, Engineering and Technology", + "synonym": [] + }, + { + "id": "501100001597", + "uri": "http://dx.doi.org/10.13039/501100001597", + "name": "Irish Research Council for the Humanities and Social Sciences", + "synonym": [] + }, + { + "id": "501100001598", + "uri": "http://dx.doi.org/10.13039/501100001598", + "name": "Mental Health Commission", + "synonym": [] + }, + { + "id": "501100001600", + "uri": "http://dx.doi.org/10.13039/501100001600", + "name": "Research and Education Foundation, Sligo General Hospital", + "synonym": [] + }, + { + "id": "501100001601", + "uri": "http://dx.doi.org/10.13039/501100001601", + "name": "Royal Irish Academy", + "synonym": [] + }, + { + "id": "501100001602", + "uri": "http://dx.doi.org/10.13039/501100001602", + "name": "Science Foundation Ireland", + "synonym": [] + }, + { + "id": "501100001603", + "uri": "http://dx.doi.org/10.13039/501100001603", + "name": "Sustainable Energy Authority of Ireland", + "synonym": [] + }, + { + "id": "501100001604", + "uri": "http://dx.doi.org/10.13039/501100001604", + "name": "Teagasc", + "synonym": [] + }, + { + "id": "501100001627", + "uri": "http://dx.doi.org/10.13039/501100001627", + "name": "Marine Institute", + "synonym": [] + }, + { + "id": "501100001628", + "uri": "http://dx.doi.org/10.13039/501100001628", + "name": "Central Remedial Clinic", + "synonym": [] + }, + { + "id": "501100001629", + "uri": "http://dx.doi.org/10.13039/501100001629", + "name": "Royal Dublin Society", + "synonym": [] + }, + { + "id": "501100001630", + "uri": "http://dx.doi.org/10.13039/501100001630", + "name": "Dublin Institute for Advanced Studies", + "synonym": [] + }, + { + "id": "501100001631", + "uri": "http://dx.doi.org/10.13039/501100001631", + "name": "University College Dublin", + "synonym": [] + }, + { + "id": "501100001633", + "uri": "http://dx.doi.org/10.13039/501100001633", + "name": "National University of Ireland, Maynooth", + "synonym": [] + }, + { + "id": "501100001634", + "uri": "http://dx.doi.org/10.13039/501100001634", + "name": "University of Galway", + "synonym": [] + }, + { + "id": "501100001635", + "uri": "http://dx.doi.org/10.13039/501100001635", + "name": "University of Limerick", + "synonym": [] + }, + { + "id": "501100001636", + "uri": "http://dx.doi.org/10.13039/501100001636", + "name": "University College Cork", + "synonym": [] + }, + { + "id": "501100001637", + "uri": "http://dx.doi.org/10.13039/501100001637", + "name": "Trinity College Dublin", + "synonym": [] + }, + { + "id": "501100001638", + "uri": "http://dx.doi.org/10.13039/501100001638", + "name": "Dublin City University", + "synonym": [] + }, + { + "id": "501100002081", + "uri": "http://dx.doi.org/10.13039/501100002081", + "name": "Irish Research Council", + "synonym": [] + }, + { + "id": "501100002736", + "uri": "http://dx.doi.org/10.13039/501100002736", + "name": "Covidien", + "synonym": [] + }, + { + "id": "501100002755", + "uri": "http://dx.doi.org/10.13039/501100002755", + "name": "Brennan and Company", + "synonym": [] + }, + { + "id": "501100002919", + "uri": "http://dx.doi.org/10.13039/501100002919", + "name": "Cork Institute of Technology", + "synonym": [] + }, + { + "id": "501100002959", + "uri": "http://dx.doi.org/10.13039/501100002959", + "name": "Dublin City Council", + "synonym": [] + }, + { + "id": "501100003036", + "uri": "http://dx.doi.org/10.13039/501100003036", + "name": "Perrigo Company Charitable Foundation", + "synonym": [] + }, + { + "id": "501100003037", + "uri": "http://dx.doi.org/10.13039/501100003037", + "name": "Elan", + "synonym": [] + }, + { + "id": "501100003496", + "uri": "http://dx.doi.org/10.13039/501100003496", + "name": "HeyStaks Technologies", + "synonym": [] + }, + { + "id": "501100003553", + "uri": "http://dx.doi.org/10.13039/501100003553", + "name": "Gaelic Athletic Association", + "synonym": [] + }, + { + "id": "501100003840", + "uri": "http://dx.doi.org/10.13039/501100003840", + "name": "Irish Institute of Clinical Neuroscience", + "synonym": [] + }, + { + "id": "501100003956", + "uri": "http://dx.doi.org/10.13039/501100003956", + "name": "Aspect Medical Systems", + "synonym": [] + }, + { + "id": "501100004162", + "uri": "http://dx.doi.org/10.13039/501100004162", + "name": "Meath Foundation", + "synonym": [] + }, + { + "id": "501100004210", + "uri": "http://dx.doi.org/10.13039/501100004210", + "name": "Our Lady's Children's Hospital, Crumlin", + "synonym": [] + }, + { + "id": "501100004321", + "uri": "http://dx.doi.org/10.13039/501100004321", + "name": "Shire", + "synonym": [] + }, + { + "id": "501100004981", + "uri": "http://dx.doi.org/10.13039/501100004981", + "name": "Athlone Institute of Technology", + "synonym": [] + }, + { + "id": "501100006518", + "uri": "http://dx.doi.org/10.13039/501100006518", + "name": "Department of Communications, Energy and Natural Resources, Ireland", + "synonym": [] + }, + { + "id": "501100006553", + "uri": "http://dx.doi.org/10.13039/501100006553", + "name": "Collaborative Centre for Applied Nanotechnology", + "synonym": [] + }, + { + "id": "501100006759", + "uri": "http://dx.doi.org/10.13039/501100006759", + "name": "CLARITY Centre for Sensor Web Technologies", + "synonym": [] + }, + { + "id": "501100009246", + "uri": "http://dx.doi.org/10.13039/501100009246", + "name": "Technological University Dublin", + "synonym": [] + }, + { + "id": "501100009269", + "uri": "http://dx.doi.org/10.13039/501100009269", + "name": "Programme of Competitive Forestry Research for Development", + "synonym": [] + }, + { + "id": "501100009315", + "uri": "http://dx.doi.org/10.13039/501100009315", + "name": "Cystinosis Ireland", + "synonym": [] + }, + { + "id": "501100010808", + "uri": "http://dx.doi.org/10.13039/501100010808", + "name": "Geological Survey of Ireland", + "synonym": [] + }, + { + "id": "501100011030", + "uri": "http://dx.doi.org/10.13039/501100011030", + "name": "Alimentary Glycoscience Research Cluster", + "synonym": [] + }, + { + "id": "501100011031", + "uri": "http://dx.doi.org/10.13039/501100011031", + "name": "Alimentary Health", + "synonym": [] + }, + { + "id": "501100011103", + "uri": "http://dx.doi.org/10.13039/501100011103", + "name": "Rann\u00eds", + "synonym": [] + }, + { + "id": "501100012354", + "uri": "http://dx.doi.org/10.13039/501100012354", + "name": "Inland Fisheries Ireland", + "synonym": [] + }, + { + "id": "501100014384", + "uri": "http://dx.doi.org/10.13039/501100014384", + "name": "X-Bolt Orthopaedics", + "synonym": [] + }, + { + "id": "501100014710", + "uri": "http://dx.doi.org/10.13039/501100014710", + "name": "PrecisionBiotics Group", + "synonym": [] + }, + { + "id": "501100014827", + "uri": "http://dx.doi.org/10.13039/501100014827", + "name": "Dormant Accounts Fund", + "synonym": [] + }, + { + "id": "501100016041", + "uri": "http://dx.doi.org/10.13039/501100016041", + "name": "St Vincents Anaesthesia Foundation", + "synonym": [] + }, + { + "id": "501100017501", + "uri": "http://dx.doi.org/10.13039/501100017501", + "name": "FotoNation", + "synonym": [] + }, + { + "id": "501100018641", + "uri": "http://dx.doi.org/10.13039/501100018641", + "name": "Dairy Research Ireland", + "synonym": [] + }, + { + "id": "501100018839", + "uri": "http://dx.doi.org/10.13039/501100018839", + "name": "Irish Centre for High-End Computing", + "synonym": [] + }, + { + "id": "501100019905", + "uri": "http://dx.doi.org/10.13039/501100019905", + "name": "Galway University Foundation", + "synonym": [] + }, + { + "id": "501100020036", + "uri": "http://dx.doi.org/10.13039/501100020036", + "name": "Dystonia Ireland", + "synonym": [] + }, + { + "id": "501100020221", + "uri": "http://dx.doi.org/10.13039/501100020221", + "name": "Irish Motor Neurone Disease Association", + "synonym": [] + }, + { + "id": "501100020270", + "uri": "http://dx.doi.org/10.13039/501100020270", + "name": "Advanced Materials and Bioengineering Research", + "synonym": [] + }, + { + "id": "501100020403", + "uri": "http://dx.doi.org/10.13039/501100020403", + "name": "Irish Composites Centre", + "synonym": [] + }, + { + "id": "501100020425", + "uri": "http://dx.doi.org/10.13039/501100020425", + "name": "Irish Thoracic Society", + "synonym": [] + }, + { + "id": "501100021102", + "uri": "http://dx.doi.org/10.13039/501100021102", + "name": "Waterford Institute of Technology", + "synonym": [] + }, + { + "id": "501100021110", + "uri": "http://dx.doi.org/10.13039/501100021110", + "name": "Irish MPS Society", + "synonym": [] + }, + { + "id": "501100021525", + "uri": "http://dx.doi.org/10.13039/501100021525", + "name": "Insight SFI Research Centre for Data Analytics", + "synonym": [] + }, + { + "id": "501100021694", + "uri": "http://dx.doi.org/10.13039/501100021694", + "name": "Elan Pharma International", + "synonym": [] + }, + { + "id": "501100021838", + "uri": "http://dx.doi.org/10.13039/501100021838", + "name": "Royal College of Physicians of Ireland", + "synonym": [] + }, + { + "id": "501100022542", + "uri": "http://dx.doi.org/10.13039/501100022542", + "name": "Breakthrough Cancer Research", + "synonym": [] + }, + { + "id": "501100022610", + "uri": "http://dx.doi.org/10.13039/501100022610", + "name": "Breast Cancer Ireland", + "synonym": [] + }, + { + "id": "501100022728", + "uri": "http://dx.doi.org/10.13039/501100022728", + "name": "Munster Technological University", + "synonym": [] + }, + { + "id": "501100022729", + "uri": "http://dx.doi.org/10.13039/501100022729", + "name": "Institute of Technology, Tralee", + "synonym": [] + }, + { + "id": "501100023273", + "uri": "http://dx.doi.org/10.13039/501100023273", + "name": "HRB Clinical Research Facility Galway", + "synonym": [] + }, + { + "id": "501100023378", + "uri": "http://dx.doi.org/10.13039/501100023378", + "name": "Lauritzson Foundation", + "synonym": [] + }, + { + "id": "501100023551", + "uri": "http://dx.doi.org/10.13039/501100023551", + "name": "Cystic Fibrosis Ireland", + "synonym": [] + }, + { + "id": "501100023970", + "uri": "http://dx.doi.org/10.13039/501100023970", + "name": "Tyndall National Institute", + "synonym": [] + }, + { + "id": "501100024094", + "uri": "http://dx.doi.org/10.13039/501100024094", + "name": "Raidi\u00f3 Teilif\u00eds \u00c9ireann", + "synonym": [] + }, + { + "id": "501100024242", + "uri": "http://dx.doi.org/10.13039/501100024242", + "name": "Synthesis and Solid State Pharmaceutical Centre", + "synonym": [] + }, + { + "id": "501100024313", + "uri": "http://dx.doi.org/10.13039/501100024313", + "name": "Irish Rugby Football Union", + "synonym": [] + }, + { + "id": "100007490", + "uri": "http://dx.doi.org/10.13039/100007490", + "name": "Bausch and Lomb Ireland", + "synonym": [] + }, + { + "id": "100007819", + "uri": "http://dx.doi.org/10.13039/100007819", + "name": "Allergan", + "synonym": [] + }, + { + "id": "100010547", + "uri": "http://dx.doi.org/10.13039/100010547", + "name": "Irish Youth Justice Service", + "synonym": [] + }, + { + "id": "100012733", + "uri": "http://dx.doi.org/10.13039/100012733", + "name": "National Parks and Wildlife Service", + "synonym": [] + }, + { + "id": "100015278", + "uri": "http://dx.doi.org/10.13039/100015278", + "name": "Pfizer Healthcare Ireland", + "synonym": [] + }, + { + "id": "100017144", + "uri": "http://dx.doi.org/10.13039/100017144", + "name": "Shell E and P Ireland", + "synonym": [] + }, + { + "id": "100022895", + "uri": "http://dx.doi.org/10.13039/100022895", + "name": "Health Research Institute, University of Limerick", + "synonym": [] + }, + { + "id": "501100001599", + "uri": "http://dx.doi.org/10.13039/501100001599", + "name": "National Council for Forest Research and Development", + "synonym": [] + }, + { + "id": "501100006554", + "uri": "http://dx.doi.org/10.13039/501100006554", + "name": "IDA Ireland", + "synonym": [] + }, + { + "id": "501100011626", + "uri": "http://dx.doi.org/10.13039/501100011626", + "name": "Energy Policy Research Centre, Economic and Social Research Institute", + "synonym": [] + }, + { + "id": "501100014531", + "uri": "http://dx.doi.org/10.13039/501100014531", + "name": "Physical Education and Sport Sciences Department, University of Limerick", + "synonym": [] + }, + { + "id": "501100014745", + "uri": "http://dx.doi.org/10.13039/501100014745", + "name": "APC Microbiome Institute", + "synonym": [] + }, + { + "id": "501100014826", + "uri": "http://dx.doi.org/10.13039/501100014826", + "name": "ADAPT - Centre for Digital Content Technology", + "synonym": [] + }, + { + "id": "501100020570", + "uri": "http://dx.doi.org/10.13039/501100020570", + "name": "College of Medicine, Nursing and Health Sciences, National University of Ireland, Galway", + "synonym": [] + }, + { + "id": "501100020871", + "uri": "http://dx.doi.org/10.13039/501100020871", + "name": "Bernal Institute, University of Limerick", + "synonym": [] + }, + { + "id": "501100023852", + "uri": "http://dx.doi.org/10.13039/501100023852", + "name": "Moore Institute for Research in the Humanities and Social Studies, University of Galway", + "synonym": [] + } +] \ No newline at end of file diff --git a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala index 9c63b709b..e0fdb9ce4 100644 --- a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala +++ b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala @@ -16,6 +16,7 @@ import org.slf4j.{Logger, LoggerFactory} import java.util import scala.collection.JavaConverters._ import scala.collection.mutable +import scala.io.Source import scala.util.matching.Regex case class CrossrefDT(doi: String, json: String, timestamp: Long) {} @@ -30,11 +31,22 @@ case class mappingAuthor( affiliation: Option[mappingAffiliation] ) {} +case class funderInfo(id:String,uri:String, name:String,synonym:List[String] ) {} + + + case class mappingFunder(name: String, DOI: Option[String], award: Option[List[String]]) {} case object Crossref2Oaf { val logger: Logger = LoggerFactory.getLogger(Crossref2Oaf.getClass) + val irishFunder: List[funderInfo] = { + val s = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/doiboost/crossref/irish_funder.json")).mkString + implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats + lazy val json: org.json4s.JValue = parse(s) + json.extract[List[funderInfo]] + } + val mappingCrossrefType = Map( "book-section" -> "publication", "book" -> "publication", @@ -88,6 +100,11 @@ case object Crossref2Oaf { "report" -> "0017 Report" ) + def getIrishId(doi:String):Option[String] = { + val id =doi.split("/").last + irishFunder.find(f => id.equalsIgnoreCase(f.id) || (f.synonym.nonEmpty && f.synonym.exists(s => s.equalsIgnoreCase(id)))).map(f => f.id) + } + def mappingResult(result: Result, json: JValue, cobjCategory: String): Result = { implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats @@ -467,6 +484,14 @@ case object Crossref2Oaf { if (funders != null) funders.foreach(funder => { if (funder.DOI.isDefined && funder.DOI.get.nonEmpty) { + + if (getIrishId(funder.DOI.get).isDefined) { + val nsPrefix = getIrishId(funder.DOI.get).get.padTo(12, '_') + val targetId = getProjectId(nsPrefix, "1e5e62235d094afd01cd56e65112fc63") + queue += generateRelation(sourceId, targetId, ModelConstants.IS_PRODUCED_BY) + queue += generateRelation(targetId, sourceId, ModelConstants.PRODUCES) + } + funder.DOI.get match { case "10.13039/100010663" | "10.13039/100010661" | "10.13039/501100007601" | "10.13039/501100000780" | "10.13039/100010665" => diff --git a/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala b/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala index fbf6f72c0..7961376c5 100644 --- a/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala +++ b/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala @@ -50,7 +50,7 @@ class CrossrefMappingTest { } } - def checkRelation(generatedOAF: List[Oaf]): Unit = { + def checkRelation(generatedOAF: List[Oaf]): Unit = { val rels: List[Relation] = generatedOAF.filter(p => p.isInstanceOf[Relation]).asInstanceOf[List[Relation]] From eed9fe090220b93789f0d051303fb718814af23d Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 6 Oct 2023 12:31:17 +0200 Subject: [PATCH 039/148] code formatting --- .../dhp/oa/merge/GroupEntitiesSparkJob.java | 4 +- .../dhp/oa/dedup/SparkPropagateRelation.java | 72 +++---- .../group/GroupEntitiesSparkJobTest.java | 190 +++++++++--------- 3 files changed, 133 insertions(+), 133 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java index 87510c108..99981bf6a 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java @@ -33,7 +33,7 @@ import scala.Tuple2; public class GroupEntitiesSparkJob { private static final Logger log = LoggerFactory.getLogger(GroupEntitiesSparkJob.class); - private static final Encoder OAFENTITY_KRYO_ENC = Encoders.kryo(OafEntity.class); + private static final Encoder OAFENTITY_KRYO_ENC = Encoders.kryo(OafEntity.class); public static void main(String[] args) throws Exception { @@ -114,7 +114,7 @@ public class GroupEntitiesSparkJob { Encoders.tuple(Encoders.STRING(), OAFENTITY_KRYO_ENC)); // pivot on "_1" (classname of the entity) - // created columns containing only entities of the same class + // created columns containing only entities of the same class for (Map.Entry e : ModelSupport.entityTypes.entrySet()) { String entity = e.getKey().name(); Class entityClass = e.getValue(); diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java index 739295c91..cb1c70059 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java @@ -67,60 +67,60 @@ public class SparkPropagateRelation extends AbstractSparkAction { log.info("graphOutputPath: '{}'", graphOutputPath); Dataset mergeRels = spark - .read() - .load(DedupUtility.createMergeRelPath(workingPath, "*", "*")) - .as(REL_BEAN_ENC); + .read() + .load(DedupUtility.createMergeRelPath(workingPath, "*", "*")) + .as(REL_BEAN_ENC); // Dataset idsToMerge = mergeRels - .where(col("relClass").equalTo(ModelConstants.MERGES)) - .select(col("source").as("dedupID"), col("target").as("mergedObjectID")) - .distinct(); + .where(col("relClass").equalTo(ModelConstants.MERGES)) + .select(col("source").as("dedupID"), col("target").as("mergedObjectID")) + .distinct(); Dataset allRels = spark - .read() - .schema(REL_BEAN_ENC.schema()) - .json(graphBasePath + "/relation"); + .read() + .schema(REL_BEAN_ENC.schema()) + .json(graphBasePath + "/relation"); Dataset dedupedRels = allRels - .joinWith(idsToMerge, allRels.col("source").equalTo(idsToMerge.col("mergedObjectID")), "left_outer") - .joinWith(idsToMerge, col("_1.target").equalTo(idsToMerge.col("mergedObjectID")), "left_outer") - .select("_1._1", "_1._2.dedupID", "_2.dedupID") - .as(Encoders.tuple(REL_BEAN_ENC, Encoders.STRING(), Encoders.STRING())) - .map((MapFunction, Relation>) t -> { - Relation rel = t._1(); - String newSource = t._2(); - String newTarget = t._3(); + .joinWith(idsToMerge, allRels.col("source").equalTo(idsToMerge.col("mergedObjectID")), "left_outer") + .joinWith(idsToMerge, col("_1.target").equalTo(idsToMerge.col("mergedObjectID")), "left_outer") + .select("_1._1", "_1._2.dedupID", "_2.dedupID") + .as(Encoders.tuple(REL_BEAN_ENC, Encoders.STRING(), Encoders.STRING())) + .map((MapFunction, Relation>) t -> { + Relation rel = t._1(); + String newSource = t._2(); + String newTarget = t._3(); - if (rel.getDataInfo() == null) { - rel.setDataInfo(new DataInfo()); - } + if (rel.getDataInfo() == null) { + rel.setDataInfo(new DataInfo()); + } - if (newSource != null || newTarget != null) { - rel.getDataInfo().setDeletedbyinference(false); + if (newSource != null || newTarget != null) { + rel.getDataInfo().setDeletedbyinference(false); - if (newSource != null) - rel.setSource(newSource); + if (newSource != null) + rel.setSource(newSource); - if (newTarget != null) - rel.setTarget(newTarget); - } + if (newTarget != null) + rel.setTarget(newTarget); + } - return rel; - }, REL_BEAN_ENC); + return rel; + }, REL_BEAN_ENC); // ids of records that are both not deletedbyinference and not invisible Dataset ids = validIds(spark, graphBasePath); // filter relations that point to valid records, can force them to be visible Dataset cleanedRels = dedupedRels - .join(ids, col("source").equalTo(ids.col("id")), "leftsemi") - .join(ids, col("target").equalTo(ids.col("id")), "leftsemi") - .as(REL_BEAN_ENC) - .map((MapFunction) r -> { - r.getDataInfo().setInvisible(false); - return r; - }, REL_KRYO_ENC); + .join(ids, col("source").equalTo(ids.col("id")), "leftsemi") + .join(ids, col("target").equalTo(ids.col("id")), "leftsemi") + .as(REL_BEAN_ENC) + .map((MapFunction) r -> { + r.getDataInfo().setInvisible(false); + return r; + }, REL_KRYO_ENC); Dataset distinctRels = cleanedRels .groupByKey( diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/group/GroupEntitiesSparkJobTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/group/GroupEntitiesSparkJobTest.java index b878e778e..0887adf45 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/group/GroupEntitiesSparkJobTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/group/GroupEntitiesSparkJobTest.java @@ -1,14 +1,14 @@ package eu.dnetlib.dhp.oa.graph.group; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import eu.dnetlib.dhp.common.HdfsSupport; -import eu.dnetlib.dhp.oa.merge.GroupEntitiesSparkJob; -import eu.dnetlib.dhp.schema.common.ModelSupport; -import eu.dnetlib.dhp.schema.oaf.OafEntity; -import eu.dnetlib.dhp.schema.oaf.Result; -import eu.dnetlib.dhp.utils.DHPUtils; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + import org.apache.commons.io.FileUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.function.FilterFunction; @@ -18,108 +18,108 @@ import org.apache.spark.sql.Encoders; import org.apache.spark.sql.SparkSession; import org.junit.jupiter.api.*; -import java.io.IOException; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; -import static org.junit.jupiter.api.Assertions.assertEquals; +import eu.dnetlib.dhp.common.HdfsSupport; +import eu.dnetlib.dhp.oa.merge.GroupEntitiesSparkJob; +import eu.dnetlib.dhp.schema.common.ModelSupport; +import eu.dnetlib.dhp.schema.oaf.OafEntity; +import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.dhp.utils.DHPUtils; @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class GroupEntitiesSparkJobTest { - private static SparkSession spark; + private static SparkSession spark; - private static ObjectMapper mapper = new ObjectMapper() - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + private static ObjectMapper mapper = new ObjectMapper() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - private static Path workingDir; - private Path dataInputPath; + private static Path workingDir; + private Path dataInputPath; - private Path checkpointPath; + private Path checkpointPath; - private Path outputPath; + private Path outputPath; - @BeforeAll - public static void beforeAll() throws IOException { - workingDir = Files.createTempDirectory(GroupEntitiesSparkJob.class.getSimpleName()); + @BeforeAll + public static void beforeAll() throws IOException { + workingDir = Files.createTempDirectory(GroupEntitiesSparkJob.class.getSimpleName()); - SparkConf conf = new SparkConf(); - conf.setAppName(GroupEntitiesSparkJob.class.getSimpleName()); - conf.setMaster("local"); - conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); - conf.registerKryoClasses(ModelSupport.getOafModelClasses()); - spark = SparkSession.builder().config(conf).getOrCreate(); - } + SparkConf conf = new SparkConf(); + conf.setAppName(GroupEntitiesSparkJob.class.getSimpleName()); + conf.setMaster("local"); + conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); + conf.registerKryoClasses(ModelSupport.getOafModelClasses()); + spark = SparkSession.builder().config(conf).getOrCreate(); + } - @BeforeEach - public void beforeEach() throws IOException, URISyntaxException { - dataInputPath = Paths.get(ClassLoader.getSystemResource("eu/dnetlib/dhp/oa/graph/group").toURI()); - checkpointPath = workingDir.resolve("grouped_entity"); - outputPath = workingDir.resolve("dispatched_entity"); - } + @BeforeEach + public void beforeEach() throws IOException, URISyntaxException { + dataInputPath = Paths.get(ClassLoader.getSystemResource("eu/dnetlib/dhp/oa/graph/group").toURI()); + checkpointPath = workingDir.resolve("grouped_entity"); + outputPath = workingDir.resolve("dispatched_entity"); + } - @AfterAll - public static void afterAll() throws IOException { - spark.stop(); - FileUtils.deleteDirectory(workingDir.toFile()); - } + @AfterAll + public static void afterAll() throws IOException { + spark.stop(); + FileUtils.deleteDirectory(workingDir.toFile()); + } - @Test - @Order(1) - void testGroupEntities() throws Exception { - GroupEntitiesSparkJob.main(new String[]{ - "-isSparkSessionManaged", - Boolean.FALSE.toString(), - "-graphInputPath", - dataInputPath.toString(), - "-checkpointPath", - checkpointPath.toString(), - "-outputPath", - outputPath.toString(), - "-filterInvisible", - Boolean.FALSE.toString() - }); + @Test + @Order(1) + void testGroupEntities() throws Exception { + GroupEntitiesSparkJob.main(new String[] { + "-isSparkSessionManaged", + Boolean.FALSE.toString(), + "-graphInputPath", + dataInputPath.toString(), + "-checkpointPath", + checkpointPath.toString(), + "-outputPath", + outputPath.toString(), + "-filterInvisible", + Boolean.FALSE.toString() + }); - Dataset checkpointTable = spark - .read() - .load(checkpointPath.toString()) - .selectExpr("COALESCE(*)") - .as(Encoders.kryo(OafEntity.class)); + Dataset checkpointTable = spark + .read() + .load(checkpointPath.toString()) + .selectExpr("COALESCE(*)") + .as(Encoders.kryo(OafEntity.class)); + assertEquals( + 1, + checkpointTable + .filter( + (FilterFunction) r -> "50|doi_________::09821844208a5cd6300b2bfb13bca1b9" + .equals(r.getId()) && + r.getCollectedfrom().stream().anyMatch(kv -> kv.getValue().equalsIgnoreCase("zenodo"))) + .count()); - assertEquals( - 1, - checkpointTable - .filter( - (FilterFunction) r -> "50|doi_________::09821844208a5cd6300b2bfb13bca1b9" - .equals(r.getId()) && - r.getCollectedfrom().stream().anyMatch(kv -> kv.getValue().equalsIgnoreCase("zenodo"))) - .count()); + Dataset output = spark + .read() + .textFile( + DHPUtils + .toSeq( + HdfsSupport + .listFiles(outputPath.toString(), spark.sparkContext().hadoopConfiguration()))) + .map((MapFunction) s -> mapper.readValue(s, Result.class), Encoders.bean(Result.class)); - - Dataset output = spark - .read() - .textFile( - DHPUtils - .toSeq( - HdfsSupport - .listFiles(outputPath.toString(), spark.sparkContext().hadoopConfiguration()))) - .map((MapFunction) s -> mapper.readValue(s, Result.class), Encoders.bean(Result.class)); - - assertEquals(3, output.count()); - assertEquals( - 2, - output - .map((MapFunction) r -> r.getResulttype().getClassid(), Encoders.STRING()) - .filter((FilterFunction) s -> s.equals("publication")) - .count()); - assertEquals( - 1, - output - .map((MapFunction) r -> r.getResulttype().getClassid(), Encoders.STRING()) - .filter((FilterFunction) s -> s.equals("dataset")) - .count()); - } -} \ No newline at end of file + assertEquals(3, output.count()); + assertEquals( + 2, + output + .map((MapFunction) r -> r.getResulttype().getClassid(), Encoders.STRING()) + .filter((FilterFunction) s -> s.equals("publication")) + .count()); + assertEquals( + 1, + output + .map((MapFunction) r -> r.getResulttype().getClassid(), Encoders.STRING()) + .filter((FilterFunction) s -> s.equals("dataset")) + .count()); + } +} From f759b18bca5260ff61a4603a023aa484585e05fe Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 6 Oct 2023 13:43:20 +0200 Subject: [PATCH 040/148] [SWH] aligned parameter name --- .../resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml index 64dc0d2aa..c625fcb5b 100644 --- a/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-swh/src/main/resources/eu/dnetlib/dhp/swh/oozie_app/workflow.xml @@ -47,7 +47,7 @@ Limit on the number of repo URLs to use (Optional); for debug purposes - resume + resumeFrom Variable that indicates the step to start from @@ -80,8 +80,8 @@ - ${wf:conf('startFrom') eq 'collect-software-repository-urls'} - ${wf:conf('startFrom') eq 'create-swh-actionsets'} + ${wf:conf('resumeFrom') eq 'collect-software-repository-urls'} + ${wf:conf('resumeFrom') eq 'create-swh-actionsets'} From 858931ccb63ee47849129686cbfe3406cee4f997 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 6 Oct 2023 14:03:33 +0200 Subject: [PATCH 041/148] [SWH] compress the output actionset --- .../src/main/java/eu/dnetlib/dhp/swh/PrepareSWHActionsets.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/PrepareSWHActionsets.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/PrepareSWHActionsets.java index 2239244d6..2691d4b7e 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/PrepareSWHActionsets.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/PrepareSWHActionsets.java @@ -11,6 +11,7 @@ import java.util.Optional; import org.apache.commons.io.IOUtils; import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.compress.GzipCodec; import org.apache.hadoop.mapred.SequenceFileOutputFormat; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaPairRDD; @@ -81,7 +82,7 @@ public class PrepareSWHActionsets { JavaPairRDD softwareRDD = prepareActionsets(spark, inputPath, softwareInputPath); softwareRDD .saveAsHadoopFile( - outputPath, Text.class, Text.class, SequenceFileOutputFormat.class); + outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, GzipCodec.class); }); } From 2f3cf6d0e700b3a96c493e1d81cb2274bb45bad2 Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Fri, 6 Oct 2023 12:35:54 +0200 Subject: [PATCH 042/148] Fix cleaning of Pmid where parsing of numbers stopped at first not leading 0' character --- .../eu/dnetlib/dhp/schema/oaf/utils/PmidCleaningRule.java | 4 ++-- .../dnetlib/dhp/schema/oaf/utils/PmidCleaningRuleTest.java | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/PmidCleaningRule.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/PmidCleaningRule.java index d0f5a3b27..c0c451b88 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/PmidCleaningRule.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/PmidCleaningRule.java @@ -7,7 +7,7 @@ import java.util.regex.Pattern; // https://researchguides.stevens.edu/c.php?g=442331&p=6577176 public class PmidCleaningRule { - public static final Pattern PATTERN = Pattern.compile("[1-9]{1,8}"); + public static final Pattern PATTERN = Pattern.compile("0*(\\d{1,8})"); public static String clean(String pmid) { String s = pmid @@ -17,7 +17,7 @@ public class PmidCleaningRule { final Matcher m = PATTERN.matcher(s); if (m.find()) { - return m.group(); + return m.group(1); } return ""; } diff --git a/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/PmidCleaningRuleTest.java b/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/PmidCleaningRuleTest.java index 9562adf7e..295eac85f 100644 --- a/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/PmidCleaningRuleTest.java +++ b/dhp-common/src/test/java/eu/dnetlib/dhp/schema/oaf/utils/PmidCleaningRuleTest.java @@ -9,10 +9,16 @@ class PmidCleaningRuleTest { @Test void testCleaning() { + // leading zeros are removed assertEquals("1234", PmidCleaningRule.clean("01234")); + // tolerant to spaces in the middle assertEquals("1234567", PmidCleaningRule.clean("0123 4567")); + // stop parsing at first not numerical char assertEquals("123", PmidCleaningRule.clean("0123x4567")); + // invalid id leading to empty result assertEquals("", PmidCleaningRule.clean("abc")); + // valid id with zeroes in the number + assertEquals("20794075", PmidCleaningRule.clean("20794075")); } } From 46034630cf62fcb5fcd66366a9be48f62f363d0b Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 6 Oct 2023 14:42:02 +0200 Subject: [PATCH 043/148] [OC] compress the output actionset --- .../actionmanager/opencitations/CreateActionSetSparkJob.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java index 2db756a94..ed0a54efa 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java @@ -10,6 +10,7 @@ import java.util.*; import org.apache.commons.cli.ParseException; import org.apache.commons.io.IOUtils; import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.compress.GzipCodec; import org.apache.hadoop.mapred.SequenceFileOutputFormat; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaPairRDD; @@ -27,7 +28,6 @@ import eu.dnetlib.dhp.actionmanager.opencitations.model.COCI; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.schema.action.AtomicAction; import eu.dnetlib.dhp.schema.common.ModelConstants; -import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.oaf.*; import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions; import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; @@ -100,7 +100,7 @@ public class CreateActionSetSparkJob implements Serializable { getTextTextJavaPairRDD(spark, inputPath, shouldDuplicateRels, COCI) .union(getTextTextJavaPairRDD(spark, inputPath, shouldDuplicateRels, POCI)) - .saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class); + .saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, GzipCodec.class); } private static JavaPairRDD getTextTextJavaPairRDD(SparkSession spark, String inputPath, From 84a58802ab51ea5a496b14059799a78366d74d07 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 6 Oct 2023 14:48:05 +0200 Subject: [PATCH 044/148] [OC] using the common pid cleaning function --- .../opencitations/CreateActionSetSparkJob.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java index ed0a54efa..a367ba852 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java @@ -7,6 +7,7 @@ import java.io.IOException; import java.io.Serializable; import java.util.*; +import eu.dnetlib.dhp.schema.oaf.utils.*; import org.apache.commons.cli.ParseException; import org.apache.commons.io.IOUtils; import org.apache.hadoop.io.Text; @@ -29,10 +30,6 @@ import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.schema.action.AtomicAction; import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.oaf.*; -import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions; -import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; -import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils; -import eu.dnetlib.dhp.schema.oaf.utils.PidType; import eu.dnetlib.dhp.utils.DHPUtils; import scala.Tuple2; @@ -136,19 +133,19 @@ public class CreateActionSetSparkJob implements Serializable { prefix = DOI_PREFIX; citing = prefix + IdentifierFactory - .md5(CleaningFunctions.normalizePidValue(PidType.doi.toString(), value.getCiting())); + .md5(PidCleaner.normalizePidValue(PidType.doi.toString(), value.getCiting())); cited = prefix + IdentifierFactory - .md5(CleaningFunctions.normalizePidValue(PidType.doi.toString(), value.getCited())); + .md5(PidCleaner.normalizePidValue(PidType.doi.toString(), value.getCited())); break; case POCI: prefix = PMID_PREFIX; citing = prefix + IdentifierFactory - .md5(CleaningFunctions.normalizePidValue(PidType.pmid.toString(), value.getCiting())); + .md5(PidCleaner.normalizePidValue(PidType.pmid.toString(), value.getCiting())); cited = prefix + IdentifierFactory - .md5(CleaningFunctions.normalizePidValue(PidType.pmid.toString(), value.getCited())); + .md5(PidCleaner.normalizePidValue(PidType.pmid.toString(), value.getCited())); break; default: throw new IllegalStateException("Invalid prefix: " + p); From ef833840c357f97595ce563393bff3331f0d987d Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 6 Oct 2023 15:48:18 +0200 Subject: [PATCH 045/148] [Doiboost] removed linkage to SFI unidentified project --- .../eu/dnetlib/dhp/doiboost/crossref/irish_funder.json | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/crossref/irish_funder.json b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/crossref/irish_funder.json index 6c6912c2e..15eb1b711 100644 --- a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/crossref/irish_funder.json +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/crossref/irish_funder.json @@ -433,12 +433,6 @@ "name": "Royal Irish Academy", "synonym": [] }, - { - "id": "501100001602", - "uri": "http://dx.doi.org/10.13039/501100001602", - "name": "Science Foundation Ireland", - "synonym": [] - }, { "id": "501100001603", "uri": "http://dx.doi.org/10.13039/501100001603", From 489a082f044cc89215f2183eb06ff764826f8578 Mon Sep 17 00:00:00 2001 From: dimitrispie Date: Mon, 9 Oct 2023 14:00:50 +0300 Subject: [PATCH 046/148] Update step16-createIndicatorsTables.sql Change scripts for gold, hybrid, bronze indicators --- .../scripts/step16-createIndicatorsTables.sql | 353 ++++++++++++------ 1 file changed, 245 insertions(+), 108 deletions(-) diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql index ae95727a6..6af486340 100755 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql @@ -1,4 +1,6 @@ -- Sprint 1 ---- +drop table if exists ${stats_db_name}.indi_pub_green_oa purge; + create table if not exists ${stats_db_name}.indi_pub_green_oa stored as parquet as select distinct p.id, coalesce(green_oa, 0) as green_oa from ${stats_db_name}.publication p @@ -12,6 +14,8 @@ from ${stats_db_name}.publication p or ri.accessright = 'Embargo' or ri.accessright = 'Open Source')) tmp on p.id= tmp.id; +drop table if exists ${stats_db_name}.indi_pub_grey_lit purge; + create table if not exists ${stats_db_name}.indi_pub_grey_lit stored as parquet as select distinct p.id, coalesce(grey_lit, 0) as grey_lit from ${stats_db_name}.publication p @@ -23,6 +27,8 @@ from ${stats_db_name}.publication p not exists (select 1 from ${stats_db_name}.result_classifications rc where type ='Other literature type' and rc.id=p.id)) tmp on p.id=tmp.id; +drop table if exists ${stats_db_name}.indi_pub_doi_from_crossref purge; + create table if not exists ${stats_db_name}.indi_pub_doi_from_crossref stored as parquet as select distinct p.id, coalesce(doi_from_crossref, 0) as doi_from_crossref from ${stats_db_name}.publication p @@ -33,6 +39,8 @@ from ${stats_db_name}.publication p on tmp.id=p.id; -- Sprint 2 ---- +drop table if exists ${stats_db_name}.indi_result_has_cc_licence purge; + create table if not exists ${stats_db_name}.indi_result_has_cc_licence stored as parquet as select distinct r.id, (case when lic='' or lic is null then 0 else 1 end) as has_cc_license from ${stats_db_name}.result r @@ -41,6 +49,8 @@ left outer join (select r.id, license.type as lic from ${stats_db_name}.result r where lower(license.type) LIKE '%creativecommons.org%' OR lower(license.type) LIKE '%cc-%') tmp on r.id= tmp.id; +drop table if exists ${stats_db_name}.indi_result_has_cc_licence_url purge; + create table if not exists ${stats_db_name}.indi_result_has_cc_licence_url stored as parquet as select distinct r.id, case when lic_host='' or lic_host is null then 0 else 1 end as has_cc_license_url from ${stats_db_name}.result r @@ -50,10 +60,14 @@ from ${stats_db_name}.result r WHERE lower(parse_url(license.type, "HOST")) = "creativecommons.org") tmp on r.id= tmp.id; +drop table if exists ${stats_db_name}.indi_pub_has_abstract purge; + create table if not exists ${stats_db_name}.indi_pub_has_abstract stored as parquet as select distinct publication.id, cast(coalesce(abstract, true) as int) has_abstract from ${stats_db_name}.publication; +drop table if exists ${stats_db_name}.indi_result_with_orcid purge; + create table if not exists ${stats_db_name}.indi_result_with_orcid stored as parquet as select distinct r.id, coalesce(has_orcid, 0) as has_orcid from ${stats_db_name}.result r @@ -61,6 +75,9 @@ from ${stats_db_name}.result r on r.id= tmp.id; ---- Sprint 3 ---- + +drop table if exists ${stats_db_name}.indi_funded_result_with_fundref purge; + create table if not exists ${stats_db_name}.indi_funded_result_with_fundref stored as parquet as select distinct r.result as id, coalesce(fundref, 0) as fundref from ${stats_db_name}.project_results r @@ -79,6 +96,8 @@ from ${stats_db_name}.project_results r create TEMPORARY TABLE ${stats_db_name}.tmp AS SELECT ro.organization organization, ro.id, o.name from ${stats_db_name}.result_organization ro join ${stats_db_name}.organization o on o.id=ro.organization where o.name is not null; +drop table if exists ${stats_db_name}.indi_result_org_collab purge; + create table if not exists ${stats_db_name}.indi_result_org_collab stored as parquet as select o1.organization org1, o1.name org1name1, o2.organization org2, o2.name org2name2, count(o1.id) as collaborations from ${stats_db_name}.tmp as o1 @@ -91,6 +110,8 @@ create TEMPORARY TABLE ${stats_db_name}.tmp AS select distinct ro.organization organization, ro.id, o.name, o.country from ${stats_db_name}.result_organization ro join ${stats_db_name}.organization o on o.id=ro.organization where country <> 'UNKNOWN' and o.name is not null; +drop table if exists ${stats_db_name}.indi_result_org_country_collab purge; + create table if not exists ${stats_db_name}.indi_result_org_country_collab stored as parquet as select o1.organization org1,o1.name org1name1, o2.country country2, count(o1.id) as collaborations from ${stats_db_name}.tmp as o1 join ${stats_db_name}.tmp as o2 on o1.id=o2.id @@ -103,6 +124,8 @@ create TEMPORARY TABLE ${stats_db_name}.tmp AS select o.id organization, o.name, ro.project as project from ${stats_db_name}.organization o join ${stats_db_name}.organization_projects ro on o.id=ro.id where o.name is not null; +drop table if exists ${stats_db_name}.indi_project_collab_org purge; + create table if not exists ${stats_db_name}.indi_project_collab_org stored as parquet as select o1.organization org1,o1.name orgname1, o2.organization org2, o2.name orgname2, count(distinct o1.project) as collaborations from ${stats_db_name}.tmp as o1 @@ -117,6 +140,8 @@ select o.id organization, o.name, o.country , ro.project as project from ${stat join ${stats_db_name}.organization_projects ro on o.id=ro.id and o.country <> 'UNKNOWN' and o.name is not null; +drop table if exists ${stats_db_name}.indi_project_collab_org_country purge; + create table if not exists ${stats_db_name}.indi_project_collab_org_country stored as parquet as select o1.organization org1,o1.name org1name, o2.country country2, count(distinct o1.project) as collaborations from ${stats_db_name}.tmp as o1 @@ -126,6 +151,8 @@ group by o1.organization, o2.country, o1.name; drop table ${stats_db_name}.tmp purge; +drop table if exists ${stats_db_name}.indi_funder_country_collab purge; + create table if not exists ${stats_db_name}.indi_funder_country_collab stored as parquet as with tmp as (select funder, project, country from ${stats_db_name}.organization_projects op join ${stats_db_name}.organization o on o.id=op.id @@ -142,6 +169,8 @@ select distinct country, ro.id as result from ${stats_db_name}.organization o join ${stats_db_name}.result_organization ro on o.id=ro.organization where country <> 'UNKNOWN' and o.name is not null; +drop table if exists ${stats_db_name}.indi_result_country_collab purge; + create table if not exists ${stats_db_name}.indi_result_country_collab stored as parquet as select o1.country country1, o2.country country2, count(o1.result) as collaborations from ${stats_db_name}.tmp as o1 @@ -152,6 +181,8 @@ group by o1.country, o2.country; drop table ${stats_db_name}.tmp purge; ---- Sprint 4 ---- +drop table if exists ${stats_db_name}.indi_pub_diamond purge; + create table if not exists ${stats_db_name}.indi_pub_diamond stored as parquet as select distinct pd.id, coalesce(in_diamond_journal, 0) as in_diamond_journal from ${stats_db_name}.publication_datasources pd @@ -162,6 +193,8 @@ from ${stats_db_name}.publication_datasources pd and (ps.journal_is_in_doaj=true or ps.journal_is_oa=true) and ps.has_apc=false) tmp on pd.id=tmp.id; +drop table if exists ${stats_db_name}.indi_pub_in_transformative purge; + create table if not exists ${stats_db_name}.indi_pub_in_transformative stored as parquet as select distinct pd.id, coalesce(is_transformative, 0) as is_transformative from ${stats_db_name}.publication pd @@ -172,6 +205,8 @@ from ${stats_db_name}.publication pd and ps.is_transformative_journal=true) tmp on pd.id=tmp.id; +drop table if exists ${stats_db_name}.indi_pub_closed_other_open purge; + create table if not exists ${stats_db_name}.indi_pub_closed_other_open stored as parquet as select distinct ri.id, coalesce(pub_closed_other_open, 0) as pub_closed_other_open from ${stats_db_name}.result_instance ri left outer join @@ -183,10 +218,14 @@ select distinct ri.id, coalesce(pub_closed_other_open, 0) as pub_closed_other_op on tmp.id=ri.id; ---- Sprint 5 ---- +drop table if exists ${stats_db_name}.indi_result_no_of_copies purge; + create table if not exists ${stats_db_name}.indi_result_no_of_copies stored as parquet as select id, count(id) as number_of_copies from ${stats_db_name}.result_instance group by id; ---- Sprint 6 ---- +drop table if exists ${stats_db_name}.indi_pub_downloads purge; + create table if not exists ${stats_db_name}.indi_pub_downloads stored as parquet as SELECT result_id, sum(downloads) no_downloads from openaire_prod_usage_stats.usage_stats join ${stats_db_name}.publication on result_id=id @@ -196,6 +235,8 @@ order by no_downloads desc; --ANALYZE TABLE ${stats_db_name}.indi_pub_downloads COMPUTE STATISTICS; +drop table if exists ${stats_db_name}.indi_pub_downloads_datasource purge; + create table if not exists ${stats_db_name}.indi_pub_downloads_datasource stored as parquet as SELECT result_id, repository_id, sum(downloads) no_downloads from openaire_prod_usage_stats.usage_stats join ${stats_db_name}.publication on result_id=id @@ -203,12 +244,16 @@ where downloads>0 GROUP BY result_id, repository_id order by result_id; +drop table if exists ${stats_db_name}.indi_pub_downloads_year purge; + create table if not exists ${stats_db_name}.indi_pub_downloads_year stored as parquet as SELECT result_id, cast(substring(us.`date`, 1,4) as int) as `year`, sum(downloads) no_downloads from openaire_prod_usage_stats.usage_stats us join ${stats_db_name}.publication on result_id=id where downloads>0 GROUP BY result_id, substring(us.`date`, 1,4); +drop table if exists ${stats_db_name}.indi_pub_downloads_datasource_year purge; + create table if not exists ${stats_db_name}.indi_pub_downloads_datasource_year stored as parquet as SELECT result_id, cast(substring(us.`date`, 1,4) as int) as `year`, repository_id, sum(downloads) no_downloads from openaire_prod_usage_stats.usage_stats us join ${stats_db_name}.publication on result_id=id @@ -216,54 +261,81 @@ where downloads>0 GROUP BY result_id, repository_id, substring(us.`date`, 1,4); ---- Sprint 7 ---- +drop table if exists ${stats_db_name}.indi_pub_gold_oa purge; + +--create table if not exists ${stats_db_name}.indi_pub_gold_oa stored as parquet as +-- WITH gold_oa AS ( SELECT +-- issn_l, +-- journal_is_in_doaj, +-- journal_is_oa, +-- issn_1 as issn +-- FROM +-- STATS_EXT.oa_journals +-- WHERE +-- issn_1 != "" +-- UNION +-- ALL SELECT +-- issn_l, +-- journal_is_in_doaj, +-- journal_is_oa, +-- issn_2 as issn +-- FROM +-- STATS_EXT.oa_journals +-- WHERE +-- issn_2 != "" ), issn AS ( SELECT +-- * +-- FROM +--( SELECT +-- id, +-- issn_printed as issn +-- FROM +-- ${stats_db_name}.datasource +-- WHERE +-- issn_printed IS NOT NULL +-- UNION ALL +-- SELECT +-- id, +-- issn_online as issn +-- FROM +-- ${stats_db_name}.datasource +-- WHERE +-- issn_online IS NOT NULL or id like '%doajarticles%') as issn +-- WHERE +-- LENGTH(issn) > 7) +--SELECT +-- DISTINCT pd.id, coalesce(is_gold, 0) as is_gold +--FROM +-- ${stats_db_name}.publication_datasources pd +-- left outer join( +-- select pd.id, 1 as is_gold FROM ${stats_db_name}.publication_datasources pd +-- JOIN issn on issn.id=pd.datasource +-- JOIN gold_oa on issn.issn = gold_oa.issn) tmp +-- on pd.id=tmp.id; + create table if not exists ${stats_db_name}.indi_pub_gold_oa stored as parquet as - WITH gold_oa AS ( SELECT - issn_l, - journal_is_in_doaj, - journal_is_oa, - issn_1 as issn - FROM - STATS_EXT.oa_journals - WHERE - issn_1 != "" - UNION - ALL SELECT - issn_l, - journal_is_in_doaj, - journal_is_oa, - issn_2 as issn - FROM - STATS_EXT.oa_journals - WHERE - issn_2 != "" ), issn AS ( SELECT - * - FROM -( SELECT - id, - issn_printed as issn - FROM - ${stats_db_name}.datasource - WHERE - issn_printed IS NOT NULL - UNION ALL - SELECT - id, - issn_online as issn - FROM - ${stats_db_name}.datasource - WHERE - issn_online IS NOT NULL or id like '%doajarticles%') as issn - WHERE - LENGTH(issn) > 7) -SELECT - DISTINCT pd.id, coalesce(is_gold, 0) as is_gold -FROM - ${stats_db_name}.publication_datasources pd - left outer join( - select pd.id, 1 as is_gold FROM ${stats_db_name}.publication_datasources pd - JOIN issn on issn.id=pd.datasource - JOIN gold_oa on issn.issn = gold_oa.issn) tmp - on pd.id=tmp.id; +with gold_oa as ( +SELECT issn,issn_l from stats_ext.issn_gold_oa_dataset_v5), +issn AS (SELECT * FROM +(SELECT id,issn_printed as issn FROM ${stats_db_name}.datasource +WHERE issn_printed IS NOT NULL +UNION ALL +SELECT id, issn_online as issn FROM ${stats_db_name}.datasource +WHERE issn_online IS NOT NULL or id like '%doajarticles%') as issn +WHERE LENGTH(issn) > 7), +alljournals AS(select issn, issn_l from stats_ext.alljournals +where journal_is_in_doaj=true or journal_is_oa=true) +SELECT DISTINCT pd.id, coalesce(is_gold, 0) as is_gold +FROM ${stats_db_name}.publication_datasources pd +left outer join ( +select pd.id, 1 as is_gold FROM ${stats_db_name}.publication_datasources pd +JOIN issn on issn.id=pd.datasource +JOIN gold_oa on issn.issn = gold_oa.issn +join alljournals on issn.issn=alljournals.issn +left outer join ${stats_db_name}.result_instance ri on ri.id=pd.id +and ri.accessright!='Closed Access' and ri.accessright_uw='gold') tmp +on pd.id=tmp.id; + +drop table if exists ${stats_db_name}.indi_pub_hybrid_oa_with_cc purge; create table if not exists ${stats_db_name}.indi_pub_hybrid_oa_with_cc stored as parquet as WITH hybrid_oa AS ( @@ -296,56 +368,71 @@ FROM ${stats_db_name}.publication_datasources pd JOIN ${stats_db_name}.indi_pub_gold_oa ga on pd.id=ga.id where cc.has_cc_license=1 and ga.is_gold=0) tmp on pd.id=tmp.id; +drop table if exists ${stats_db_name}.indi_pub_hybrid purge; + +--create table if not exists ${stats_db_name}.indi_pub_hybrid stored as parquet as +-- WITH gold_oa AS ( SELECT +-- issn_l, +-- journal_is_in_doaj, +-- journal_is_oa, +-- issn_1 as issn, +-- has_apc +-- FROM +-- STATS_EXT.oa_journals +-- WHERE +-- issn_1 != "" +-- UNION +-- ALL SELECT +-- issn_l, +-- journal_is_in_doaj, +-- journal_is_oa, +-- issn_2 as issn, +-- has_apc +-- FROM +-- STATS_EXT.oa_journals +-- WHERE +-- issn_2 != "" ), issn AS ( SELECT +-- * +-- FROM +--( SELECT +-- id, +-- issn_printed as issn +-- FROM +-- ${stats_db_name}.datasource +-- WHERE +-- issn_printed IS NOT NULL +-- UNION ALL +-- SELECT +-- id, +-- issn_online as issn +-- FROM +-- ${stats_db_name}.datasource +-- WHERE +-- issn_online IS NOT NULL or id like '%doajarticles%') as issn +-- WHERE +-- LENGTH(issn) > 7) +--select distinct pd.id, coalesce(is_hybrid, 0) as is_hybrid +--from ${stats_db_name}.publication_datasources pd +-- left outer join ( +-- select pd.id, 1 as is_hybrid from ${stats_db_name}.publication_datasources pd +-- join ${stats_db_name}.datasource d on d.id=pd.datasource +-- join issn on issn.id=pd.datasource +-- join gold_oa on issn.issn=gold_oa.issn +-- where (gold_oa.journal_is_in_doaj=false or gold_oa.journal_is_oa=false))tmp +-- on pd.id=tmp.id; + create table if not exists ${stats_db_name}.indi_pub_hybrid stored as parquet as - WITH gold_oa AS ( SELECT - issn_l, - journal_is_in_doaj, - journal_is_oa, - issn_1 as issn, - has_apc - FROM - STATS_EXT.oa_journals - WHERE - issn_1 != "" - UNION - ALL SELECT - issn_l, - journal_is_in_doaj, - journal_is_oa, - issn_2 as issn, - has_apc - FROM - STATS_EXT.oa_journals - WHERE - issn_2 != "" ), issn AS ( SELECT - * - FROM -( SELECT - id, - issn_printed as issn - FROM - ${stats_db_name}.datasource - WHERE - issn_printed IS NOT NULL - UNION ALL - SELECT - id, - issn_online as issn - FROM - ${stats_db_name}.datasource - WHERE - issn_online IS NOT NULL or id like '%doajarticles%') as issn - WHERE - LENGTH(issn) > 7) -select distinct pd.id, coalesce(is_hybrid, 0) as is_hybrid -from ${stats_db_name}.publication_datasources pd - left outer join ( - select pd.id, 1 as is_hybrid from ${stats_db_name}.publication_datasources pd - join ${stats_db_name}.datasource d on d.id=pd.datasource - join issn on issn.id=pd.datasource - join gold_oa on issn.issn=gold_oa.issn - where (gold_oa.journal_is_in_doaj=false or gold_oa.journal_is_oa=false))tmp - on pd.id=tmp.id; +select pd.id,coalesce(is_hybrid,0) is_hybrid from ${stats_db_name}.publication_datasources pd +left outer join (select pd.id, 1 as is_hybrid from ${stats_db_name}.publication_datasources pd +join ${stats_db_name}.datasource d on pd.datasource=d.id +join ${stats_db_name}.result_instance ri on ri.id=pd.id +join ${stats_db_name}.indi_pub_gold_oa indi_gold on indi_gold.id=pd.id +join ${stats_db_name}.result_accessroute ra on ra.id=pd.id +where d.type like '%Journal%' and ri.accessright!='Closed Access' and (ri.accessright_uw!='gold' +or indi_gold.is_gold=0) and (ra.accessroute='hybrid' or ri.license is not null)) tmp +on pd.id=tmp.id; + +drop table if exists ${stats_db_name}.indi_org_fairness purge; create table if not exists ${stats_db_name}.indi_org_fairness stored as parquet as --return results with PIDs, and rich metadata group by organization @@ -381,6 +468,8 @@ select ro.organization, count(distinct ro.id) no_allresults from ${stats_db_name where cast(year as int)>2003 group by ro.organization; +drop table if exists ${stats_db_name}.indi_org_fairness_pub_pr purge; + create table if not exists ${stats_db_name}.indi_org_fairness_pub_pr stored as parquet as select ar.organization, rf.no_result_fair/ar.no_allresults org_fairness from ${stats_db_name}.allresults ar @@ -400,6 +489,8 @@ CREATE TEMPORARY TABLE ${stats_db_name}.allresults as select year, ro.organizati where cast(year as int)>2003 group by ro.organization, year; +drop table if exists ${stats_db_name}.indi_org_fairness_pub_year purge; + create table if not exists ${stats_db_name}.indi_org_fairness_pub_year stored as parquet as select allresults.year, allresults.organization, result_fair.no_result_fair/allresults.no_allresults org_fairness from ${stats_db_name}.allresults @@ -422,6 +513,8 @@ CREATE TEMPORARY TABLE ${stats_db_name}.allresults as where cast(year as int)>2003 group by ro.organization; +drop table if exists ${stats_db_name}.indi_org_fairness_pub purge; + create table if not exists ${stats_db_name}.indi_org_fairness_pub as select ar.organization, rf.no_result_fair/ar.no_allresults org_fairness from ${stats_db_name}.allresults ar join ${stats_db_name}.result_fair rf @@ -443,6 +536,8 @@ CREATE TEMPORARY TABLE ${stats_db_name}.allresults as where cast(year as int)>2003 group by ro.organization, year; +drop table if exists ${stats_db_name}.indi_org_fairness_year purge; + create table if not exists ${stats_db_name}.indi_org_fairness_year stored as parquet as select cast(allresults.year as int) year, allresults.organization, result_fair.no_result_fair/allresults.no_allresults org_fairness from ${stats_db_name}.allresults @@ -464,6 +559,8 @@ CREATE TEMPORARY TABLE ${stats_db_name}.allresults as where cast(year as int) >2003 group by ro.organization, year; +drop table if exists ${stats_db_name}.indi_org_findable_year purge; + create table if not exists ${stats_db_name}.indi_org_findable_year stored as parquet as select cast(allresults.year as int) year, allresults.organization, result_with_pid.no_result_with_pid/allresults.no_allresults org_findable from ${stats_db_name}.allresults @@ -485,6 +582,8 @@ select ro.organization, count(distinct ro.id) no_allresults from ${stats_db_name where cast(year as int) >2003 group by ro.organization; +drop table if exists ${stats_db_name}.indi_org_findable purge; + create table if not exists ${stats_db_name}.indi_org_findable stored as parquet as select allresults.organization, result_with_pid.no_result_with_pid/allresults.no_allresults org_findable from ${stats_db_name}.allresults @@ -549,6 +648,8 @@ select software_oa.organization, software_oa.no_oasoftware/allsoftware.no_allsof from ${stats_db_name}.allsoftware join ${stats_db_name}.software_oa on allsoftware.organization=software_oa.organization; +drop table if exists ${stats_db_name}.indi_org_openess purge; + create table if not exists ${stats_db_name}.indi_org_openess stored as parquet as select allpubsshare.organization, (p+if(isnull(s),0,s)+if(isnull(d),0,d))/(1+(case when s is null then 0 else 1 end) @@ -624,6 +725,7 @@ select allsoftware.year, software_oa.organization, software_oa.no_oasoftware/all from ${stats_db_name}.allsoftware join ${stats_db_name}.software_oa on allsoftware.organization=software_oa.organization where cast(allsoftware.year as INT)=cast(software_oa.year as int); +drop table if exists ${stats_db_name}.indi_org_openess_year purge; create table if not exists ${stats_db_name}.indi_org_openess_year stored as parquet as select cast(allpubsshare.year as int) year, allpubsshare.organization, @@ -647,6 +749,8 @@ DROP TABLE ${stats_db_name}.allpubsshare purge; DROP TABLE ${stats_db_name}.alldatasetssshare purge; DROP TABLE ${stats_db_name}.allsoftwaresshare purge; +drop table if exists ${stats_db_name}.indi_pub_has_preprint purge; + create table if not exists ${stats_db_name}.indi_pub_has_preprint stored as parquet as select distinct p.id, coalesce(has_preprint, 0) as has_preprint from ${stats_db_name}.publication_classifications p @@ -655,6 +759,7 @@ from ${stats_db_name}.publication_classifications p from ${stats_db_name}.publication_classifications p where p.type='Preprint') tmp on p.id= tmp.id; +drop table if exists ${stats_db_name}.indi_pub_in_subscribed purge; create table if not exists ${stats_db_name}.indi_pub_in_subscribed stored as parquet as select distinct p.id, coalesce(is_subscription, 0) as is_subscription @@ -667,6 +772,8 @@ from ${stats_db_name}.publication p where g.is_gold=0 and h.is_hybrid=0 and t.is_transformative=0) tmp on p.id=tmp.id; +drop table if exists ${stats_db_name}.indi_result_with_pid purge; + create table if not exists ${stats_db_name}.indi_result_with_pid as select distinct p.id, coalesce(result_with_pid, 0) as result_with_pid from ${stats_db_name}.result p @@ -679,6 +786,8 @@ CREATE TEMPORARY TABLE ${stats_db_name}.pub_fos_totals as select rf.id, count(distinct lvl3) totals from ${stats_db_name}.result_fos rf group by rf.id; +drop table if exists ${stats_db_name}.indi_pub_interdisciplinarity purge; + create table if not exists ${stats_db_name}.indi_pub_interdisciplinarity as select distinct p.id as id, coalesce(is_interdisciplinary, 0) as is_interdisciplinary @@ -689,18 +798,31 @@ where totals>1) tmp on p.id=tmp.id; drop table ${stats_db_name}.pub_fos_totals purge; -create table if not exists ${stats_db_name}.indi_pub_bronze_oa stored as parquet as -select distinct p.id, coalesce(is_bronze_oa,0) as is_bronze_oa -from ${stats_db_name}.publication p -left outer join -(select p.id, 1 as is_bronze_oa from ${stats_db_name}.publication p -join ${stats_db_name}.indi_result_has_cc_licence cc on cc.id=p.id -join ${stats_db_name}.indi_pub_gold_oa ga on ga.id=p.id -join ${stats_db_name}.result_instance ri on ri.id=p.id -join ${stats_db_name}.datasource d on d.id=ri.hostedby -where cc.has_cc_license=0 and ga.is_gold=0 -and (d.type='Journal' or d.type='Journal Aggregator/Publisher') -and ri.accessright='Open Access') tmp on tmp.id=p.id; +drop table if exists ${stats_db_name}.indi_pub_bronze_oa purge; + +--create table if not exists ${stats_db_name}.indi_pub_bronze_oa stored as parquet as +--select distinct p.id, coalesce(is_bronze_oa,0) as is_bronze_oa +--from ${stats_db_name}.publication p +--left outer join +--(select p.id, 1 as is_bronze_oa from ${stats_db_name}.publication p +--join ${stats_db_name}.indi_result_has_cc_licence cc on cc.id=p.id +--join ${stats_db_name}.indi_pub_gold_oa ga on ga.id=p.id +--join ${stats_db_name}.result_instance ri on ri.id=p.id +--join ${stats_db_name}.datasource d on d.id=ri.hostedby +--where cc.has_cc_license=0 and ga.is_gold=0 +--and (d.type='Journal' or d.type='Journal Aggregator/Publisher') +--and ri.accessright='Open Access') tmp on tmp.id=p.id; + +create table ${stats_db_name}.indi_pub_bronze stored as parquet as +select pd.id,coalesce(is_bronze_oa,0) is_bronze_oa from ${stats_db_name}.publication_datasources pd +left outer join (select pd.id, 1 as is_bronze_oa from ${stats_db_name}.publication_datasources pd +join ${stats_db_name}.datasource d on pd.datasource=d.id +join ${stats_db_name}.result_instance ri on ri.id=pd.id +join ${stats_db_name}.indi_pub_gold_oa indi_gold on indi_gold.id=pd.id +join ${stats_db_name}.result_accessroute ra on ra.id=pd.id +where d.type like '%Journal%' and ri.accessright!='Closed Access' and (ri.accessright_uw!='gold' +or indi_gold.is_gold=0) and (ra.accessroute='bronze' or ri.license is null)) tmp +on pd.id=tmp.id; CREATE TEMPORARY TABLE ${stats_db_name}.project_year_result_year as select p.id project_id, acronym, r.id result_id, r.year, p.end_year @@ -709,6 +831,8 @@ join ${stats_db_name}.result_projects rp on p.id=rp.project join ${stats_db_name}.result r on r.id=rp.id where p.end_year is NOT NULL and r.year is not null; +drop table if exists ${stats_db_name}.indi_is_project_result_after purge; + create table if not exists ${stats_db_name}.indi_is_project_result_after stored as parquet as select pry.project_id, pry.acronym, pry.result_id, coalesce(is_project_result_after, 0) as is_project_result_after @@ -719,6 +843,8 @@ where pry.year>pry.end_year) tmp on pry.result_id=tmp.result_id; drop table ${stats_db_name}.project_year_result_year purge; +drop table ${stats_db_name}.indi_is_funder_plan_s purge; + create table if not exists ${stats_db_name}.indi_is_funder_plan_s stored as parquet as select distinct f.id, f.name, coalesce(is_funder_plan_s, 0) as is_funder_plan_s from ${stats_db_name}.funder f @@ -727,6 +853,7 @@ from ${stats_db_name}.funder f on f.name= tmp.name; --Funder Fairness +drop table ${stats_db_name}.indi_funder_fairness purge; create table if not exists ${stats_db_name}.indi_funder_fairness stored as parquet as with result_fair as @@ -745,6 +872,8 @@ from allresults join result_fair on result_fair.funder=allresults.funder; --RIs Fairness +drop table ${stats_db_name}.indi_ris_fairness purge; + create table if not exists ${stats_db_name}.indi_ris_fairness stored as parquet as with result_contexts as (select distinct rc.id, context.name ri_initiative from ${stats_db_name}.result_concepts rc @@ -830,6 +959,8 @@ select software_oa.funder, software_oa.no_oasoftware/allsoftware.no_allsoftware from ${stats_db_name}.allsoftware join ${stats_db_name}.software_oa on allsoftware.funder=software_oa.funder; +drop table ${stats_db_name}.indi_funder_openess purge; + create table if not exists ${stats_db_name}.indi_funder_openess stored as parquet as select allpubsshare.funder, (p+if(isnull(s),0,s)+if(isnull(d),0,d))/(1+(case when s is null then 0 else 1 end) @@ -916,6 +1047,8 @@ select software_oa.ri_initiative, software_oa.no_oasoftware/allsoftware.no_allso from ${stats_db_name}.allsoftware join ${stats_db_name}.software_oa on allsoftware.ri_initiative=software_oa.ri_initiative; +drop table ${stats_db_name}.indi_ris_openess purge; + create table if not exists ${stats_db_name}.indi_ris_openess stored as parquet as select allpubsshare.ri_initiative, (p+if(isnull(s),0,s)+if(isnull(d),0,d))/(1+(case when s is null then 0 else 1 end) @@ -940,6 +1073,8 @@ DROP TABLE ${stats_db_name}.alldatasetssshare purge; DROP TABLE ${stats_db_name}.allsoftwaresshare purge; --Funder Findability +drop table ${stats_db_name}.indi_funder_findable purge; + create table if not exists ${stats_db_name}.indi_funder_findable stored as parquet as with result_findable as (select p.funder funder, count(distinct rp.id) no_result_findable from ${stats_db_name}.result_projects rp @@ -958,6 +1093,8 @@ from allresults join result_findable on result_findable.funder=allresults.funder; --RIs Findability +drop table ${stats_db_name}.indi_ris_findable purge; + create table if not exists ${stats_db_name}.indi_ris_findable stored as parquet as with result_contexts as (select distinct rc.id, context.name ri_initiative from ${stats_db_name}.result_concepts rc From 17586f0ff8d0e8d6225ecff52fa072ed4e66c3d4 Mon Sep 17 00:00:00 2001 From: dimitrispie Date: Mon, 9 Oct 2023 14:21:31 +0300 Subject: [PATCH 047/148] Update step20-createMonitorDB.sql Add result_orcid table to monitor dbs --- .../oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql index 586bee347..d5d242230 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql @@ -162,6 +162,8 @@ create table TARGET.result_fos stored as parquet as select * from SOURCE.result_ create table TARGET.result_accessroute stored as parquet as select * from SOURCE.result_accessroute orig where exists (select 1 from TARGET.result r where r.id=orig.id); --ANALYZE TABLE TARGET.result_accessroute COMPUTE STATISTICS; +create table TARGET.result_orcid stored as parquet as select * from SOURCE.result_orcid orig where exists (select 1 from TARGET.result r where r.id=orig.id); + create view TARGET.foo1 as select * from SOURCE.result_result rr where rr.source in (select id from TARGET.result); create view TARGET.foo2 as select * from SOURCE.result_result rr where rr.target in (select id from TARGET.result); create table TARGET.result_result STORED AS PARQUET as select distinct * from (select * from TARGET.foo1 union all select * from TARGET.foo2) foufou; From 3d6be209899806d9c319eb034f26b1248e671091 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Mon, 9 Oct 2023 14:26:33 +0200 Subject: [PATCH 048/148] changes to use the API instead of the IS the get the information for the communities to be used during bulktagging and context propagation --- .../bipfinder/SparkAtomicActionScoreJob.java | 2 +- .../dhp/oa/dedup/SparkPropagateRelation.java | 72 +++---- .../eu/dnetlib/dhp/api/QueryCommunityAPI.java | 78 +++++++ .../main/java/eu/dnetlib/dhp/api/Utils.java | 136 +++++++++++++ .../api/model/CommunityContentprovider.java | 43 ++++ .../model/CommunityEntityMap.java} | 6 +- .../dnetlib/dhp/api/model/CommunityModel.java | 108 ++++++++++ .../dhp/api/model/CommunitySummary.java | 16 ++ .../dnetlib/dhp/api/model/ContentModel.java | 50 +++++ .../dnetlib/dhp/api/model/DatasourceList.java | 12 ++ .../dhp/api/model/OrganizationList.java | 15 ++ .../dnetlib/dhp/api/model/ProjectModel.java | 23 +++ .../dnetlib/dhp/bulktag/SparkBulkTagJob.java | 4 +- .../dhp/bulktag/community/Community.java | 10 +- .../community/CommunityConfiguration.java | 8 +- .../CommunityConfigurationFactory.java | 13 +- .../dhp/bulktag/community/Constraint.java | 20 +- .../community/SelectionConstraints.java | 3 +- .../PrepareResultCommunitySet.java | 17 +- .../oozie_app/workflow.xml | 8 +- .../dnetlib/dhp/bulktag/BulkTagJobTest.java | 38 ++++ .../CommunityConfigurationFactoryTest.java | 2 +- .../dhp/bulktag/QueryCommunityAPITest.java | 91 +++++++++ .../group/GroupEntitiesSparkJobTest.java | 190 +++++++++--------- 24 files changed, 787 insertions(+), 178 deletions(-) create mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java create mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java create mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityContentprovider.java rename dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/{resulttocommunityfromorganization/OrganizationMap.java => api/model/CommunityEntityMap.java} (59%) create mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityModel.java create mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunitySummary.java create mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/ContentModel.java create mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/DatasourceList.java create mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/OrganizationList.java create mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/ProjectModel.java create mode 100644 dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/QueryCommunityAPITest.java diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/SparkAtomicActionScoreJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/SparkAtomicActionScoreJob.java index 040c89782..8adc88920 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/SparkAtomicActionScoreJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/SparkAtomicActionScoreJob.java @@ -95,7 +95,7 @@ public class SparkAtomicActionScoreJob implements Serializable { return projectScores.map((MapFunction) bipProjectScores -> { Project project = new Project(); - project.setId(bipProjectScores.getProjectId()); + // project.setId(bipProjectScores.getProjectId()); project.setMeasures(bipProjectScores.toMeasures()); return project; }, Encoders.bean(Project.class)) diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java index 739295c91..cb1c70059 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkPropagateRelation.java @@ -67,60 +67,60 @@ public class SparkPropagateRelation extends AbstractSparkAction { log.info("graphOutputPath: '{}'", graphOutputPath); Dataset mergeRels = spark - .read() - .load(DedupUtility.createMergeRelPath(workingPath, "*", "*")) - .as(REL_BEAN_ENC); + .read() + .load(DedupUtility.createMergeRelPath(workingPath, "*", "*")) + .as(REL_BEAN_ENC); // Dataset idsToMerge = mergeRels - .where(col("relClass").equalTo(ModelConstants.MERGES)) - .select(col("source").as("dedupID"), col("target").as("mergedObjectID")) - .distinct(); + .where(col("relClass").equalTo(ModelConstants.MERGES)) + .select(col("source").as("dedupID"), col("target").as("mergedObjectID")) + .distinct(); Dataset allRels = spark - .read() - .schema(REL_BEAN_ENC.schema()) - .json(graphBasePath + "/relation"); + .read() + .schema(REL_BEAN_ENC.schema()) + .json(graphBasePath + "/relation"); Dataset dedupedRels = allRels - .joinWith(idsToMerge, allRels.col("source").equalTo(idsToMerge.col("mergedObjectID")), "left_outer") - .joinWith(idsToMerge, col("_1.target").equalTo(idsToMerge.col("mergedObjectID")), "left_outer") - .select("_1._1", "_1._2.dedupID", "_2.dedupID") - .as(Encoders.tuple(REL_BEAN_ENC, Encoders.STRING(), Encoders.STRING())) - .map((MapFunction, Relation>) t -> { - Relation rel = t._1(); - String newSource = t._2(); - String newTarget = t._3(); + .joinWith(idsToMerge, allRels.col("source").equalTo(idsToMerge.col("mergedObjectID")), "left_outer") + .joinWith(idsToMerge, col("_1.target").equalTo(idsToMerge.col("mergedObjectID")), "left_outer") + .select("_1._1", "_1._2.dedupID", "_2.dedupID") + .as(Encoders.tuple(REL_BEAN_ENC, Encoders.STRING(), Encoders.STRING())) + .map((MapFunction, Relation>) t -> { + Relation rel = t._1(); + String newSource = t._2(); + String newTarget = t._3(); - if (rel.getDataInfo() == null) { - rel.setDataInfo(new DataInfo()); - } + if (rel.getDataInfo() == null) { + rel.setDataInfo(new DataInfo()); + } - if (newSource != null || newTarget != null) { - rel.getDataInfo().setDeletedbyinference(false); + if (newSource != null || newTarget != null) { + rel.getDataInfo().setDeletedbyinference(false); - if (newSource != null) - rel.setSource(newSource); + if (newSource != null) + rel.setSource(newSource); - if (newTarget != null) - rel.setTarget(newTarget); - } + if (newTarget != null) + rel.setTarget(newTarget); + } - return rel; - }, REL_BEAN_ENC); + return rel; + }, REL_BEAN_ENC); // ids of records that are both not deletedbyinference and not invisible Dataset ids = validIds(spark, graphBasePath); // filter relations that point to valid records, can force them to be visible Dataset cleanedRels = dedupedRels - .join(ids, col("source").equalTo(ids.col("id")), "leftsemi") - .join(ids, col("target").equalTo(ids.col("id")), "leftsemi") - .as(REL_BEAN_ENC) - .map((MapFunction) r -> { - r.getDataInfo().setInvisible(false); - return r; - }, REL_KRYO_ENC); + .join(ids, col("source").equalTo(ids.col("id")), "leftsemi") + .join(ids, col("target").equalTo(ids.col("id")), "leftsemi") + .as(REL_BEAN_ENC) + .map((MapFunction) r -> { + r.getDataInfo().setInvisible(false); + return r; + }, REL_KRYO_ENC); Dataset distinctRels = cleanedRels .groupByKey( diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java new file mode 100644 index 000000000..635ee2027 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java @@ -0,0 +1,78 @@ + +package eu.dnetlib.dhp.api; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; + +import org.apache.http.HttpHeaders; +import org.jetbrains.annotations.NotNull; + +import com.google.gson.Gson; + +/** + * @author miriam.baglioni + * @Date 06/10/23 + */ +public class QueryCommunityAPI { + private static final String baseUrl = "https://services.openaire.eu/openaire/"; + + private static String get(String geturl) throws IOException{ + URL url = new URL(geturl); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setDoOutput(true); + conn.setRequestMethod("GET"); + + int responseCode = conn.getResponseCode(); + String body = getBody(conn); + conn.disconnect(); + if (responseCode != HttpURLConnection.HTTP_OK) + throw new IOException("Unexpected code " + responseCode + body); + + return body; + } + + public static String communities() throws IOException { + return get(baseUrl + "community/communities"); + } + + public static String community(String id) throws IOException { + return get(baseUrl + "community/" + id); + } + + public static String communityDatasource(String id)throws IOException{ + return get(baseUrl + "community/" + id + "/contentproviders"); + + } + + public static String communityPropagationOrganization(String id) throws IOException { + return get(baseUrl + "community/" + id + "/propagationOrganizations"); + } + + public static String communityProjects(String id, String page, String size) throws IOException{ + return get(baseUrl + "community/" + id +"/projects/" + page + "/" + size); + } + + @NotNull + private static String getBody(HttpURLConnection conn) throws IOException { + String body = "{}"; + try (BufferedReader br = new BufferedReader( + new InputStreamReader(conn.getInputStream(), "utf-8"))) { + StringBuilder response = new StringBuilder(); + String responseLine = null; + while ((responseLine = br.readLine()) != null) { + response.append(responseLine.trim()); + } + + body = response.toString(); + + } + return body; + } + + +} diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java new file mode 100644 index 000000000..5b3004a5d --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java @@ -0,0 +1,136 @@ +package eu.dnetlib.dhp.api; + +import com.amazonaws.util.StringUtils; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Maps; +import eu.dnetlib.dhp.api.model.*; +import eu.dnetlib.dhp.bulktag.community.Community; +import eu.dnetlib.dhp.bulktag.community.CommunityConfiguration; +import eu.dnetlib.dhp.bulktag.community.Provider; +import eu.dnetlib.dhp.bulktag.criteria.VerbResolver; +import eu.dnetlib.dhp.bulktag.criteria.VerbResolverFactory; + +import javax.management.Query; +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * @author miriam.baglioni + * @Date 09/10/23 + */ +public class Utils implements Serializable { + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final VerbResolver resolver = VerbResolverFactory.newInstance(); + + public static CommunityConfiguration getCommunityConfiguration() throws IOException { + final Map communities = Maps.newHashMap(); + List validCommunities = new ArrayList<>(); + getValidCommunities() + .forEach(community -> { + try { + CommunityModel cm = MAPPER.readValue(QueryCommunityAPI.community(community.getId()), CommunityModel.class); + validCommunities.add(getCommunity(cm)); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + validCommunities.forEach(community ->{ + try { + DatasourceList dl = MAPPER.readValue(QueryCommunityAPI.communityDatasource(community.getId()), DatasourceList.class); + community.setProviders(dl.stream().map(d -> { +// if(d.getEnabled() == null || Boolean.FALSE.equals(d.getEnabled())) +// return null; + Provider p = new Provider(); + p.setOpenaireId("10|" + d.getOpenaireId()); + p.setSelectionConstraints(d.getSelectioncriteria()); + if(p.getSelectionConstraints() != null) + p.getSelectionConstraints().setSelection(resolver); + return p; + }) + .filter(Objects::nonNull) + .collect(Collectors.toList())); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + validCommunities.forEach(community ->{ + if(community.isValid()) + communities.put(community.getId(), community); + }); + return new CommunityConfiguration(communities); + } + + private static Community getCommunity(CommunityModel cm){ + Community c = new Community(); + c.setId(cm.getId()); + c.setZenodoCommunities(cm.getOtherZenodoCommunities()); + if(!StringUtils.isNullOrEmpty(cm.getZenodoCommunity())) + c.getZenodoCommunities().add(cm.getZenodoCommunity()); + c.setSubjects(cm.getSubjects()); + c.getSubjects().addAll(cm.getFos()); + c.getSubjects().addAll(cm.getSdg()); + c.setConstraints(cm.getAdvancedConstraints()); + if(c.getConstraints()!=null) + c.getConstraints().setSelection(resolver); + c.setRemoveConstraints(cm.getRemoveConstraints()); + if(c.getRemoveConstraints()!=null) + c.getRemoveConstraints().setSelection(resolver); + return c; + } + + public static List getValidCommunities() throws IOException { + return MAPPER.readValue(QueryCommunityAPI.communities(), CommunitySummary.class) + .stream() + .filter(community -> !community.getStatus().equals("hidden") && + (community.getType().equals("ri") || community.getType().equals("community"))) + .collect(Collectors.toList()); + } + public static CommunityEntityMap getCommunityOrganization() throws IOException { + CommunityEntityMap organizationMap = new CommunityEntityMap(); + getValidCommunities() + .forEach(community -> { + String id = community.getId(); + try { + List associatedOrgs = MAPPER.readValue(QueryCommunityAPI.communityPropagationOrganization(id), OrganizationList.class); + if(associatedOrgs.size() >0){ + organizationMap.put(id, associatedOrgs); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + return organizationMap; + } + + public static CommunityEntityMap getCommunityProjects()throws IOException{ + CommunityEntityMap projectMap = new CommunityEntityMap(); + getValidCommunities() + .forEach(community ->{ + int page = -1; + int size = 100; + ContentModel cm = new ContentModel(); + List projectList = new ArrayList<>(); + do{ + page ++; + try { + cm = MAPPER.readValue( QueryCommunityAPI.communityProjects(community.getId(), String.valueOf(page), String.valueOf(size)), ContentModel.class); + if (cm.getContent().size() > 0){ + + cm.getContent().forEach(p -> + projectList.add ("40|" + p.getOpenaireId())); + projectMap.put(community.getId(), projectList); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + }while (!cm.getLast()); + }); + return projectMap; + } +} diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityContentprovider.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityContentprovider.java new file mode 100644 index 000000000..5378e556a --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityContentprovider.java @@ -0,0 +1,43 @@ +package eu.dnetlib.dhp.api.model; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.google.gson.Gson; +import eu.dnetlib.dhp.bulktag.community.SelectionConstraints; + + +@JsonAutoDetect +@JsonIgnoreProperties(ignoreUnknown = true) +public class CommunityContentprovider { + private String openaireId; + private SelectionConstraints selectioncriteria; + + private String enabled; + + public String getEnabled() { + return enabled; + } + + public void setEnabled(String enabled) { + this.enabled = enabled; + } + + public String getOpenaireId() { + return openaireId; + } + + public void setOpenaireId(final String openaireId) { + this.openaireId = openaireId; + } + + + public SelectionConstraints getSelectioncriteria() { + + return this.selectioncriteria; + } + + public void setSelectioncriteria(SelectionConstraints selectioncriteria) { + this.selectioncriteria = selectioncriteria; + + } +} diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/OrganizationMap.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityEntityMap.java similarity index 59% rename from dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/OrganizationMap.java rename to dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityEntityMap.java index 7d786058a..ca3eb2857 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/OrganizationMap.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityEntityMap.java @@ -1,13 +1,13 @@ -package eu.dnetlib.dhp.resulttocommunityfromorganization; +package eu.dnetlib.dhp.api.model; import java.util.ArrayList; import java.util.HashMap; import java.util.List; -public class OrganizationMap extends HashMap> { +public class CommunityEntityMap extends HashMap> { - public OrganizationMap() { + public CommunityEntityMap() { super(); } diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityModel.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityModel.java new file mode 100644 index 000000000..c5b3fac7c --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityModel.java @@ -0,0 +1,108 @@ + +package eu.dnetlib.dhp.api.model; + +import java.io.Serializable; +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import eu.dnetlib.dhp.bulktag.community.SelectionConstraints; + + +/** + * @author miriam.baglioni + * @Date 06/10/23 + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class CommunityModel implements Serializable { + private String id; + private String type; + private String status; + + private String zenodoCommunity; + private List subjects; + private List otherZenodoCommunities; + private List fos; + private List sdg; + private SelectionConstraints advancedConstraints; + private SelectionConstraints removeConstraints; + + public String getZenodoCommunity() { + return zenodoCommunity; + } + + public void setZenodoCommunity(String zenodoCommunity) { + this.zenodoCommunity = zenodoCommunity; + } + + public List getSubjects() { + return subjects; + } + + public void setSubjects(List subjects) { + this.subjects = subjects; + } + + public List getOtherZenodoCommunities() { + return otherZenodoCommunities; + } + + public void setOtherZenodoCommunities(List otherZenodoCommunities) { + this.otherZenodoCommunities = otherZenodoCommunities; + } + + public List getFos() { + return fos; + } + + public void setFos(List fos) { + this.fos = fos; + } + + public List getSdg() { + return sdg; + } + + public void setSdg(List sdg) { + this.sdg = sdg; + } + + public SelectionConstraints getRemoveConstraints() { + return removeConstraints; + } + + public void setRemoveConstraints(SelectionConstraints removeConstraints) { + this.removeConstraints = removeConstraints; + } + + public SelectionConstraints getAdvancedConstraints() { + return advancedConstraints; + } + + public void setAdvancedConstraints(SelectionConstraints advancedConstraints) { + this.advancedConstraints = advancedConstraints; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } +} diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunitySummary.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunitySummary.java new file mode 100644 index 000000000..774b6c874 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunitySummary.java @@ -0,0 +1,16 @@ +package eu.dnetlib.dhp.api.model; + +import java.io.Serializable; +import java.util.ArrayList; + +/** + * @author miriam.baglioni + * @Date 06/10/23 + */ +public class CommunitySummary extends ArrayList implements Serializable { + public CommunitySummary() { + super(); + } + } + + diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/ContentModel.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/ContentModel.java new file mode 100644 index 000000000..a755a9455 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/ContentModel.java @@ -0,0 +1,50 @@ +package eu.dnetlib.dhp.api.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import java.io.Serializable; +import java.util.List; + +/** + * @author miriam.baglioni + * @Date 09/10/23 + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class ContentModel implements Serializable { + private List content; + private Integer totalPages; + private Boolean last; + private Integer number; + + public List getContent() { + return content; + } + + public void setContent(List content) { + this.content = content; + } + + public Integer getTotalPages() { + return totalPages; + } + + public void setTotalPages(Integer totalPages) { + this.totalPages = totalPages; + } + + public Boolean getLast() { + return last; + } + + public void setLast(Boolean last) { + this.last = last; + } + + public Integer getNumber() { + return number; + } + + public void setNumber(Integer number) { + this.number = number; + } +} diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/DatasourceList.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/DatasourceList.java new file mode 100644 index 000000000..8e31d7612 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/DatasourceList.java @@ -0,0 +1,12 @@ +package eu.dnetlib.dhp.api.model; + +import eu.dnetlib.dhp.api.model.CommunityContentprovider; + + +import java.io.Serializable; +import java.util.ArrayList; +public class DatasourceList extends ArrayList implements Serializable { + public DatasourceList(){ + super(); + } +} \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/OrganizationList.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/OrganizationList.java new file mode 100644 index 000000000..6895967ff --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/OrganizationList.java @@ -0,0 +1,15 @@ +package eu.dnetlib.dhp.api.model; + +import java.io.Serializable; +import java.util.ArrayList; + +/** + * @author miriam.baglioni + * @Date 09/10/23 + */ +public class OrganizationList extends ArrayList implements Serializable { + + public OrganizationList(){ + super(); + } +} diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/ProjectModel.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/ProjectModel.java new file mode 100644 index 000000000..50bdcf649 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/ProjectModel.java @@ -0,0 +1,23 @@ +package eu.dnetlib.dhp.api.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import java.io.Serializable; + +/** + * @author miriam.baglioni + * @Date 09/10/23 + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class ProjectModel implements Serializable { + + private String openaireId; + + public String getOpenaireId() { + return openaireId; + } + + public void setOpenaireId(String openaireId) { + this.openaireId = openaireId; + } +} diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java index 3186ed5c0..b61bc2c9f 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java @@ -6,10 +6,10 @@ import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; import java.util.*; +import eu.dnetlib.dhp.api.Utils; import org.apache.commons.io.IOUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.function.FilterFunction; -import org.apache.spark.api.java.function.ForeachFunction; import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoders; @@ -87,7 +87,7 @@ public class SparkBulkTagJob { if (isTest) { cc = CommunityConfigurationFactory.newInstance(taggingConf); } else { - cc = QueryInformationSystem.getCommunityConfiguration(parser.get("isLookUpUrl")); + cc = Utils.getCommunityConfiguration();//QueryInformationSystem.getCommunityConfiguration(parser.get("isLookUpUrl")); } runWithSparkSession( diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Community.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Community.java index b44376e22..d281f8783 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Community.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Community.java @@ -4,16 +4,18 @@ package eu.dnetlib.dhp.bulktag.community; import java.io.Serializable; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import com.google.gson.Gson; + /** Created by miriam on 01/08/2018. */ public class Community implements Serializable { private String id; private List subjects = new ArrayList<>(); private List providers = new ArrayList<>(); - private List zenodoCommunities = new ArrayList<>(); + private List zenodoCommunities = new ArrayList<>(); private SelectionConstraints constraints = new SelectionConstraints(); private SelectionConstraints removeConstraints = new SelectionConstraints(); @@ -26,7 +28,7 @@ public class Community implements Serializable { return !getSubjects().isEmpty() || !getProviders().isEmpty() || !getZenodoCommunities().isEmpty() - || getConstraints().getCriteria() != null; + || (Optional.ofNullable(getConstraints()).isPresent() && getConstraints().getCriteria() != null); } public String getId() { @@ -53,11 +55,11 @@ public class Community implements Serializable { this.providers = providers; } - public List getZenodoCommunities() { + public List getZenodoCommunities() { return zenodoCommunities; } - public void setZenodoCommunities(List zenodoCommunities) { + public void setZenodoCommunities(List zenodoCommunities) { this.zenodoCommunities = zenodoCommunities; } diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfiguration.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfiguration.java index e061ccd5e..a658c7ff5 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfiguration.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfiguration.java @@ -81,7 +81,7 @@ public class CommunityConfiguration implements Serializable { this.removeConstraintsMap = removeConstraintsMap; } - CommunityConfiguration(final Map communities) { + public CommunityConfiguration(final Map communities) { this.communities = communities; init(); } @@ -117,10 +117,10 @@ public class CommunityConfiguration implements Serializable { add(d.getOpenaireId(), new Pair<>(id, d.getSelectionConstraints()), datasourceMap); } // get zenodo communities - for (ZenodoCommunity zc : c.getZenodoCommunities()) { + for (String zc : c.getZenodoCommunities()) { add( - zc.getZenodoCommunityId(), - new Pair<>(id, zc.getSelCriteria()), + zc, + new Pair<>(id, null), zenodocommunityMap); } selectionConstraintsMap.put(id, c.getConstraints()); diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfigurationFactory.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfigurationFactory.java index 7b9e03ef6..013bf7168 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfigurationFactory.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfigurationFactory.java @@ -5,7 +5,6 @@ import java.io.StringReader; import java.util.ArrayList; import java.util.List; import java.util.Map; - import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -143,16 +142,16 @@ public class CommunityConfigurationFactory { return providerList; } - private static List parseZenodoCommunities(final Node node) { + private static List parseZenodoCommunities(final Node node) { final List list = node.selectNodes("./zenodocommunities/zenodocommunity"); - final List zenodoCommunityList = new ArrayList<>(); + final List zenodoCommunityList = new ArrayList<>(); for (Node n : list) { - ZenodoCommunity zc = new ZenodoCommunity(); - zc.setZenodoCommunityId(n.selectSingleNode("./zenodoid").getText()); - zc.setSelCriteria(n.selectSingleNode("./selcriteria")); +// ZenodoCommunity zc = new ZenodoCommunity(); +// zc.setZenodoCommunityId(n.selectSingleNode("./zenodoid").getText()); +// zc.setSelCriteria(n.selectSingleNode("./selcriteria")); - zenodoCommunityList.add(zc); + zenodoCommunityList.add(n.selectSingleNode("./zenodoid").getText()); } log.info("size of the zenodo community list " + zenodoCommunityList.size()); diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Constraint.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Constraint.java index ed58cc14d..03ae1117b 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Constraint.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Constraint.java @@ -6,12 +6,14 @@ import java.lang.reflect.InvocationTargetException; import eu.dnetlib.dhp.bulktag.criteria.Selection; import eu.dnetlib.dhp.bulktag.criteria.VerbResolver; +import org.apache.htrace.fasterxml.jackson.annotation.JsonIgnore; public class Constraint implements Serializable { private String verb; private String field; private String value; // private String element; + @JsonIgnore private Selection selection; public String getVerb() { @@ -37,11 +39,11 @@ public class Constraint implements Serializable { public void setValue(String value) { this.value = value; } - - public void setSelection(Selection sel) { - selection = sel; - } - +//@JsonIgnore + //public void setSelection(Selection sel) { +// selection = sel; +// } +@JsonIgnore public void setSelection(VerbResolver resolver) throws InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { @@ -52,11 +54,5 @@ public class Constraint implements Serializable { return selection.apply(metadata); } -// public String getElement() { -// return element; -// } -// -// public void setElement(String element) { -// this.element = element; -// } + } diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/SelectionConstraints.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/SelectionConstraints.java index c7dcce812..8e6d25e9b 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/SelectionConstraints.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/SelectionConstraints.java @@ -7,11 +7,12 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import eu.dnetlib.dhp.bulktag.criteria.VerbResolver; - +@JsonAutoDetect public class SelectionConstraints implements Serializable { private List criteria; diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunitySet.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunitySet.java index 0fc8cb390..ff6b73286 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunitySet.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunitySet.java @@ -6,12 +6,12 @@ import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession; import java.util.*; +import eu.dnetlib.dhp.api.Utils; +import eu.dnetlib.dhp.api.model.CommunityEntityMap; import org.apache.commons.io.IOUtils; import org.apache.hadoop.io.compress.GzipCodec; import org.apache.spark.SparkConf; -import org.apache.spark.api.java.function.FilterFunction; import org.apache.spark.api.java.function.MapFunction; -import org.apache.spark.api.java.function.MapGroupsFunction; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoders; import org.apache.spark.sql.SparkSession; @@ -48,10 +48,11 @@ public class PrepareResultCommunitySet { final String outputPath = parser.get("outputPath"); log.info("outputPath: {}", outputPath); - final OrganizationMap organizationMap = new Gson() - .fromJson( - parser.get("organizationtoresultcommunitymap"), - OrganizationMap.class); +// final CommunityEntityMap organizationMap = new Gson() +// .fromJson( +// parser.get("organizationtoresultcommunitymap"), +// CommunityEntityMap.class); + final CommunityEntityMap organizationMap = Utils.getCommunityOrganization(); log.info("organizationMap: {}", new Gson().toJson(organizationMap)); SparkConf conf = new SparkConf(); @@ -70,7 +71,7 @@ public class PrepareResultCommunitySet { SparkSession spark, String inputPath, String outputPath, - OrganizationMap organizationMap) { + CommunityEntityMap organizationMap) { Dataset relation = readPath(spark, inputPath, Relation.class); relation.createOrReplaceTempView("relation"); @@ -115,7 +116,7 @@ public class PrepareResultCommunitySet { } private static MapFunction mapResultCommunityFn( - OrganizationMap organizationMap) { + CommunityEntityMap organizationMap) { return value -> { String rId = value.getResultId(); Optional> orgs = Optional.ofNullable(value.getMerges()); diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml index 6a329fdc4..d9805ab7b 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml @@ -4,10 +4,10 @@ sourcePath the source path - - organizationtoresultcommunitymap - organization community map - + + + + outputPath the output path diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java index 660a55472..745cd7e6f 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java @@ -1568,4 +1568,42 @@ public class BulkTagJobTest { } + @Test + void newConfTest() throws Exception { + final String pathMap = BulkTagJobTest.pathMap; + SparkBulkTagJob + .main( + new String[] { + "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), + "-sourcePath", + getClass().getResource("/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates").getPath(), + "-taggingConf", taggingConf, + "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Dataset", + "-outputPath", workingDir.toString() + "/dataset", + "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + "-pathMap", pathMap + }); + + final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .textFile(workingDir.toString() + "/dataset") + .map(item -> OBJECT_MAPPER.readValue(item, Dataset.class)); + + Assertions.assertEquals(10, tmp.count()); + org.apache.spark.sql.Dataset verificationDataset = spark + .createDataset(tmp.rdd(), Encoders.bean(Dataset.class)); + + verificationDataset.createOrReplaceTempView("dataset"); + + String query = "select id, MyT.id community " + + "from dataset " + + "lateral view explode(context) c as MyT " + + "lateral view explode(MyT.datainfo) d as MyD " + + "where MyD.inferenceprovenance = 'bulktagging'"; + + Assertions.assertEquals(0, spark.sql(query).count()); + } + } diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/CommunityConfigurationFactoryTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/CommunityConfigurationFactoryTest.java index c8fd62c8e..7e12bf9c3 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/CommunityConfigurationFactoryTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/CommunityConfigurationFactoryTest.java @@ -47,7 +47,7 @@ class CommunityConfigurationFactoryTest { sc.setVerb("not_contains"); sc.setField("contributor"); sc.setValue("DARIAH"); - sc.setSelection(resolver.getSelectionCriteria(sc.getVerb(), sc.getValue())); + sc.setSelection(resolver);//.getSelectionCriteria(sc.getVerb(), sc.getValue())); String metadata = "This work has been partially supported by DARIAH-EU infrastructure"; Assertions.assertFalse(sc.verifyCriteria(metadata)); } diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/QueryCommunityAPITest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/QueryCommunityAPITest.java new file mode 100644 index 000000000..1b3403535 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/QueryCommunityAPITest.java @@ -0,0 +1,91 @@ + +package eu.dnetlib.dhp.bulktag; + +import eu.dnetlib.dhp.api.Utils; +import eu.dnetlib.dhp.api.model.CommunityEntityMap; +import eu.dnetlib.dhp.bulktag.community.Community; +import eu.dnetlib.dhp.bulktag.community.CommunityConfiguration; + +import eu.dnetlib.dhp.api.model.CommunityModel; +import eu.dnetlib.dhp.api.model.CommunitySummary; +import eu.dnetlib.dhp.api.model.DatasourceList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.api.QueryCommunityAPI; + +import java.util.List; + +/** + * @author miriam.baglioni + * @Date 06/10/23 + */ +public class QueryCommunityAPITest { + + @Test + void communityList() throws Exception { + String body = QueryCommunityAPI.communities(); + new ObjectMapper() + .readValue(body, CommunitySummary.class) + .forEach(p -> { + try { + System.out.println(new ObjectMapper().writeValueAsString(p)); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }); + } + + @Test + void community() throws Exception { + String id = "dh-ch"; + String body = QueryCommunityAPI.community(id); + System.out.println(new ObjectMapper().writeValueAsString(new ObjectMapper() + .readValue(body, CommunityModel.class))) + ; + } + + @Test + void communityDatasource() throws Exception { + String id = "dh-ch"; + String body = QueryCommunityAPI.communityDatasource(id); + new ObjectMapper() + .readValue(body, DatasourceList.class) + .forEach(ds-> { + try { + System.out.println(new ObjectMapper().writeValueAsString(ds)); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }); + ; + } + + @Test + void validCommunities() throws Exception { + CommunityConfiguration cc = Utils.getCommunityConfiguration(); + System.out.println(cc.getCommunities().keySet()); + Community community =cc.getCommunities().get("aurora"); + Assertions.assertEquals(0, community.getSubjects().size()); + Assertions.assertEquals(null, community.getConstraints()); + Assertions.assertEquals(null, community.getRemoveConstraints()); + Assertions.assertEquals(2, community.getZenodoCommunities().size()); + Assertions.assertTrue(community.getZenodoCommunities().stream().anyMatch(c -> c.equals("aurora-universities-network"))); + Assertions.assertTrue(community.getZenodoCommunities().stream().anyMatch(c -> c.equals("university-of-innsbruck"))); + Assertions.assertEquals(35, community.getProviders().size()); + Assertions.assertEquals(35, community.getProviders().stream().filter(p->p.getSelectionConstraints()==null).count()); + } + + @Test + void getCommunityProjects() throws Exception { + CommunityEntityMap projectMap = Utils.getCommunityProjects(); + Assertions.assertFalse(projectMap.containsKey("mes")); + Assertions.assertEquals(33, projectMap.size()); + Assertions.assertTrue(projectMap.keySet().stream().allMatch(k -> projectMap.get(k).stream().allMatch(p -> p.startsWith("40|")))); + } + +} + diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/group/GroupEntitiesSparkJobTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/group/GroupEntitiesSparkJobTest.java index b878e778e..0887adf45 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/group/GroupEntitiesSparkJobTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/group/GroupEntitiesSparkJobTest.java @@ -1,14 +1,14 @@ package eu.dnetlib.dhp.oa.graph.group; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import eu.dnetlib.dhp.common.HdfsSupport; -import eu.dnetlib.dhp.oa.merge.GroupEntitiesSparkJob; -import eu.dnetlib.dhp.schema.common.ModelSupport; -import eu.dnetlib.dhp.schema.oaf.OafEntity; -import eu.dnetlib.dhp.schema.oaf.Result; -import eu.dnetlib.dhp.utils.DHPUtils; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + import org.apache.commons.io.FileUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.function.FilterFunction; @@ -18,108 +18,108 @@ import org.apache.spark.sql.Encoders; import org.apache.spark.sql.SparkSession; import org.junit.jupiter.api.*; -import java.io.IOException; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; -import static org.junit.jupiter.api.Assertions.assertEquals; +import eu.dnetlib.dhp.common.HdfsSupport; +import eu.dnetlib.dhp.oa.merge.GroupEntitiesSparkJob; +import eu.dnetlib.dhp.schema.common.ModelSupport; +import eu.dnetlib.dhp.schema.oaf.OafEntity; +import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.dhp.utils.DHPUtils; @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class GroupEntitiesSparkJobTest { - private static SparkSession spark; + private static SparkSession spark; - private static ObjectMapper mapper = new ObjectMapper() - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + private static ObjectMapper mapper = new ObjectMapper() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - private static Path workingDir; - private Path dataInputPath; + private static Path workingDir; + private Path dataInputPath; - private Path checkpointPath; + private Path checkpointPath; - private Path outputPath; + private Path outputPath; - @BeforeAll - public static void beforeAll() throws IOException { - workingDir = Files.createTempDirectory(GroupEntitiesSparkJob.class.getSimpleName()); + @BeforeAll + public static void beforeAll() throws IOException { + workingDir = Files.createTempDirectory(GroupEntitiesSparkJob.class.getSimpleName()); - SparkConf conf = new SparkConf(); - conf.setAppName(GroupEntitiesSparkJob.class.getSimpleName()); - conf.setMaster("local"); - conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); - conf.registerKryoClasses(ModelSupport.getOafModelClasses()); - spark = SparkSession.builder().config(conf).getOrCreate(); - } + SparkConf conf = new SparkConf(); + conf.setAppName(GroupEntitiesSparkJob.class.getSimpleName()); + conf.setMaster("local"); + conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); + conf.registerKryoClasses(ModelSupport.getOafModelClasses()); + spark = SparkSession.builder().config(conf).getOrCreate(); + } - @BeforeEach - public void beforeEach() throws IOException, URISyntaxException { - dataInputPath = Paths.get(ClassLoader.getSystemResource("eu/dnetlib/dhp/oa/graph/group").toURI()); - checkpointPath = workingDir.resolve("grouped_entity"); - outputPath = workingDir.resolve("dispatched_entity"); - } + @BeforeEach + public void beforeEach() throws IOException, URISyntaxException { + dataInputPath = Paths.get(ClassLoader.getSystemResource("eu/dnetlib/dhp/oa/graph/group").toURI()); + checkpointPath = workingDir.resolve("grouped_entity"); + outputPath = workingDir.resolve("dispatched_entity"); + } - @AfterAll - public static void afterAll() throws IOException { - spark.stop(); - FileUtils.deleteDirectory(workingDir.toFile()); - } + @AfterAll + public static void afterAll() throws IOException { + spark.stop(); + FileUtils.deleteDirectory(workingDir.toFile()); + } - @Test - @Order(1) - void testGroupEntities() throws Exception { - GroupEntitiesSparkJob.main(new String[]{ - "-isSparkSessionManaged", - Boolean.FALSE.toString(), - "-graphInputPath", - dataInputPath.toString(), - "-checkpointPath", - checkpointPath.toString(), - "-outputPath", - outputPath.toString(), - "-filterInvisible", - Boolean.FALSE.toString() - }); + @Test + @Order(1) + void testGroupEntities() throws Exception { + GroupEntitiesSparkJob.main(new String[] { + "-isSparkSessionManaged", + Boolean.FALSE.toString(), + "-graphInputPath", + dataInputPath.toString(), + "-checkpointPath", + checkpointPath.toString(), + "-outputPath", + outputPath.toString(), + "-filterInvisible", + Boolean.FALSE.toString() + }); - Dataset checkpointTable = spark - .read() - .load(checkpointPath.toString()) - .selectExpr("COALESCE(*)") - .as(Encoders.kryo(OafEntity.class)); + Dataset checkpointTable = spark + .read() + .load(checkpointPath.toString()) + .selectExpr("COALESCE(*)") + .as(Encoders.kryo(OafEntity.class)); + assertEquals( + 1, + checkpointTable + .filter( + (FilterFunction) r -> "50|doi_________::09821844208a5cd6300b2bfb13bca1b9" + .equals(r.getId()) && + r.getCollectedfrom().stream().anyMatch(kv -> kv.getValue().equalsIgnoreCase("zenodo"))) + .count()); - assertEquals( - 1, - checkpointTable - .filter( - (FilterFunction) r -> "50|doi_________::09821844208a5cd6300b2bfb13bca1b9" - .equals(r.getId()) && - r.getCollectedfrom().stream().anyMatch(kv -> kv.getValue().equalsIgnoreCase("zenodo"))) - .count()); + Dataset output = spark + .read() + .textFile( + DHPUtils + .toSeq( + HdfsSupport + .listFiles(outputPath.toString(), spark.sparkContext().hadoopConfiguration()))) + .map((MapFunction) s -> mapper.readValue(s, Result.class), Encoders.bean(Result.class)); - - Dataset output = spark - .read() - .textFile( - DHPUtils - .toSeq( - HdfsSupport - .listFiles(outputPath.toString(), spark.sparkContext().hadoopConfiguration()))) - .map((MapFunction) s -> mapper.readValue(s, Result.class), Encoders.bean(Result.class)); - - assertEquals(3, output.count()); - assertEquals( - 2, - output - .map((MapFunction) r -> r.getResulttype().getClassid(), Encoders.STRING()) - .filter((FilterFunction) s -> s.equals("publication")) - .count()); - assertEquals( - 1, - output - .map((MapFunction) r -> r.getResulttype().getClassid(), Encoders.STRING()) - .filter((FilterFunction) s -> s.equals("dataset")) - .count()); - } -} \ No newline at end of file + assertEquals(3, output.count()); + assertEquals( + 2, + output + .map((MapFunction) r -> r.getResulttype().getClassid(), Encoders.STRING()) + .filter((FilterFunction) s -> s.equals("publication")) + .count()); + assertEquals( + 1, + output + .map((MapFunction) r -> r.getResulttype().getClassid(), Encoders.STRING()) + .filter((FilterFunction) s -> s.equals("dataset")) + .count()); + } +} From a3d01ccb246e52e7977aef8c57b15d964dcc9834 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Mon, 9 Oct 2023 14:52:17 +0200 Subject: [PATCH 049/148] refactoring --- .../CreateActionSetSparkJob.java | 2 +- .../doiboost/crossref/Crossref2Oaf.scala | 16 +- .../crossref/CrossrefMappingTest.scala | 2 +- .../eu/dnetlib/dhp/api/QueryCommunityAPI.java | 9 +- .../main/java/eu/dnetlib/dhp/api/Utils.java | 236 +++++++++--------- .../api/model/CommunityContentprovider.java | 6 +- .../dnetlib/dhp/api/model/CommunityModel.java | 2 +- .../dhp/api/model/CommunitySummary.java | 11 +- .../dnetlib/dhp/api/model/ContentModel.java | 61 ++--- .../dnetlib/dhp/api/model/DatasourceList.java | 11 +- .../dhp/api/model/OrganizationList.java | 7 +- .../dnetlib/dhp/api/model/ProjectModel.java | 19 +- .../dnetlib/dhp/bulktag/SparkBulkTagJob.java | 4 +- .../dhp/bulktag/community/Community.java | 1 - .../CommunityConfigurationFactory.java | 1 + .../dhp/bulktag/community/Constraint.java | 9 +- .../community/SelectionConstraints.java | 1 + .../PrepareResultCommunitySet.java | 4 +- .../dnetlib/dhp/bulktag/BulkTagJobTest.java | 38 +-- .../CommunityConfigurationFactoryTest.java | 2 +- .../dhp/bulktag/QueryCommunityAPITest.java | 65 +++-- 21 files changed, 268 insertions(+), 239 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java index a367ba852..b707fdcd3 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java @@ -7,7 +7,6 @@ import java.io.IOException; import java.io.Serializable; import java.util.*; -import eu.dnetlib.dhp.schema.oaf.utils.*; import org.apache.commons.cli.ParseException; import org.apache.commons.io.IOUtils; import org.apache.hadoop.io.Text; @@ -30,6 +29,7 @@ import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.schema.action.AtomicAction; import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.oaf.*; +import eu.dnetlib.dhp.schema.oaf.utils.*; import eu.dnetlib.dhp.utils.DHPUtils; import scala.Tuple2; diff --git a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala index e0fdb9ce4..565d34e62 100644 --- a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala +++ b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala @@ -31,9 +31,7 @@ case class mappingAuthor( affiliation: Option[mappingAffiliation] ) {} -case class funderInfo(id:String,uri:String, name:String,synonym:List[String] ) {} - - +case class funderInfo(id: String, uri: String, name: String, synonym: List[String]) {} case class mappingFunder(name: String, DOI: Option[String], award: Option[List[String]]) {} @@ -41,7 +39,9 @@ case object Crossref2Oaf { val logger: Logger = LoggerFactory.getLogger(Crossref2Oaf.getClass) val irishFunder: List[funderInfo] = { - val s = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/doiboost/crossref/irish_funder.json")).mkString + val s = Source + .fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/doiboost/crossref/irish_funder.json")) + .mkString implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats lazy val json: org.json4s.JValue = parse(s) json.extract[List[funderInfo]] @@ -100,9 +100,11 @@ case object Crossref2Oaf { "report" -> "0017 Report" ) - def getIrishId(doi:String):Option[String] = { - val id =doi.split("/").last - irishFunder.find(f => id.equalsIgnoreCase(f.id) || (f.synonym.nonEmpty && f.synonym.exists(s => s.equalsIgnoreCase(id)))).map(f => f.id) + def getIrishId(doi: String): Option[String] = { + val id = doi.split("/").last + irishFunder + .find(f => id.equalsIgnoreCase(f.id) || (f.synonym.nonEmpty && f.synonym.exists(s => s.equalsIgnoreCase(id)))) + .map(f => f.id) } def mappingResult(result: Result, json: JValue, cobjCategory: String): Result = { diff --git a/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala b/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala index 7961376c5..fbf6f72c0 100644 --- a/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala +++ b/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala @@ -50,7 +50,7 @@ class CrossrefMappingTest { } } - def checkRelation(generatedOAF: List[Oaf]): Unit = { + def checkRelation(generatedOAF: List[Oaf]): Unit = { val rels: List[Relation] = generatedOAF.filter(p => p.isInstanceOf[Relation]).asInstanceOf[List[Relation]] diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java index 635ee2027..cc615ba46 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java @@ -21,7 +21,7 @@ import com.google.gson.Gson; public class QueryCommunityAPI { private static final String baseUrl = "https://services.openaire.eu/openaire/"; - private static String get(String geturl) throws IOException{ + private static String get(String geturl) throws IOException { URL url = new URL(geturl); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setDoOutput(true); @@ -44,7 +44,7 @@ public class QueryCommunityAPI { return get(baseUrl + "community/" + id); } - public static String communityDatasource(String id)throws IOException{ + public static String communityDatasource(String id) throws IOException { return get(baseUrl + "community/" + id + "/contentproviders"); } @@ -53,8 +53,8 @@ public class QueryCommunityAPI { return get(baseUrl + "community/" + id + "/propagationOrganizations"); } - public static String communityProjects(String id, String page, String size) throws IOException{ - return get(baseUrl + "community/" + id +"/projects/" + page + "/" + size); + public static String communityProjects(String id, String page, String size) throws IOException { + return get(baseUrl + "community/" + id + "/projects/" + page + "/" + size); } @NotNull @@ -74,5 +74,4 @@ public class QueryCommunityAPI { return body; } - } diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java index 5b3004a5d..c1aaa14c4 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java @@ -1,16 +1,6 @@ + package eu.dnetlib.dhp.api; -import com.amazonaws.util.StringUtils; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Maps; -import eu.dnetlib.dhp.api.model.*; -import eu.dnetlib.dhp.bulktag.community.Community; -import eu.dnetlib.dhp.bulktag.community.CommunityConfiguration; -import eu.dnetlib.dhp.bulktag.community.Provider; -import eu.dnetlib.dhp.bulktag.criteria.VerbResolver; -import eu.dnetlib.dhp.bulktag.criteria.VerbResolverFactory; - -import javax.management.Query; import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; @@ -19,118 +9,140 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import javax.management.Query; + +import com.amazonaws.util.StringUtils; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Maps; + +import eu.dnetlib.dhp.api.model.*; +import eu.dnetlib.dhp.bulktag.community.Community; +import eu.dnetlib.dhp.bulktag.community.CommunityConfiguration; +import eu.dnetlib.dhp.bulktag.community.Provider; +import eu.dnetlib.dhp.bulktag.criteria.VerbResolver; +import eu.dnetlib.dhp.bulktag.criteria.VerbResolverFactory; + /** * @author miriam.baglioni * @Date 09/10/23 */ public class Utils implements Serializable { - private static final ObjectMapper MAPPER = new ObjectMapper(); - private static final VerbResolver resolver = VerbResolverFactory.newInstance(); + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final VerbResolver resolver = VerbResolverFactory.newInstance(); - public static CommunityConfiguration getCommunityConfiguration() throws IOException { - final Map communities = Maps.newHashMap(); - List validCommunities = new ArrayList<>(); - getValidCommunities() - .forEach(community -> { - try { - CommunityModel cm = MAPPER.readValue(QueryCommunityAPI.community(community.getId()), CommunityModel.class); - validCommunities.add(getCommunity(cm)); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - validCommunities.forEach(community ->{ - try { - DatasourceList dl = MAPPER.readValue(QueryCommunityAPI.communityDatasource(community.getId()), DatasourceList.class); - community.setProviders(dl.stream().map(d -> { + public static CommunityConfiguration getCommunityConfiguration() throws IOException { + final Map communities = Maps.newHashMap(); + List validCommunities = new ArrayList<>(); + getValidCommunities() + .forEach(community -> { + try { + CommunityModel cm = MAPPER + .readValue(QueryCommunityAPI.community(community.getId()), CommunityModel.class); + validCommunities.add(getCommunity(cm)); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + validCommunities.forEach(community -> { + try { + DatasourceList dl = MAPPER + .readValue(QueryCommunityAPI.communityDatasource(community.getId()), DatasourceList.class); + community.setProviders(dl.stream().map(d -> { // if(d.getEnabled() == null || Boolean.FALSE.equals(d.getEnabled())) // return null; - Provider p = new Provider(); - p.setOpenaireId("10|" + d.getOpenaireId()); - p.setSelectionConstraints(d.getSelectioncriteria()); - if(p.getSelectionConstraints() != null) - p.getSelectionConstraints().setSelection(resolver); - return p; - }) - .filter(Objects::nonNull) - .collect(Collectors.toList())); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + Provider p = new Provider(); + p.setOpenaireId("10|" + d.getOpenaireId()); + p.setSelectionConstraints(d.getSelectioncriteria()); + if (p.getSelectionConstraints() != null) + p.getSelectionConstraints().setSelection(resolver); + return p; + }) + .filter(Objects::nonNull) + .collect(Collectors.toList())); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); - validCommunities.forEach(community ->{ - if(community.isValid()) - communities.put(community.getId(), community); - }); - return new CommunityConfiguration(communities); - } + validCommunities.forEach(community -> { + if (community.isValid()) + communities.put(community.getId(), community); + }); + return new CommunityConfiguration(communities); + } - private static Community getCommunity(CommunityModel cm){ - Community c = new Community(); - c.setId(cm.getId()); - c.setZenodoCommunities(cm.getOtherZenodoCommunities()); - if(!StringUtils.isNullOrEmpty(cm.getZenodoCommunity())) - c.getZenodoCommunities().add(cm.getZenodoCommunity()); - c.setSubjects(cm.getSubjects()); - c.getSubjects().addAll(cm.getFos()); - c.getSubjects().addAll(cm.getSdg()); - c.setConstraints(cm.getAdvancedConstraints()); - if(c.getConstraints()!=null) - c.getConstraints().setSelection(resolver); - c.setRemoveConstraints(cm.getRemoveConstraints()); - if(c.getRemoveConstraints()!=null) - c.getRemoveConstraints().setSelection(resolver); - return c; - } + private static Community getCommunity(CommunityModel cm) { + Community c = new Community(); + c.setId(cm.getId()); + c.setZenodoCommunities(cm.getOtherZenodoCommunities()); + if (!StringUtils.isNullOrEmpty(cm.getZenodoCommunity())) + c.getZenodoCommunities().add(cm.getZenodoCommunity()); + c.setSubjects(cm.getSubjects()); + c.getSubjects().addAll(cm.getFos()); + c.getSubjects().addAll(cm.getSdg()); + c.setConstraints(cm.getAdvancedConstraints()); + if (c.getConstraints() != null) + c.getConstraints().setSelection(resolver); + c.setRemoveConstraints(cm.getRemoveConstraints()); + if (c.getRemoveConstraints() != null) + c.getRemoveConstraints().setSelection(resolver); + return c; + } - public static List getValidCommunities() throws IOException { - return MAPPER.readValue(QueryCommunityAPI.communities(), CommunitySummary.class) - .stream() - .filter(community -> !community.getStatus().equals("hidden") && - (community.getType().equals("ri") || community.getType().equals("community"))) - .collect(Collectors.toList()); - } - public static CommunityEntityMap getCommunityOrganization() throws IOException { - CommunityEntityMap organizationMap = new CommunityEntityMap(); - getValidCommunities() - .forEach(community -> { - String id = community.getId(); - try { - List associatedOrgs = MAPPER.readValue(QueryCommunityAPI.communityPropagationOrganization(id), OrganizationList.class); - if(associatedOrgs.size() >0){ - organizationMap.put(id, associatedOrgs); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - return organizationMap; - } + public static List getValidCommunities() throws IOException { + return MAPPER + .readValue(QueryCommunityAPI.communities(), CommunitySummary.class) + .stream() + .filter( + community -> !community.getStatus().equals("hidden") && + (community.getType().equals("ri") || community.getType().equals("community"))) + .collect(Collectors.toList()); + } - public static CommunityEntityMap getCommunityProjects()throws IOException{ - CommunityEntityMap projectMap = new CommunityEntityMap(); - getValidCommunities() - .forEach(community ->{ - int page = -1; - int size = 100; - ContentModel cm = new ContentModel(); - List projectList = new ArrayList<>(); - do{ - page ++; - try { - cm = MAPPER.readValue( QueryCommunityAPI.communityProjects(community.getId(), String.valueOf(page), String.valueOf(size)), ContentModel.class); - if (cm.getContent().size() > 0){ + public static CommunityEntityMap getCommunityOrganization() throws IOException { + CommunityEntityMap organizationMap = new CommunityEntityMap(); + getValidCommunities() + .forEach(community -> { + String id = community.getId(); + try { + List associatedOrgs = MAPPER + .readValue(QueryCommunityAPI.communityPropagationOrganization(id), OrganizationList.class); + if (associatedOrgs.size() > 0) { + organizationMap.put(id, associatedOrgs); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + return organizationMap; + } - cm.getContent().forEach(p -> - projectList.add ("40|" + p.getOpenaireId())); - projectMap.put(community.getId(), projectList); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - }while (!cm.getLast()); - }); - return projectMap; - } + public static CommunityEntityMap getCommunityProjects() throws IOException { + CommunityEntityMap projectMap = new CommunityEntityMap(); + getValidCommunities() + .forEach(community -> { + int page = -1; + int size = 100; + ContentModel cm = new ContentModel(); + List projectList = new ArrayList<>(); + do { + page++; + try { + cm = MAPPER + .readValue( + QueryCommunityAPI + .communityProjects(community.getId(), String.valueOf(page), String.valueOf(size)), + ContentModel.class); + if (cm.getContent().size() > 0) { + + cm.getContent().forEach(p -> projectList.add("40|" + p.getOpenaireId())); + projectMap.put(community.getId(), projectList); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } while (!cm.getLast()); + }); + return projectMap; + } } diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityContentprovider.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityContentprovider.java index 5378e556a..9fab5a80c 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityContentprovider.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityContentprovider.java @@ -1,10 +1,11 @@ + package eu.dnetlib.dhp.api.model; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.google.gson.Gson; -import eu.dnetlib.dhp.bulktag.community.SelectionConstraints; +import eu.dnetlib.dhp.bulktag.community.SelectionConstraints; @JsonAutoDetect @JsonIgnoreProperties(ignoreUnknown = true) @@ -30,10 +31,9 @@ public class CommunityContentprovider { this.openaireId = openaireId; } - public SelectionConstraints getSelectioncriteria() { - return this.selectioncriteria; + return this.selectioncriteria; } public void setSelectioncriteria(SelectionConstraints selectioncriteria) { diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityModel.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityModel.java index c5b3fac7c..745e7efc2 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityModel.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityModel.java @@ -5,8 +5,8 @@ import java.io.Serializable; import java.util.List; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import eu.dnetlib.dhp.bulktag.community.SelectionConstraints; +import eu.dnetlib.dhp.bulktag.community.SelectionConstraints; /** * @author miriam.baglioni diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunitySummary.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunitySummary.java index 774b6c874..a0541f7ee 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunitySummary.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunitySummary.java @@ -1,3 +1,4 @@ + package eu.dnetlib.dhp.api.model; import java.io.Serializable; @@ -8,9 +9,7 @@ import java.util.ArrayList; * @Date 06/10/23 */ public class CommunitySummary extends ArrayList implements Serializable { - public CommunitySummary() { - super(); - } - } - - + public CommunitySummary() { + super(); + } +} diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/ContentModel.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/ContentModel.java index a755a9455..469709f59 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/ContentModel.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/ContentModel.java @@ -1,50 +1,51 @@ -package eu.dnetlib.dhp.api.model; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +package eu.dnetlib.dhp.api.model; import java.io.Serializable; import java.util.List; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + /** * @author miriam.baglioni * @Date 09/10/23 */ @JsonIgnoreProperties(ignoreUnknown = true) public class ContentModel implements Serializable { - private List content; - private Integer totalPages; - private Boolean last; - private Integer number; + private List content; + private Integer totalPages; + private Boolean last; + private Integer number; - public List getContent() { - return content; - } + public List getContent() { + return content; + } - public void setContent(List content) { - this.content = content; - } + public void setContent(List content) { + this.content = content; + } - public Integer getTotalPages() { - return totalPages; - } + public Integer getTotalPages() { + return totalPages; + } - public void setTotalPages(Integer totalPages) { - this.totalPages = totalPages; - } + public void setTotalPages(Integer totalPages) { + this.totalPages = totalPages; + } - public Boolean getLast() { - return last; - } + public Boolean getLast() { + return last; + } - public void setLast(Boolean last) { - this.last = last; - } + public void setLast(Boolean last) { + this.last = last; + } - public Integer getNumber() { - return number; - } + public Integer getNumber() { + return number; + } - public void setNumber(Integer number) { - this.number = number; - } + public void setNumber(Integer number) { + this.number = number; + } } diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/DatasourceList.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/DatasourceList.java index 8e31d7612..30d0241c3 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/DatasourceList.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/DatasourceList.java @@ -1,12 +1,13 @@ + package eu.dnetlib.dhp.api.model; -import eu.dnetlib.dhp.api.model.CommunityContentprovider; - - import java.io.Serializable; import java.util.ArrayList; + +import eu.dnetlib.dhp.api.model.CommunityContentprovider; + public class DatasourceList extends ArrayList implements Serializable { - public DatasourceList(){ + public DatasourceList() { super(); } -} \ No newline at end of file +} diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/OrganizationList.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/OrganizationList.java index 6895967ff..3c81ad179 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/OrganizationList.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/OrganizationList.java @@ -1,3 +1,4 @@ + package eu.dnetlib.dhp.api.model; import java.io.Serializable; @@ -9,7 +10,7 @@ import java.util.ArrayList; */ public class OrganizationList extends ArrayList implements Serializable { - public OrganizationList(){ - super(); - } + public OrganizationList() { + super(); + } } diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/ProjectModel.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/ProjectModel.java index 50bdcf649..3495d6a63 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/ProjectModel.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/ProjectModel.java @@ -1,9 +1,10 @@ + package eu.dnetlib.dhp.api.model; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; - import java.io.Serializable; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + /** * @author miriam.baglioni * @Date 09/10/23 @@ -11,13 +12,13 @@ import java.io.Serializable; @JsonIgnoreProperties(ignoreUnknown = true) public class ProjectModel implements Serializable { - private String openaireId; + private String openaireId; - public String getOpenaireId() { - return openaireId; - } + public String getOpenaireId() { + return openaireId; + } - public void setOpenaireId(String openaireId) { - this.openaireId = openaireId; - } + public void setOpenaireId(String openaireId) { + this.openaireId = openaireId; + } } diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java index b61bc2c9f..0d98e4958 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java @@ -6,7 +6,6 @@ import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; import java.util.*; -import eu.dnetlib.dhp.api.Utils; import org.apache.commons.io.IOUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.function.FilterFunction; @@ -21,6 +20,7 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; +import eu.dnetlib.dhp.api.Utils; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.bulktag.community.*; import eu.dnetlib.dhp.schema.oaf.Datasource; @@ -87,7 +87,7 @@ public class SparkBulkTagJob { if (isTest) { cc = CommunityConfigurationFactory.newInstance(taggingConf); } else { - cc = Utils.getCommunityConfiguration();//QueryInformationSystem.getCommunityConfiguration(parser.get("isLookUpUrl")); + cc = Utils.getCommunityConfiguration();// QueryInformationSystem.getCommunityConfiguration(parser.get("isLookUpUrl")); } runWithSparkSession( diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Community.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Community.java index d281f8783..9cd3a8f82 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Community.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Community.java @@ -8,7 +8,6 @@ import java.util.Optional; import com.google.gson.Gson; - /** Created by miriam on 01/08/2018. */ public class Community implements Serializable { diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfigurationFactory.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfigurationFactory.java index 013bf7168..955ca3856 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfigurationFactory.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfigurationFactory.java @@ -5,6 +5,7 @@ import java.io.StringReader; import java.util.ArrayList; import java.util.List; import java.util.Map; + import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Constraint.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Constraint.java index 03ae1117b..82a6a3b85 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Constraint.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Constraint.java @@ -4,9 +4,10 @@ package eu.dnetlib.dhp.bulktag.community; import java.io.Serializable; import java.lang.reflect.InvocationTargetException; +import org.apache.htrace.fasterxml.jackson.annotation.JsonIgnore; + import eu.dnetlib.dhp.bulktag.criteria.Selection; import eu.dnetlib.dhp.bulktag.criteria.VerbResolver; -import org.apache.htrace.fasterxml.jackson.annotation.JsonIgnore; public class Constraint implements Serializable { private String verb; @@ -39,11 +40,12 @@ public class Constraint implements Serializable { public void setValue(String value) { this.value = value; } + //@JsonIgnore - //public void setSelection(Selection sel) { + // public void setSelection(Selection sel) { // selection = sel; // } -@JsonIgnore + @JsonIgnore public void setSelection(VerbResolver resolver) throws InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { @@ -54,5 +56,4 @@ public class Constraint implements Serializable { return selection.apply(metadata); } - } diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/SelectionConstraints.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/SelectionConstraints.java index 8e6d25e9b..57cc658fc 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/SelectionConstraints.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/SelectionConstraints.java @@ -12,6 +12,7 @@ import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import eu.dnetlib.dhp.bulktag.criteria.VerbResolver; + @JsonAutoDetect public class SelectionConstraints implements Serializable { private List criteria; diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunitySet.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunitySet.java index ff6b73286..e0670b80f 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunitySet.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunitySet.java @@ -6,8 +6,6 @@ import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession; import java.util.*; -import eu.dnetlib.dhp.api.Utils; -import eu.dnetlib.dhp.api.model.CommunityEntityMap; import org.apache.commons.io.IOUtils; import org.apache.hadoop.io.compress.GzipCodec; import org.apache.spark.SparkConf; @@ -20,6 +18,8 @@ import org.slf4j.LoggerFactory; import com.google.gson.Gson; +import eu.dnetlib.dhp.api.Utils; +import eu.dnetlib.dhp.api.model.CommunityEntityMap; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.oaf.Relation; diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java index 745cd7e6f..11dad9055 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java @@ -1572,36 +1572,36 @@ public class BulkTagJobTest { void newConfTest() throws Exception { final String pathMap = BulkTagJobTest.pathMap; SparkBulkTagJob - .main( - new String[] { - "-isTest", Boolean.TRUE.toString(), - "-isSparkSessionManaged", Boolean.FALSE.toString(), - "-sourcePath", - getClass().getResource("/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates").getPath(), - "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Dataset", - "-outputPath", workingDir.toString() + "/dataset", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, - "-pathMap", pathMap - }); + .main( + new String[] { + "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), + "-sourcePath", + getClass().getResource("/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates").getPath(), + "-taggingConf", taggingConf, + "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Dataset", + "-outputPath", workingDir.toString() + "/dataset", + "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + "-pathMap", pathMap + }); final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); JavaRDD tmp = sc - .textFile(workingDir.toString() + "/dataset") - .map(item -> OBJECT_MAPPER.readValue(item, Dataset.class)); + .textFile(workingDir.toString() + "/dataset") + .map(item -> OBJECT_MAPPER.readValue(item, Dataset.class)); Assertions.assertEquals(10, tmp.count()); org.apache.spark.sql.Dataset verificationDataset = spark - .createDataset(tmp.rdd(), Encoders.bean(Dataset.class)); + .createDataset(tmp.rdd(), Encoders.bean(Dataset.class)); verificationDataset.createOrReplaceTempView("dataset"); String query = "select id, MyT.id community " - + "from dataset " - + "lateral view explode(context) c as MyT " - + "lateral view explode(MyT.datainfo) d as MyD " - + "where MyD.inferenceprovenance = 'bulktagging'"; + + "from dataset " + + "lateral view explode(context) c as MyT " + + "lateral view explode(MyT.datainfo) d as MyD " + + "where MyD.inferenceprovenance = 'bulktagging'"; Assertions.assertEquals(0, spark.sql(query).count()); } diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/CommunityConfigurationFactoryTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/CommunityConfigurationFactoryTest.java index 7e12bf9c3..5f0b1d7f1 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/CommunityConfigurationFactoryTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/CommunityConfigurationFactoryTest.java @@ -47,7 +47,7 @@ class CommunityConfigurationFactoryTest { sc.setVerb("not_contains"); sc.setField("contributor"); sc.setValue("DARIAH"); - sc.setSelection(resolver);//.getSelectionCriteria(sc.getVerb(), sc.getValue())); + sc.setSelection(resolver);// .getSelectionCriteria(sc.getVerb(), sc.getValue())); String metadata = "This work has been partially supported by DARIAH-EU infrastructure"; Assertions.assertFalse(sc.verifyCriteria(metadata)); } diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/QueryCommunityAPITest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/QueryCommunityAPITest.java index 1b3403535..b0043d1e8 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/QueryCommunityAPITest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/QueryCommunityAPITest.java @@ -1,14 +1,8 @@ package eu.dnetlib.dhp.bulktag; -import eu.dnetlib.dhp.api.Utils; -import eu.dnetlib.dhp.api.model.CommunityEntityMap; -import eu.dnetlib.dhp.bulktag.community.Community; -import eu.dnetlib.dhp.bulktag.community.CommunityConfiguration; +import java.util.List; -import eu.dnetlib.dhp.api.model.CommunityModel; -import eu.dnetlib.dhp.api.model.CommunitySummary; -import eu.dnetlib.dhp.api.model.DatasourceList; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -16,8 +10,13 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import eu.dnetlib.dhp.api.QueryCommunityAPI; - -import java.util.List; +import eu.dnetlib.dhp.api.Utils; +import eu.dnetlib.dhp.api.model.CommunityEntityMap; +import eu.dnetlib.dhp.api.model.CommunityModel; +import eu.dnetlib.dhp.api.model.CommunitySummary; +import eu.dnetlib.dhp.api.model.DatasourceList; +import eu.dnetlib.dhp.bulktag.community.Community; +import eu.dnetlib.dhp.bulktag.community.CommunityConfiguration; /** * @author miriam.baglioni @@ -43,9 +42,12 @@ public class QueryCommunityAPITest { void community() throws Exception { String id = "dh-ch"; String body = QueryCommunityAPI.community(id); - System.out.println(new ObjectMapper().writeValueAsString(new ObjectMapper() - .readValue(body, CommunityModel.class))) - ; + System.out + .println( + new ObjectMapper() + .writeValueAsString( + new ObjectMapper() + .readValue(body, CommunityModel.class))); } @Test @@ -53,14 +55,14 @@ public class QueryCommunityAPITest { String id = "dh-ch"; String body = QueryCommunityAPI.communityDatasource(id); new ObjectMapper() - .readValue(body, DatasourceList.class) - .forEach(ds-> { - try { - System.out.println(new ObjectMapper().writeValueAsString(ds)); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - }); + .readValue(body, DatasourceList.class) + .forEach(ds -> { + try { + System.out.println(new ObjectMapper().writeValueAsString(ds)); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }); ; } @@ -68,24 +70,33 @@ public class QueryCommunityAPITest { void validCommunities() throws Exception { CommunityConfiguration cc = Utils.getCommunityConfiguration(); System.out.println(cc.getCommunities().keySet()); - Community community =cc.getCommunities().get("aurora"); + Community community = cc.getCommunities().get("aurora"); Assertions.assertEquals(0, community.getSubjects().size()); Assertions.assertEquals(null, community.getConstraints()); Assertions.assertEquals(null, community.getRemoveConstraints()); Assertions.assertEquals(2, community.getZenodoCommunities().size()); - Assertions.assertTrue(community.getZenodoCommunities().stream().anyMatch(c -> c.equals("aurora-universities-network"))); - Assertions.assertTrue(community.getZenodoCommunities().stream().anyMatch(c -> c.equals("university-of-innsbruck"))); + Assertions + .assertTrue( + community.getZenodoCommunities().stream().anyMatch(c -> c.equals("aurora-universities-network"))); + Assertions + .assertTrue(community.getZenodoCommunities().stream().anyMatch(c -> c.equals("university-of-innsbruck"))); Assertions.assertEquals(35, community.getProviders().size()); - Assertions.assertEquals(35, community.getProviders().stream().filter(p->p.getSelectionConstraints()==null).count()); + Assertions + .assertEquals( + 35, community.getProviders().stream().filter(p -> p.getSelectionConstraints() == null).count()); } @Test void getCommunityProjects() throws Exception { CommunityEntityMap projectMap = Utils.getCommunityProjects(); Assertions.assertFalse(projectMap.containsKey("mes")); - Assertions.assertEquals(33, projectMap.size()); - Assertions.assertTrue(projectMap.keySet().stream().allMatch(k -> projectMap.get(k).stream().allMatch(p -> p.startsWith("40|")))); + Assertions.assertEquals(33, projectMap.size()); + Assertions + .assertTrue( + projectMap + .keySet() + .stream() + .allMatch(k -> projectMap.get(k).stream().allMatch(p -> p.startsWith("40|")))); } } - From 9a98f408b36d6ebcd0b1bdeaaa64565c0a899f03 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 10 Oct 2023 09:36:11 +0200 Subject: [PATCH 050/148] code formatting --- .../opencitations/CreateActionSetSparkJob.java | 2 +- .../dnetlib/doiboost/crossref/Crossref2Oaf.scala | 16 +++++++++------- .../doiboost/crossref/CrossrefMappingTest.scala | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java index a367ba852..b707fdcd3 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java @@ -7,7 +7,6 @@ import java.io.IOException; import java.io.Serializable; import java.util.*; -import eu.dnetlib.dhp.schema.oaf.utils.*; import org.apache.commons.cli.ParseException; import org.apache.commons.io.IOUtils; import org.apache.hadoop.io.Text; @@ -30,6 +29,7 @@ import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.schema.action.AtomicAction; import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.oaf.*; +import eu.dnetlib.dhp.schema.oaf.utils.*; import eu.dnetlib.dhp.utils.DHPUtils; import scala.Tuple2; diff --git a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala index e0fdb9ce4..565d34e62 100644 --- a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala +++ b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala @@ -31,9 +31,7 @@ case class mappingAuthor( affiliation: Option[mappingAffiliation] ) {} -case class funderInfo(id:String,uri:String, name:String,synonym:List[String] ) {} - - +case class funderInfo(id: String, uri: String, name: String, synonym: List[String]) {} case class mappingFunder(name: String, DOI: Option[String], award: Option[List[String]]) {} @@ -41,7 +39,9 @@ case object Crossref2Oaf { val logger: Logger = LoggerFactory.getLogger(Crossref2Oaf.getClass) val irishFunder: List[funderInfo] = { - val s = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/doiboost/crossref/irish_funder.json")).mkString + val s = Source + .fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/doiboost/crossref/irish_funder.json")) + .mkString implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats lazy val json: org.json4s.JValue = parse(s) json.extract[List[funderInfo]] @@ -100,9 +100,11 @@ case object Crossref2Oaf { "report" -> "0017 Report" ) - def getIrishId(doi:String):Option[String] = { - val id =doi.split("/").last - irishFunder.find(f => id.equalsIgnoreCase(f.id) || (f.synonym.nonEmpty && f.synonym.exists(s => s.equalsIgnoreCase(id)))).map(f => f.id) + def getIrishId(doi: String): Option[String] = { + val id = doi.split("/").last + irishFunder + .find(f => id.equalsIgnoreCase(f.id) || (f.synonym.nonEmpty && f.synonym.exists(s => s.equalsIgnoreCase(id)))) + .map(f => f.id) } def mappingResult(result: Result, json: JValue, cobjCategory: String): Result = { diff --git a/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala b/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala index 7961376c5..fbf6f72c0 100644 --- a/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala +++ b/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala @@ -50,7 +50,7 @@ class CrossrefMappingTest { } } - def checkRelation(generatedOAF: List[Oaf]): Unit = { + def checkRelation(generatedOAF: List[Oaf]): Unit = { val rels: List[Relation] = generatedOAF.filter(p => p.isInstanceOf[Relation]).asInstanceOf[List[Relation]] From 110ce4b40fc54c2d60fe8120927e76b84580b8c9 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Tue, 10 Oct 2023 09:46:40 +0200 Subject: [PATCH 051/148] extend the fos model to include the level4 and the scores for level3 and level4. removed bip indicators from the instance --- .../dnetlib/dhp/actionmanager/Constants.java | 20 ++++-- .../GetFOSSparkJob.java | 9 ++- .../PrepareFOSSparkJob.java | 27 +++++-- .../SparkSaveUnresolved.java | 6 +- .../model/FOSDataModel.java | 63 +++++++++++++++-- .../CreateActionSetSparkJob.java | 2 +- .../oozie_app/workflow.xml | 56 +++++++-------- .../createunresolvedentities/GetFosTest.java | 39 +++++++++-- .../createunresolvedentities/PrepareTest.java | 70 +++++++++++++++++++ .../createunresolvedentities/ProduceTest.java | 34 +++++++++ .../createunresolvedentities/fos/fos_sbs2.csv | 26 +++++++ .../fos/fos_sbs_2.json | 25 +++++++ .../doiboost/crossref/Crossref2Oaf.scala | 16 +++-- .../crossref/CrossrefMappingTest.scala | 2 +- 14 files changed, 334 insertions(+), 61 deletions(-) create mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos_sbs2.csv create mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos_sbs_2.json diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/Constants.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/Constants.java index 62556b16b..006d3af76 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/Constants.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/Constants.java @@ -40,6 +40,7 @@ public class Constants { public static final String SDG_CLASS_NAME = "Sustainable Development Goals"; public static final String NULL = "NULL"; + public static final String NA = "N/A"; public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @@ -61,10 +62,16 @@ public class Constants { .map((MapFunction) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz)); } - public static Subject getSubject(String sbj, String classid, String classname, - String diqualifierclassid) { - if (sbj == null || sbj.equals(NULL)) + public static Subject getSubject(String sbj, String classid, String classname, String diqualifierclassid, + Boolean split) { + if (sbj == null || sbj.equals(NULL) || sbj.startsWith(NA)) return null; + String trust = ""; + String subject = sbj; + if (split) { + sbj = subject.split("@@")[0]; + trust = subject.split("@@")[1]; + } Subject s = new Subject(); s.setValue(sbj); s @@ -89,9 +96,14 @@ public class Constants { UPDATE_CLASS_NAME, ModelConstants.DNET_PROVENANCE_ACTIONS, ModelConstants.DNET_PROVENANCE_ACTIONS), - "")); + trust)); return s; + } + + public static Subject getSubject(String sbj, String classid, String classname, + String diqualifierclassid) { + return getSubject(sbj, classid, classname, diqualifierclassid, false); } diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/GetFOSSparkJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/GetFOSSparkJob.java index 0cc2f93df..abea6acd7 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/GetFOSSparkJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/GetFOSSparkJob.java @@ -75,9 +75,12 @@ public class GetFOSSparkJob implements Serializable { fosData.map((MapFunction) r -> { FOSDataModel fosDataModel = new FOSDataModel(); fosDataModel.setDoi(r.getString(0).toLowerCase()); - fosDataModel.setLevel1(r.getString(1)); - fosDataModel.setLevel2(r.getString(2)); - fosDataModel.setLevel3(r.getString(3)); + fosDataModel.setLevel1(r.getString(2)); + fosDataModel.setLevel2(r.getString(3)); + fosDataModel.setLevel3(r.getString(4)); + fosDataModel.setLevel4(r.getString(5)); + fosDataModel.setScoreL3(String.valueOf(r.getDouble(6))); + fosDataModel.setScoreL4(String.valueOf(r.getDouble(7))); return fosDataModel; }, Encoders.bean(FOSDataModel.class)) .write() diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareFOSSparkJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareFOSSparkJob.java index 4d2d25215..57ad8b96a 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareFOSSparkJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareFOSSparkJob.java @@ -78,12 +78,20 @@ public class PrepareFOSSparkJob implements Serializable { HashSet level1 = new HashSet<>(); HashSet level2 = new HashSet<>(); HashSet level3 = new HashSet<>(); - addLevels(level1, level2, level3, first); - it.forEachRemaining(v -> addLevels(level1, level2, level3, v)); + HashSet level4 = new HashSet<>(); + addLevels(level1, level2, level3, level4, first); + it.forEachRemaining(v -> addLevels(level1, level2, level3, level4, v)); List sbjs = new ArrayList<>(); - level1.forEach(l -> sbjs.add(getSubject(l, FOS_CLASS_ID, FOS_CLASS_NAME, UPDATE_SUBJECT_FOS_CLASS_ID))); - level2.forEach(l -> sbjs.add(getSubject(l, FOS_CLASS_ID, FOS_CLASS_NAME, UPDATE_SUBJECT_FOS_CLASS_ID))); - level3.forEach(l -> sbjs.add(getSubject(l, FOS_CLASS_ID, FOS_CLASS_NAME, UPDATE_SUBJECT_FOS_CLASS_ID))); + level1 + .forEach(l -> add(sbjs, getSubject(l, FOS_CLASS_ID, FOS_CLASS_NAME, UPDATE_SUBJECT_FOS_CLASS_ID))); + level2 + .forEach(l -> add(sbjs, getSubject(l, FOS_CLASS_ID, FOS_CLASS_NAME, UPDATE_SUBJECT_FOS_CLASS_ID))); + level3 + .forEach( + l -> add(sbjs, getSubject(l, FOS_CLASS_ID, FOS_CLASS_NAME, UPDATE_SUBJECT_FOS_CLASS_ID, true))); + level4 + .forEach( + l -> add(sbjs, getSubject(l, FOS_CLASS_ID, FOS_CLASS_NAME, UPDATE_SUBJECT_FOS_CLASS_ID, true))); r.setSubject(sbjs); r .setDataInfo( @@ -106,11 +114,18 @@ public class PrepareFOSSparkJob implements Serializable { .json(outputPath + "/fos"); } + private static void add(List sbsjs, Subject sbj) { + if (sbj != null) + sbsjs.add(sbj); + } + private static void addLevels(HashSet level1, HashSet level2, HashSet level3, + HashSet level4, FOSDataModel first) { level1.add(first.getLevel1()); level2.add(first.getLevel2()); - level3.add(first.getLevel3()); + level3.add(first.getLevel3() + "@@" + first.getScoreL3()); + level4.add(first.getLevel4() + "@@" + first.getScoreL4()); } } diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/SparkSaveUnresolved.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/SparkSaveUnresolved.java index 3b9775094..93bbfcc88 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/SparkSaveUnresolved.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/SparkSaveUnresolved.java @@ -69,9 +69,9 @@ public class SparkSaveUnresolved implements Serializable { .mapGroups((MapGroupsFunction) (k, it) -> { Result ret = it.next(); it.forEachRemaining(r -> { - if (r.getInstance() != null) { - ret.setInstance(r.getInstance()); - } +// if (r.getInstance() != null) { +// ret.setInstance(r.getInstance()); +// } if (r.getSubject() != null) { if (ret.getSubject() != null) ret.getSubject().addAll(r.getSubject()); diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/model/FOSDataModel.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/model/FOSDataModel.java index e98ba74a1..a82d7bfd6 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/model/FOSDataModel.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/model/FOSDataModel.java @@ -11,21 +11,43 @@ public class FOSDataModel implements Serializable { private String doi; @CsvBindByPosition(position = 1) +// @CsvBindByName(column = "doi") + private String oaid; + @CsvBindByPosition(position = 2) // @CsvBindByName(column = "level1") private String level1; - @CsvBindByPosition(position = 2) + @CsvBindByPosition(position = 3) // @CsvBindByName(column = "level2") private String level2; - @CsvBindByPosition(position = 3) + @CsvBindByPosition(position = 4) // @CsvBindByName(column = "level3") private String level3; + @CsvBindByPosition(position = 5) +// @CsvBindByName(column = "level3") + private String level4; + @CsvBindByPosition(position = 6) + private String scoreL3; + @CsvBindByPosition(position = 7) + private String scoreL4; + public FOSDataModel() { } + public FOSDataModel(String doi, String level1, String level2, String level3, String level4, String l3score, + String l4score) { + this.doi = doi; + this.level1 = level1; + this.level2 = level2; + this.level3 = level3; + this.level4 = level4; + this.scoreL3 = l3score; + this.scoreL4 = l4score; + } + public FOSDataModel(String doi, String level1, String level2, String level3) { this.doi = doi; this.level1 = level1; @@ -33,8 +55,41 @@ public class FOSDataModel implements Serializable { this.level3 = level3; } - public static FOSDataModel newInstance(String d, String level1, String level2, String level3) { - return new FOSDataModel(d, level1, level2, level3); + public static FOSDataModel newInstance(String d, String level1, String level2, String level3, String level4, + String scorel3, String scorel4) { + return new FOSDataModel(d, level1, level2, level3, level4, scorel3, scorel4); + } + + public String getOaid() { + return oaid; + } + + public void setOaid(String oaid) { + this.oaid = oaid; + } + + public String getLevel4() { + return level4; + } + + public void setLevel4(String level4) { + this.level4 = level4; + } + + public String getScoreL3() { + return scoreL3; + } + + public void setScoreL3(String scoreL3) { + this.scoreL3 = scoreL3; + } + + public String getScoreL4() { + return scoreL4; + } + + public void setScoreL4(String scoreL4) { + this.scoreL4 = scoreL4; } public String getDoi() { diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java index a367ba852..b707fdcd3 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java @@ -7,7 +7,6 @@ import java.io.IOException; import java.io.Serializable; import java.util.*; -import eu.dnetlib.dhp.schema.oaf.utils.*; import org.apache.commons.cli.ParseException; import org.apache.commons.io.IOUtils; import org.apache.hadoop.io.Text; @@ -30,6 +29,7 @@ import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.schema.action.AtomicAction; import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.oaf.*; +import eu.dnetlib.dhp.schema.oaf.utils.*; import eu.dnetlib.dhp.utils.DHPUtils; import scala.Tuple2; diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/oozie_app/workflow.xml index c8af64594..a2935a71d 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/oozie_app/workflow.xml @@ -6,10 +6,10 @@ the input path of the resources to be extended - - bipScorePath - the path where to find the bipFinder scores - + + + + outputPath the path where to store the actionset @@ -77,34 +77,34 @@ - + - - - yarn - cluster - Produces the unresolved from BIP! Finder - eu.dnetlib.dhp.actionmanager.createunresolvedentities.PrepareBipFinder - dhp-aggregation-${projectVersion}.jar - - --executor-memory=${sparkExecutorMemory} - --executor-cores=${sparkExecutorCores} - --driver-memory=${sparkDriverMemory} - --conf spark.extraListeners=${spark2ExtraListeners} - --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} - --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} - --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} - - --sourcePath${bipScorePath} - --outputPath${workingDir}/prepared - - - - + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/GetFosTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/GetFosTest.java index 7e0acc2bb..d4fe129df 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/GetFosTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/GetFosTest.java @@ -13,10 +13,7 @@ import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SparkSession; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,6 +65,7 @@ public class GetFosTest { } @Test + @Disabled void test3() throws Exception { final String sourcePath = getClass() .getResource("/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos_sbs.tsv") @@ -96,4 +94,37 @@ public class GetFosTest { tmp.foreach(t -> Assertions.assertTrue(t.getLevel3() != null)); } + + @Test + void test4() throws Exception { + final String sourcePath = getClass() + .getResource("/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos_sbs2.csv") + .getPath(); + + final String outputPath = workingDir.toString() + "/fos.json"; + GetFOSSparkJob + .main( + new String[] { + "--isSparkSessionManaged", Boolean.FALSE.toString(), + "--sourcePath", sourcePath, + "--delimiter", ",", + "-outputPath", outputPath + + }); + + final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .textFile(outputPath) + .map(item -> OBJECT_MAPPER.readValue(item, FOSDataModel.class)); + + tmp.foreach(t -> Assertions.assertTrue(t.getDoi() != null)); + tmp.foreach(t -> Assertions.assertTrue(t.getLevel1() != null)); + tmp.foreach(t -> Assertions.assertTrue(t.getLevel2() != null)); + tmp.foreach(t -> Assertions.assertTrue(t.getLevel3() != null)); + tmp.foreach(t -> Assertions.assertTrue(t.getLevel4() != null)); + tmp.foreach(t -> Assertions.assertTrue(t.getScoreL3() != null)); + tmp.foreach(t -> Assertions.assertTrue(t.getScoreL4() != null)); + + } } diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareTest.java index cc8108bde..ccb0ebbff 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareTest.java @@ -222,6 +222,76 @@ public class PrepareTest { } + @Test + void fosPrepareTest2() throws Exception { + final String sourcePath = getClass() + .getResource("/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos_sbs_2.json") + .getPath(); + + PrepareFOSSparkJob + .main( + new String[] { + "--isSparkSessionManaged", Boolean.FALSE.toString(), + "--sourcePath", sourcePath, + + "-outputPath", workingDir.toString() + "/work" + + }); + + final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .textFile(workingDir.toString() + "/work/fos") + .map(item -> OBJECT_MAPPER.readValue(item, Result.class)); + + String doi1 = "unresolved::10.1016/j.revmed.2006.07.012::doi"; + + assertEquals(13, tmp.count()); + assertEquals(1, tmp.filter(row -> row.getId().equals(doi1)).count()); + + Result result = tmp + .filter(r -> r.getId().equals(doi1)) + .first(); + + result.getSubject().forEach(s -> System.out.println(s.getValue() + " trust = " + s.getDataInfo().getTrust())); + Assertions.assertEquals(6, result.getSubject().size()); + + assertTrue( + result + .getSubject() + .stream() + .anyMatch( + s -> s.getValue().contains("03 medical and health sciences") + && s.getDataInfo().getTrust().equals(""))); + + assertTrue( + result + .getSubject() + .stream() + .anyMatch( + s -> s.getValue().contains("0302 clinical medicine") && s.getDataInfo().getTrust().equals(""))); + + assertTrue( + result + .getSubject() + .stream() + .anyMatch( + s -> s + .getValue() + .contains("030204 cardiovascular system & hematology") + && s.getDataInfo().getTrust().equals("0.5101401805877686"))); + assertTrue( + result + .getSubject() + .stream() + .anyMatch( + s -> s + .getValue() + .contains("03020409 Hematology/Coagulopathies") + && s.getDataInfo().getTrust().equals("0.0546871414174914"))); + + } + @Test void sdgPrepareTest() throws Exception { final String sourcePath = getClass() diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/ProduceTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/ProduceTest.java index c3c110f09..fce6c1e97 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/ProduceTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/ProduceTest.java @@ -379,6 +379,40 @@ public class ProduceTest { .map(item -> OBJECT_MAPPER.readValue(item, Result.class)); } + @Test + public JavaRDD getResultFosJavaRDD() throws Exception { + + final String fosPath = getClass() + .getResource("/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos_sbs_2.json") + .getPath(); + + PrepareFOSSparkJob + .main( + new String[] { + "--isSparkSessionManaged", Boolean.FALSE.toString(), + "--sourcePath", fosPath, + "-outputPath", workingDir.toString() + "/work" + }); + + SparkSaveUnresolved.main(new String[] { + "--isSparkSessionManaged", Boolean.FALSE.toString(), + "--sourcePath", workingDir.toString() + "/work", + + "-outputPath", workingDir.toString() + "/unresolved" + + }); + + final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .textFile(workingDir.toString() + "/unresolved") + .map(item -> OBJECT_MAPPER.readValue(item, Result.class)); + tmp.foreach(r -> System.out.println(new ObjectMapper().writeValueAsString(r))); + + return tmp; + + } + @Test void prepareTest5Subjects() throws Exception { final String doi = "unresolved::10.1063/5.0032658::doi"; diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos_sbs2.csv b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos_sbs2.csv new file mode 100644 index 000000000..3b1f2304f --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos_sbs2.csv @@ -0,0 +1,26 @@ +DOI,OAID,level1,level2,level3,level4,score_for_L3,score_for_L4 +10.1016/j.anucene.2006.02.004,doi_________::00059d9963edf633bec756fb21b5bd72,02 engineering and technology,"0202 electrical engineering, electronic engineering, information engineering",020209 energy,02020908 Climate change policy/Ethanol fuel,0.5,0.5 +10.1016/j.anucene.2006.02.004,doi_________::00059d9963edf633bec756fb21b5bd72,02 engineering and technology,0211 other engineering and technologies,021108 energy,02110808 Climate change policy/Ethanol fuel,0.5,0.5 +10.1016/j.revmed.2006.07.010,doi_________::0026476c1651a92c933d752ff12496c7,03 medical and health sciences,0302 clinical medicine,030220 oncology & carcinogenesis,N/A,0.5036656856536865,0.0 +10.1016/j.revmed.2006.07.010,doi_________::0026476c1651a92c933d752ff12496c7,03 medical and health sciences,0302 clinical medicine,030212 general & internal medicine,N/A,0.4963343143463135,0.0 +10.20965/jrm.2006.p0312,doi_________::0028336a2f3826cc83c47dbefac71543,02 engineering and technology,0209 industrial biotechnology,020901 industrial engineering & automation,02090104 Robotics/Robots,0.6111094951629639,0.5053805979936855 +10.20965/jrm.2006.p0312,doi_________::0028336a2f3826cc83c47dbefac71543,01 natural sciences,0104 chemical sciences,010401 analytical chemistry,N/A,0.3888905048370361,0.0 +10.1111/j.1747-7379.2006.040_1.x,doi_________::002c7077e7c114a8304eb90f59e45fa4,05 social sciences,0506 political science,050602 political science & public administration,05060202 Ethnic groups/Ethnicity,0.6159052848815918,0.7369035568037298 +10.1111/j.1747-7379.2006.040_1.x,doi_________::002c7077e7c114a8304eb90f59e45fa4,05 social sciences,0502 economics and business,050207 economics,N/A,0.3840946555137634,0.0 +10.1007/s10512-006-0049-9,doi_________::003f29f9254819cf4c78558b1bc25f10,02 engineering and technology,"0202 electrical engineering, electronic engineering, information engineering",020209 energy,02020908 Climate change policy/Ethanol fuel,0.5,0.5 +10.1007/s10512-006-0049-9,doi_________::003f29f9254819cf4c78558b1bc25f10,02 engineering and technology,0211 other engineering and technologies,021108 energy,02110808 Climate change policy/Ethanol fuel,0.5,0.5 +10.1111/j.1365-2621.2005.01045.x,doi_________::00419355b4c3e0646bd0e1b301164c8e,04 agricultural and veterinary sciences,0404 agricultural biotechnology,040401 food science,04040102 Food science/Food industry,0.5,0.5 +10.1111/j.1365-2621.2005.01045.x,doi_________::00419355b4c3e0646bd0e1b301164c8e,04 agricultural and veterinary sciences,0405 other agricultural sciences,040502 food science,04050202 Food science/Food industry,0.5,0.5 +10.1002/chin.200617262,doi_________::004c8cef80668904961b9e62841793c8,01 natural sciences,0104 chemical sciences,010405 organic chemistry,01040508 Functional groups/Ethers,0.5566747188568115,0.5582916736602783 +10.1002/chin.200617262,doi_________::004c8cef80668904961b9e62841793c8,01 natural sciences,0104 chemical sciences,010402 general chemistry,01040207 Chemical synthesis/Total synthesis,0.4433253407478332,0.4417082965373993 +10.1016/j.revmed.2006.07.012,doi_________::005b1d0fb650b680abaf6cfe26a21604,03 medical and health sciences,0302 clinical medicine,030204 cardiovascular system & hematology,03020409 Hematology/Coagulopathies,0.5101401805877686,0.0546871414174914 +10.1016/j.revmed.2006.07.012,doi_________::005b1d0fb650b680abaf6cfe26a21604,03 medical and health sciences,0301 basic medicine,030105 genetics & heredity,N/A,0.4898599088191986,0.0 +10.4109/jslab.17.132,doi_________::00889baa06de363e37930daaf8e800c0,03 medical and health sciences,0301 basic medicine,030104 developmental biology,N/A,0.5,0.0 +10.4109/jslab.17.132,doi_________::00889baa06de363e37930daaf8e800c0,03 medical and health sciences,0303 health sciences,030304 developmental biology,N/A,0.5,0.0 +10.1108/00251740610715687,doi_________::0092cb1b1920d556719385a26363ecaa,05 social sciences,0502 economics and business,050203 business & management,05020311 International business/International trade,0.605047881603241,0.2156608108845153 +10.1108/00251740610715687,doi_________::0092cb1b1920d556719385a26363ecaa,05 social sciences,0502 economics and business,050211 marketing,N/A,0.394952118396759,0.0 +10.1080/03067310500248098,doi_________::00a76678d230e3f20b6356804448028f,04 agricultural and veterinary sciences,0404 agricultural biotechnology,040401 food science,04040102 Food science/Food industry,0.5,0.5 +10.1080/03067310500248098,doi_________::00a76678d230e3f20b6356804448028f,04 agricultural and veterinary sciences,0405 other agricultural sciences,040502 food science,04050202 Food science/Food industry,0.5,0.5 +10.3152/147154306781778533,doi_________::00acc520f3939e5a6675343881fed4f2,05 social sciences,0502 economics and business,050203 business & management,05020307 Innovation/Product management,0.5293408632278442,0.5326762795448303 +10.3152/147154306781778533,doi_________::00acc520f3939e5a6675343881fed4f2,05 social sciences,0509 other social sciences,050905 science studies,05090502 Social philosophy/Capitalism,0.4706590473651886,0.4673237204551697 +10.1785/0120050806,doi_________::00d5831d329e7ae4523d78bfc3042e98,02 engineering and technology,0211 other engineering and technologies,021101 geological & geomatics engineering,02110103 Concrete/Building materials,0.5343400835990906,0.3285667930180677 \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos_sbs_2.json b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos_sbs_2.json new file mode 100644 index 000000000..00ffad70c --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos_sbs_2.json @@ -0,0 +1,25 @@ +{"doi":"10.1016/j.anucene.2006.02.004","level1":"02 engineering and technology","level2":"0202 electrical engineering, electronic engineering, information engineering","level3":"020209 energy","level4":"02020908 Climate change policy/Ethanol fuel","scoreL3":"0.5","scoreL4":"0.5"} +{"doi":"10.1016/j.anucene.2006.02.004","level1":"02 engineering and technology","level2":"0211 other engineering and technologies","level3":"021108 energy","level4":"02110808 Climate change policy/Ethanol fuel","scoreL3":"0.5","scoreL4":"0.5"} +{"doi":"10.1016/j.revmed.2006.07.010","level1":"03 medical and health sciences","level2":"0302 clinical medicine","level3":"030220 oncology & carcinogenesis","level4":"N/A","scoreL3":"0.5036656856536865","scoreL4":"0.0"} +{"doi":"10.1016/j.revmed.2006.07.010","level1":"03 medical and health sciences","level2":"0302 clinical medicine","level3":"030212 general & internal medicine","level4":"N/A","scoreL3":"0.4963343143463135","scoreL4":"0.0"} +{"doi":"10.20965/jrm.2006.p0312","level1":"02 engineering and technology","level2":"0209 industrial biotechnology","level3":"020901 industrial engineering & automation","level4":"02090104 Robotics/Robots","scoreL3":"0.6111094951629639","scoreL4":"0.5053805979936855"} +{"doi":"10.20965/jrm.2006.p0312","level1":"01 natural sciences","level2":"0104 chemical sciences","level3":"010401 analytical chemistry","level4":"N/A","scoreL3":"0.3888905048370361","scoreL4":"0.0"} +{"doi":"10.1111/j.1747-7379.2006.040_1.x","level1":"05 social sciences","level2":"0506 political science","level3":"050602 political science & public administration","level4":"05060202 Ethnic groups/Ethnicity","scoreL3":"0.6159052848815918","scoreL4":"0.7369035568037298"} +{"doi":"10.1111/j.1747-7379.2006.040_1.x","level1":"05 social sciences","level2":"0502 economics and business","level3":"050207 economics","level4":"N/A","scoreL3":"0.3840946555137634","scoreL4":"0.0"} +{"doi":"10.1007/s10512-006-0049-9","level1":"02 engineering and technology","level2":"0202 electrical engineering, electronic engineering, information engineering","level3":"020209 energy","level4":"02020908 Climate change policy/Ethanol fuel","scoreL3":"0.5","scoreL4":"0.5"} +{"doi":"10.1007/s10512-006-0049-9","level1":"02 engineering and technology","level2":"0211 other engineering and technologies","level3":"021108 energy","level4":"02110808 Climate change policy/Ethanol fuel","scoreL3":"0.5","scoreL4":"0.5"} +{"doi":"10.1111/j.1365-2621.2005.01045.x","level1":"04 agricultural and veterinary sciences","level2":"0404 agricultural biotechnology","level3":"040401 food science","level4":"04040102 Food science/Food industry","scoreL3":"0.5","scoreL4":"0.5"} +{"doi":"10.1111/j.1365-2621.2005.01045.x","level1":"04 agricultural and veterinary sciences","level2":"0405 other agricultural sciences","level3":"040502 food science","level4":"04050202 Food science/Food industry","scoreL3":"0.5","scoreL4":"0.5"} +{"doi":"10.1002/chin.200617262","level1":"01 natural sciences","level2":"0104 chemical sciences","level3":"010405 organic chemistry","level4":"01040508 Functional groups/Ethers","scoreL3":"0.5566747188568115","scoreL4":"0.5582916736602783"} +{"doi":"10.1002/chin.200617262","level1":"01 natural sciences","level2":"0104 chemical sciences","level3":"010402 general chemistry","level4":"01040207 Chemical synthesis/Total synthesis","scoreL3":"0.4433253407478332","scoreL4":"0.4417082965373993"} +{"doi":"10.1016/j.revmed.2006.07.012","level1":"03 medical and health sciences","level2":"0302 clinical medicine","level3":"030204 cardiovascular system & hematology","level4":"03020409 Hematology/Coagulopathies","scoreL3":"0.5101401805877686","scoreL4":"0.0546871414174914"} +{"doi":"10.1016/j.revmed.2006.07.012","level1":"03 medical and health sciences","level2":"0301 basic medicine","level3":"030105 genetics & heredity","level4":"N/A","scoreL3":"0.4898599088191986","scoreL4":"0.0"} +{"doi":"10.4109/jslab.17.132","level1":"03 medical and health sciences","level2":"0301 basic medicine","level3":"030104 developmental biology","level4":"N/A","scoreL3":"0.5","scoreL4":"0.0"} +{"doi":"10.4109/jslab.17.132","level1":"03 medical and health sciences","level2":"0303 health sciences","level3":"030304 developmental biology","level4":"N/A","scoreL3":"0.5","scoreL4":"0.0"} +{"doi":"10.1108/00251740610715687","level1":"05 social sciences","level2":"0502 economics and business","level3":"050203 business & management","level4":"05020311 International business/International trade","scoreL3":"0.605047881603241","scoreL4":"0.2156608108845153"} +{"doi":"10.1108/00251740610715687","level1":"05 social sciences","level2":"0502 economics and business","level3":"050211 marketing","level4":"N/A","scoreL3":"0.394952118396759","scoreL4":"0.0"} +{"doi":"10.1080/03067310500248098","level1":"04 agricultural and veterinary sciences","level2":"0404 agricultural biotechnology","level3":"040401 food science","level4":"04040102 Food science/Food industry","scoreL3":"0.5","scoreL4":"0.5"} +{"doi":"10.1080/03067310500248098","level1":"04 agricultural and veterinary sciences","level2":"0405 other agricultural sciences","level3":"040502 food science","level4":"04050202 Food science/Food industry","scoreL3":"0.5","scoreL4":"0.5"} +{"doi":"10.3152/147154306781778533","level1":"05 social sciences","level2":"0502 economics and business","level3":"050203 business & management","level4":"05020307 Innovation/Product management","scoreL3":"0.5293408632278442","scoreL4":"0.5326762795448303"} +{"doi":"10.3152/147154306781778533","level1":"05 social sciences","level2":"0509 other social sciences","level3":"050905 science studies","level4":"05090502 Social philosophy/Capitalism","scoreL3":"0.4706590473651886","scoreL4":"0.4673237204551697"} +{"doi":"10.1785/0120050806","level1":"02 engineering and technology","level2":"0211 other engineering and technologies","level3":"021101 geological & geomatics engineering","level4":"02110103 Concrete/Building materials","scoreL3":"0.5343400835990906","scoreL4":"0.3285667930180677"} diff --git a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala index e0fdb9ce4..565d34e62 100644 --- a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala +++ b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala @@ -31,9 +31,7 @@ case class mappingAuthor( affiliation: Option[mappingAffiliation] ) {} -case class funderInfo(id:String,uri:String, name:String,synonym:List[String] ) {} - - +case class funderInfo(id: String, uri: String, name: String, synonym: List[String]) {} case class mappingFunder(name: String, DOI: Option[String], award: Option[List[String]]) {} @@ -41,7 +39,9 @@ case object Crossref2Oaf { val logger: Logger = LoggerFactory.getLogger(Crossref2Oaf.getClass) val irishFunder: List[funderInfo] = { - val s = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/doiboost/crossref/irish_funder.json")).mkString + val s = Source + .fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/doiboost/crossref/irish_funder.json")) + .mkString implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats lazy val json: org.json4s.JValue = parse(s) json.extract[List[funderInfo]] @@ -100,9 +100,11 @@ case object Crossref2Oaf { "report" -> "0017 Report" ) - def getIrishId(doi:String):Option[String] = { - val id =doi.split("/").last - irishFunder.find(f => id.equalsIgnoreCase(f.id) || (f.synonym.nonEmpty && f.synonym.exists(s => s.equalsIgnoreCase(id)))).map(f => f.id) + def getIrishId(doi: String): Option[String] = { + val id = doi.split("/").last + irishFunder + .find(f => id.equalsIgnoreCase(f.id) || (f.synonym.nonEmpty && f.synonym.exists(s => s.equalsIgnoreCase(id)))) + .map(f => f.id) } def mappingResult(result: Result, json: JValue, cobjCategory: String): Result = { diff --git a/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala b/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala index 7961376c5..fbf6f72c0 100644 --- a/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala +++ b/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala @@ -50,7 +50,7 @@ class CrossrefMappingTest { } } - def checkRelation(generatedOAF: List[Oaf]): Unit = { + def checkRelation(generatedOAF: List[Oaf]): Unit = { val rels: List[Relation] = generatedOAF.filter(p => p.isInstanceOf[Relation]).asInstanceOf[List[Relation]] From ed9282ef2a3e40a76308fbff9226a3bfa6a90df6 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 10 Oct 2023 09:52:03 +0200 Subject: [PATCH 052/148] removed module dhp-stats-monitor-update --- .../oozie_app/config-default.xml | 30 ---- .../oozie_app/copyDataToImpalaCluster.sh | 75 --------- .../oozie_app/finalizeImpalaCluster.sh | 29 ---- .../graph/stats-monitor/oozie_app/monitor.sh | 54 ------- .../oozie_app/scripts/updateMonitorDB.sql | 138 ---------------- .../oozie_app/scripts/updateMonitorDBAll.sql | 150 ------------------ .../scripts/updateMonitorDB_institutions.sql | 12 -- .../stats-monitor/oozie_app/workflow.xml | 110 ------------- 8 files changed, 598 deletions(-) delete mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/config-default.xml delete mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh delete mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/finalizeImpalaCluster.sh delete mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/monitor.sh delete mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB.sql delete mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDBAll.sql delete mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB_institutions.sql delete mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/workflow.xml diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/config-default.xml b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/config-default.xml deleted file mode 100644 index b2a1322e6..000000000 --- a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/config-default.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - jobTracker - ${jobTracker} - - - nameNode - ${nameNode} - - - oozie.use.system.libpath - true - - - oozie.action.sharelib.for.spark - spark2 - - - hive_metastore_uris - thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083 - - - hive_jdbc_url - jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000/;UseNativeQuery=1;?spark.executor.memory=22166291558;spark.yarn.executor.memoryOverhead=3225;spark.driver.memory=15596411699;spark.yarn.driver.memoryOverhead=1228 - - - oozie.wf.workflow.notification.url - {serviceUrl}/v1/oozieNotification/jobUpdate?jobId=$jobId%26status=$status - - \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh deleted file mode 100644 index 1587f7152..000000000 --- a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh +++ /dev/null @@ -1,75 +0,0 @@ -export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs -export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami) -if ! [ -L $link_folder ] -then - rm -Rf "$link_folder" - ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder} -fi - -#export HADOOP_USER_NAME=$2 - -function copydb() { - - export HADOOP_USER="dimitris.pierrakos" - export HADOOP_USER_NAME='dimitris.pierrakos' - - db=$1 - FILE=("hive_wf_tmp_"$RANDOM) - hdfs dfs -mkdir hdfs://impala-cluster-mn1.openaire.eu:8020/tmp/$FILE/ - - # change ownership to impala -# hdfs dfs -conf /etc/impala_cluster/hdfs-site.xml -chmod -R 777 /tmp/$FILE/${db}.db - hdfs dfs -conf /etc/impala_cluster/hdfs-site.xml -chmod -R 777 /tmp/$FILE/ - - - # copy the databases from ocean to impala - echo "copying $db" - hadoop distcp -Dmapreduce.map.memory.mb=6144 -pb hdfs://nameservice1/user/hive/warehouse/${db}.db hdfs://impala-cluster-mn1.openaire.eu:8020/tmp/$FILE/ - - hdfs dfs -conf /etc/impala_cluster/hdfs-site.xml -chmod -R 777 /tmp/$FILE/${db}.db - - # drop tables from db - for i in `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} --delimited -q "show tables"`; - do - `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "drop table $i;"`; - done - - # drop views from db - for i in `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} --delimited -q "show tables"`; - do - `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "drop view $i;"`; - done - - # delete the database - impala-shell -i impala-cluster-dn1.openaire.eu -q "drop database if exists ${db} cascade"; - - # create the databases - impala-shell -i impala-cluster-dn1.openaire.eu -q "create database ${db}"; - - impala-shell -q "INVALIDATE METADATA" - echo "creating schema for ${db}" - for (( k = 0; k < 5; k ++ )); do - for i in `impala-shell -d ${db} --delimited -q "show tables"`; - do - impala-shell -d ${db} --delimited -q "show create table $i"; - done | sed 's/"$/;/' | sed 's/^"//' | sed 's/[[:space:]]\date[[:space:]]/`date`/g' | impala-shell --user $HADOOP_USER_NAME -i impala-cluster-dn1.openaire.eu -c -f - - done - - # load the data from /tmp in the respective tables - echo "copying data in tables and computing stats" - for i in `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} --delimited -q "show tables"`; - do - impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "load data inpath '/tmp/$FILE/${db}.db/$i' into table $i"; - impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "compute stats $i"; - done - - # deleting the remaining directory from hdfs -hdfs dfs -conf /etc/impala_cluster/hdfs-site.xml -rm -R /tmp/$FILE/${db}.db -} - -MONITOR_DB=$1 -#HADOOP_USER_NAME=$2 - -copydb $MONITOR_DB'_institutions' -copydb $MONITOR_DB - diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/finalizeImpalaCluster.sh b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/finalizeImpalaCluster.sh deleted file mode 100644 index a7227e0c8..000000000 --- a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/finalizeImpalaCluster.sh +++ /dev/null @@ -1,29 +0,0 @@ -export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs -export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami) -if ! [ -L $link_folder ] -then - rm -Rf "$link_folder" - ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder} -fi - -function createShadowDB() { - SOURCE=$1 - SHADOW=$2 - - # drop views from db - for i in `impala-shell -i impala-cluster-dn1.openaire.eu -d ${SHADOW} --delimited -q "show tables"`; - do - `impala-shell -i impala-cluster-dn1.openaire.eu -d ${SHADOW} -q "drop view $i;"`; - done - - impala-shell -i impala-cluster-dn1.openaire.eu -q "drop database ${SHADOW} CASCADE"; - impala-shell -i impala-cluster-dn1.openaire.eu -q "create database if not exists ${SHADOW}"; -# impala-shell -i impala-cluster-dn1.openaire.eu -d ${SHADOW} -q "show tables" | sed "s/^/drop view if exists ${SHADOW}./" | sed "s/$/;/" | impala-shell -i impala-cluster-dn1.openaire.eu -f - - impala-shell -i impala-cluster-dn1.openaire.eu -d ${SOURCE} -q "show tables" --delimited | sed "s/\(.*\)/create view ${SHADOW}.\1 as select * from ${SOURCE}.\1;/" | impala-shell -i impala-cluster-dn1.openaire.eu -f - -} - -MONITOR_DB=$1 -MONITOR_DB_SHADOW=$2 - -createShadowDB $MONITOR_DB'_institutions' $MONITOR_DB'_institutions_shadow' -createShadowDB $MONITOR_DB $MONITOR_DB'_shadow' diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/monitor.sh b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/monitor.sh deleted file mode 100644 index 4f1889c9e..000000000 --- a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/monitor.sh +++ /dev/null @@ -1,54 +0,0 @@ -export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs -export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami) -if ! [ -L $link_folder ] -then - rm -Rf "$link_folder" - ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder} -fi - -export SOURCE=$1 -export TARGET=$2 -export SHADOW=$3 -export SCRIPT_PATH=$4 -export SCRIPT_PATH2=$5 -export SCRIPT_PATH2=$6 - -export HIVE_OPTS="-hiveconf mapred.job.queue.name=analytics -hiveconf hive.spark.client.connect.timeout=120000ms -hiveconf hive.spark.client.server.connect.timeout=300000ms -hiveconf spark.executor.memory=19166291558 -hiveconf spark.yarn.executor.memoryOverhead=3225 -hiveconf spark.driver.memory=11596411699 -hiveconf spark.yarn.driver.memoryOverhead=1228" -export HADOOP_USER_NAME="oozie" - -echo "Getting file from " $4 -hdfs dfs -copyToLocal $4 - -echo "Getting file from " $5 -hdfs dfs -copyToLocal $5 - -echo "Getting file from " $6 -hdfs dfs -copyToLocal $6 - -#update Institutions DB -cat updateMonitorDB_institutions.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2_institutions/g1" > foo -hive $HIVE_OPTS -f foo -cat updateMonitorDB.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2_institutions/g1" > foo -hive $HIVE_OPTS -f foo - -echo "Hive shell finished" - -echo "Updating shadow monitor insitutions database" -hive -e "drop database if exists ${SHADOW}_institutions cascade" -hive -e "create database if not exists ${SHADOW}_institutions" -hive $HIVE_OPTS --database ${2}_institutions -e "show tables" | grep -v WARN | sed "s/\(.*\)/create view ${SHADOW}_institutions.\1 as select * from ${2}_institutions.\1;/" > foo -hive -f foo -echo "Shadow db monitor insitutions ready!" - -#update Monitor DB -cat updateMonitorDBAll.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2/g1" > foo -hive $HIVE_OPTS -f foo - -echo "Hive shell finished" - -echo "Updating shadow monitor database" -hive -e "drop database if exists ${SHADOW} cascade" -hive -e "create database if not exists ${SHADOW}" -hive $HIVE_OPTS --database ${2} -e "show tables" | grep -v WARN | sed "s/\(.*\)/create view ${SHADOW}.\1 as select * from ${2}.\1;/" > foo -hive -f foo -echo "Shadow db monitor insitutions ready!" diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB.sql b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB.sql deleted file mode 100644 index 248b7e564..000000000 --- a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB.sql +++ /dev/null @@ -1,138 +0,0 @@ -INSERT INTO TARGET.result select * from TARGET.result_new; -ANALYZE TABLE TARGET.result COMPUTE STATISTICS; - -INSERT INTO TARGET.result_citations select * from SOURCE.result_citations orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_citations COMPUTE STATISTICS; - -INSERT INTO TARGET.result_references_oc select * from SOURCE.result_references_oc orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_references_oc COMPUTE STATISTICS; - -INSERT INTO TARGET.result_classifications select * from SOURCE.result_classifications orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_classifications COMPUTE STATISTICS; - -INSERT INTO TARGET.result_apc select * from SOURCE.result_apc orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_apc COMPUTE STATISTICS; - -INSERT INTO TARGET.result_concepts select * from SOURCE.result_concepts orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_concepts COMPUTE STATISTICS; - -INSERT INTO TARGET.result_datasources select * from SOURCE.result_datasources orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_datasources COMPUTE STATISTICS; - -INSERT INTO TARGET.result_fundercount select * from SOURCE.result_fundercount orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_fundercount COMPUTE STATISTICS; - -INSERT INTO TARGET.result_gold select * from SOURCE.result_gold orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_gold COMPUTE STATISTICS; - -INSERT INTO TARGET.result_greenoa select * from SOURCE.result_greenoa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_greenoa COMPUTE STATISTICS; - -INSERT INTO TARGET.result_languages select * from SOURCE.result_languages orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_languages COMPUTE STATISTICS; - -INSERT INTO TARGET.result_licenses select * from SOURCE.result_licenses orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_licenses COMPUTE STATISTICS; - -INSERT INTO TARGET.result_oids select * from SOURCE.result_oids orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_oids COMPUTE STATISTICS; - -INSERT INTO TARGET.result_organization select * from SOURCE.result_organization orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_organization COMPUTE STATISTICS; - -INSERT INTO TARGET.result_peerreviewed select * from SOURCE.result_peerreviewed orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_peerreviewed COMPUTE STATISTICS; - -INSERT INTO TARGET.result_pids select * from SOURCE.result_pids orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_pids COMPUTE STATISTICS; - -INSERT INTO TARGET.result_projectcount select * from SOURCE.result_projectcount orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_projectcount COMPUTE STATISTICS; - -INSERT INTO TARGET.result_projects select * from SOURCE.result_projects orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_projects COMPUTE STATISTICS; - -INSERT INTO TARGET.result_refereed select * from SOURCE.result_refereed orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_refereed COMPUTE STATISTICS; - -INSERT INTO TARGET.result_sources select * from SOURCE.result_sources orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_sources COMPUTE STATISTICS; - -INSERT INTO TARGET.result_topics select * from SOURCE.result_topics orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_topics COMPUTE STATISTICS; - -INSERT INTO TARGET.result_fos select * from SOURCE.result_fos orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_fos COMPUTE STATISTICS; - -INSERT INTO TARGET.result_accessroute select * from SOURCE.result_accessroute orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_accessroute COMPUTE STATISTICS; - -create or replace view TARGET.foo1 as select * from SOURCE.result_result rr where rr.source in (select id from TARGET.result_new); -create or replace view TARGET.foo2 as select * from SOURCE.result_result rr where rr.target in (select id from TARGET.result_new); -insert into TARGET.result_result select distinct * from (select * from TARGET.foo1 union all select * from TARGET.foo2) foufou; -drop view TARGET.foo1; -drop view TARGET.foo2; -ANALYZE TABLE TARGET.result_result COMPUTE STATISTICS; - - --- indicators --- Sprint 1 ---- -INSERT INTO TARGET.indi_pub_green_oa select * from SOURCE.indi_pub_green_oa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_green_oa COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_grey_lit select * from SOURCE.indi_pub_grey_lit orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_grey_lit COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_doi_from_crossref select * from SOURCE.indi_pub_doi_from_crossref orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_doi_from_crossref COMPUTE STATISTICS; --- Sprint 2 ---- -INSERT INTO TARGET.indi_result_has_cc_licence select * from SOURCE.indi_result_has_cc_licence orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_has_cc_licence COMPUTE STATISTICS; -INSERT INTO TARGET.indi_result_has_cc_licence_url select * from SOURCE.indi_result_has_cc_licence_url orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_has_cc_licence_url COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_has_abstract select * from SOURCE.indi_pub_has_abstract orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_has_abstract COMPUTE STATISTICS; -INSERT INTO TARGET.indi_result_with_orcid select * from SOURCE.indi_result_with_orcid orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_with_orcid COMPUTE STATISTICS; ----- Sprint 3 ---- -INSERT INTO TARGET.indi_funded_result_with_fundref select * from SOURCE.indi_funded_result_with_fundref orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_funded_result_with_fundref COMPUTE STATISTICS; - ----- Sprint 4 ---- -INSERT INTO TARGET.indi_pub_diamond select * from SOURCE.indi_pub_diamond orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_diamond COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_in_transformative select * from SOURCE.indi_pub_in_transformative orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_in_transformative COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_closed_other_open select * from SOURCE.indi_pub_closed_other_open orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_closed_other_open COMPUTE STATISTICS; ----- Sprint 5 ---- -INSERT INTO TARGET.indi_result_no_of_copies select * from SOURCE.indi_result_no_of_copies orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_no_of_copies COMPUTE STATISTICS; ----- Sprint 6 ---- -INSERT INTO TARGET.indi_pub_hybrid_oa_with_cc select * from SOURCE.indi_pub_hybrid_oa_with_cc orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_hybrid_oa_with_cc COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_bronze_oa select * from SOURCE.indi_pub_bronze_oa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_bronze_oa COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_downloads select * from SOURCE.indi_pub_downloads orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); -ANALYZE TABLE TARGET.indi_pub_downloads COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_downloads_datasource select * from SOURCE.indi_pub_downloads_datasource orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); -ANALYZE TABLE TARGET.indi_pub_downloads_datasource COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_downloads_year select * from SOURCE.indi_pub_downloads_year orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); -ANALYZE TABLE TARGET.indi_pub_downloads_year COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_downloads_datasource_year select * from SOURCE.indi_pub_downloads_datasource_year orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); -ANALYZE TABLE TARGET.indi_pub_downloads_datasource_year COMPUTE STATISTICS; ----- Sprint 7 ---- -INSERT INTO TARGET.indi_pub_gold_oa select * from SOURCE.indi_pub_gold_oa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_gold_oa COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_hybrid select * from SOURCE.indi_pub_hybrid orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_hybrid COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_has_preprint select * from SOURCE.indi_pub_has_preprint orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_has_preprint COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_in_subscribed select * from SOURCE.indi_pub_in_subscribed orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_in_subscribed COMPUTE STATISTICS; -INSERT INTO TARGET.indi_result_with_pid select * from SOURCE.indi_result_with_pid orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_with_pid COMPUTE STATISTICS; -INSERT INTO TARGET.indi_impact_measures select * from SOURCE.indi_impact_measures orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_impact_measures COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_interdisciplinarity select * from SOURCE.indi_pub_interdisciplinarity orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_interdisciplinarity COMPUTE STATISTICS; - -DROP TABLE IF EXISTS TARGET.result_new; \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDBAll.sql b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDBAll.sql deleted file mode 100644 index 478e3824e..000000000 --- a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDBAll.sql +++ /dev/null @@ -1,150 +0,0 @@ -DROP TABLE IF EXISTS TARGET.result_new; - -create table TARGET.result_new as - select distinct * from ( - select * from SOURCE.result r where exists (select 1 from SOURCE.result_organization ro where ro.id=r.id and ro.organization in ( - 'openorgs____::4d4051b56708688235252f1d8fddb8c1', --Iscte - Instituto Universitário de Lisboa - 'openorgs____::ab4ac74c35fa5dada770cf08e5110fab' -- Universidade Católica Portuguesa - ) )) foo; - -INSERT INTO TARGET.result select * from TARGET.result_new; -ANALYZE TABLE TARGET.result_new COMPUTE STATISTICS; - -INSERT INTO TARGET.result select * from TARGET.result_new; -ANALYZE TABLE TARGET.result COMPUTE STATISTICS; - -INSERT INTO TARGET.result_citations select * from SOURCE.result_citations orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_citations COMPUTE STATISTICS; - -INSERT INTO TARGET.result_references_oc select * from SOURCE.result_references_oc orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_references_oc COMPUTE STATISTICS; - -INSERT INTO TARGET.result_classifications select * from SOURCE.result_classifications orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_classifications COMPUTE STATISTICS; - -INSERT INTO TARGET.result_apc select * from SOURCE.result_apc orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_apc COMPUTE STATISTICS; - -INSERT INTO TARGET.result_concepts select * from SOURCE.result_concepts orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_concepts COMPUTE STATISTICS; - -INSERT INTO TARGET.result_datasources select * from SOURCE.result_datasources orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_datasources COMPUTE STATISTICS; - -INSERT INTO TARGET.result_fundercount select * from SOURCE.result_fundercount orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_fundercount COMPUTE STATISTICS; - -INSERT INTO TARGET.result_gold select * from SOURCE.result_gold orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_gold COMPUTE STATISTICS; - -INSERT INTO TARGET.result_greenoa select * from SOURCE.result_greenoa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_greenoa COMPUTE STATISTICS; - -INSERT INTO TARGET.result_languages select * from SOURCE.result_languages orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_languages COMPUTE STATISTICS; - -INSERT INTO TARGET.result_licenses select * from SOURCE.result_licenses orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_licenses COMPUTE STATISTICS; - -INSERT INTO TARGET.result_oids select * from SOURCE.result_oids orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_oids COMPUTE STATISTICS; - -INSERT INTO TARGET.result_organization select * from SOURCE.result_organization orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_organization COMPUTE STATISTICS; - -INSERT INTO TARGET.result_peerreviewed select * from SOURCE.result_peerreviewed orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_peerreviewed COMPUTE STATISTICS; - -INSERT INTO TARGET.result_pids select * from SOURCE.result_pids orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_pids COMPUTE STATISTICS; - -INSERT INTO TARGET.result_projectcount select * from SOURCE.result_projectcount orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_projectcount COMPUTE STATISTICS; - -INSERT INTO TARGET.result_projects select * from SOURCE.result_projects orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_projects COMPUTE STATISTICS; - -INSERT INTO TARGET.result_refereed select * from SOURCE.result_refereed orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_refereed COMPUTE STATISTICS; - -INSERT INTO TARGET.result_sources select * from SOURCE.result_sources orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_sources COMPUTE STATISTICS; - -INSERT INTO TARGET.result_topics select * from SOURCE.result_topics orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_topics COMPUTE STATISTICS; - -INSERT INTO TARGET.result_fos select * from SOURCE.result_fos orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_fos COMPUTE STATISTICS; - -INSERT INTO TARGET.result_accessroute select * from SOURCE.result_accessroute orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_accessroute COMPUTE STATISTICS; - -create or replace view TARGET.foo1 as select * from SOURCE.result_result rr where rr.source in (select id from TARGET.result_new); -create or replace view TARGET.foo2 as select * from SOURCE.result_result rr where rr.target in (select id from TARGET.result_new); -insert into TARGET.result_result select distinct * from (select * from TARGET.foo1 union all select * from TARGET.foo2) foufou; -drop view TARGET.foo1; -drop view TARGET.foo2; -ANALYZE TABLE TARGET.result_result COMPUTE STATISTICS; - - --- indicators --- Sprint 1 ---- -INSERT INTO TARGET.indi_pub_green_oa select * from SOURCE.indi_pub_green_oa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_green_oa COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_grey_lit select * from SOURCE.indi_pub_grey_lit orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_grey_lit COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_doi_from_crossref select * from SOURCE.indi_pub_doi_from_crossref orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_doi_from_crossref COMPUTE STATISTICS; --- Sprint 2 ---- -INSERT INTO TARGET.indi_result_has_cc_licence select * from SOURCE.indi_result_has_cc_licence orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_has_cc_licence COMPUTE STATISTICS; -INSERT INTO TARGET.indi_result_has_cc_licence_url select * from SOURCE.indi_result_has_cc_licence_url orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_has_cc_licence_url COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_has_abstract select * from SOURCE.indi_pub_has_abstract orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_has_abstract COMPUTE STATISTICS; -INSERT INTO TARGET.indi_result_with_orcid select * from SOURCE.indi_result_with_orcid orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_with_orcid COMPUTE STATISTICS; ----- Sprint 3 ---- -INSERT INTO TARGET.indi_funded_result_with_fundref select * from SOURCE.indi_funded_result_with_fundref orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_funded_result_with_fundref COMPUTE STATISTICS; - ----- Sprint 4 ---- -INSERT INTO TARGET.indi_pub_diamond select * from SOURCE.indi_pub_diamond orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_diamond COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_in_transformative select * from SOURCE.indi_pub_in_transformative orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_in_transformative COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_closed_other_open select * from SOURCE.indi_pub_closed_other_open orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_closed_other_open COMPUTE STATISTICS; ----- Sprint 5 ---- -INSERT INTO TARGET.indi_result_no_of_copies select * from SOURCE.indi_result_no_of_copies orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_no_of_copies COMPUTE STATISTICS; ----- Sprint 6 ---- -INSERT INTO TARGET.indi_pub_hybrid_oa_with_cc select * from SOURCE.indi_pub_hybrid_oa_with_cc orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_hybrid_oa_with_cc COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_bronze_oa select * from SOURCE.indi_pub_bronze_oa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_bronze_oa COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_downloads select * from SOURCE.indi_pub_downloads orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); -ANALYZE TABLE TARGET.indi_pub_downloads COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_downloads_datasource select * from SOURCE.indi_pub_downloads_datasource orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); -ANALYZE TABLE TARGET.indi_pub_downloads_datasource COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_downloads_year select * from SOURCE.indi_pub_downloads_year orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); -ANALYZE TABLE TARGET.indi_pub_downloads_year COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_downloads_datasource_year select * from SOURCE.indi_pub_downloads_datasource_year orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); -ANALYZE TABLE TARGET.indi_pub_downloads_datasource_year COMPUTE STATISTICS; ----- Sprint 7 ---- -INSERT INTO TARGET.indi_pub_gold_oa select * from SOURCE.indi_pub_gold_oa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_gold_oa COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_hybrid select * from SOURCE.indi_pub_hybrid orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_hybrid COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_has_preprint select * from SOURCE.indi_pub_has_preprint orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_has_preprint COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_in_subscribed select * from SOURCE.indi_pub_in_subscribed orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_in_subscribed COMPUTE STATISTICS; -INSERT INTO TARGET.indi_result_with_pid select * from SOURCE.indi_result_with_pid orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_with_pid COMPUTE STATISTICS; -INSERT INTO TARGET.indi_impact_measures select * from SOURCE.indi_impact_measures orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_impact_measures COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_interdisciplinarity select * from SOURCE.indi_pub_interdisciplinarity orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_interdisciplinarity COMPUTE STATISTICS; - -DROP TABLE IF EXISTS TARGET.result_new; diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB_institutions.sql b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB_institutions.sql deleted file mode 100644 index 236f3733f..000000000 --- a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB_institutions.sql +++ /dev/null @@ -1,12 +0,0 @@ -DROP TABLE IF EXISTS TARGET.result_new; - -create table TARGET.result_new as - select distinct * from ( - select * from SOURCE.result r where exists (select 1 from SOURCE.result_organization ro where ro.id=r.id and ro.organization in ( - 'openorgs____::4d4051b56708688235252f1d8fddb8c1', --Iscte - Instituto Universitário de Lisboa - 'openorgs____::ab4ac74c35fa5dada770cf08e5110fab' -- Universidade Católica Portuguesa - ) )) foo; - -INSERT INTO TARGET.result select * from TARGET.result_new; -ANALYZE TABLE TARGET.result_new COMPUTE STATISTICS; - diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/workflow.xml b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/workflow.xml deleted file mode 100644 index 7b999a843..000000000 --- a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/workflow.xml +++ /dev/null @@ -1,110 +0,0 @@ - - - - stats_db_name - the target stats database name - - - monitor_db_name - the target monitor db name - - - monitor_db_shadow_name - the name of the shadow monitor db - - - hive_metastore_uris - hive server metastore URIs - - - hive_jdbc_url - hive server jdbc url - - - hive_timeout - the time period, in seconds, after which Hive fails a transaction if a Hive client has not sent a hearbeat. The default value is 300 seconds. - - - hadoop_user_name - user name of the wf owner - - - - - ${jobTracker} - ${nameNode} - - - hive.metastore.uris - ${hive_metastore_uris} - - - hive.txn.timeout - ${hive_timeout} - - - mapred.job.queue.name - analytics - - - - - - - - ${wf:conf('resumeFrom') eq 'Step1-updateMonitorDB'} - ${wf:conf('resumeFrom') eq 'Step2-copyDataToImpalaCluster'} - ${wf:conf('resumeFrom') eq 'Step3-finalizeImpalaCluster'} - - - - - - Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] - - - - - ${jobTracker} - ${nameNode} - monitor.sh - ${stats_db_name} - ${monitor_db_name} - ${monitor_db_shadow_name} - ${wf:appPath()}/scripts/updateMonitorDB_institutions.sql - ${wf:appPath()}/scripts/updateMonitorDB.sql - ${wf:appPath()}/scripts/updateMonitorDBAll.sql - monitor.sh - - - - - - - - ${jobTracker} - ${nameNode} - copyDataToImpalaCluster.sh - ${monitor_db_name} - ${hadoop_user_name} - copyDataToImpalaCluster.sh - - - - - - - - ${jobTracker} - ${nameNode} - finalizeImpalaCluster.sh - ${monitor_db_name} - ${monitor_db_shadow_name} - finalizeImpalaCluster.sh - - - - - - - From a431b04814dc55c56ab2bde7d2a2663b7fc0950a Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Tue, 10 Oct 2023 12:53:57 +0200 Subject: [PATCH 053/148] leftover for the properties and removal of bipfinder --- .../PrepareBipFinder.java | 178 ------------------ .../oozie_app/workflow.xml | 31 +-- .../createunresolvedentities/PrepareTest.java | 139 -------------- .../createunresolvedentities/ProduceTest.java | 30 --- 4 files changed, 1 insertion(+), 377 deletions(-) delete mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareBipFinder.java diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareBipFinder.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareBipFinder.java deleted file mode 100644 index 0507f90e5..000000000 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareBipFinder.java +++ /dev/null @@ -1,178 +0,0 @@ - -package eu.dnetlib.dhp.actionmanager.createunresolvedentities; - -import static eu.dnetlib.dhp.actionmanager.Constants.*; -import static eu.dnetlib.dhp.actionmanager.Constants.UPDATE_CLASS_NAME; -import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; - -import java.io.Serializable; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; - -import org.apache.commons.io.IOUtils; -import org.apache.spark.SparkConf; -import org.apache.spark.api.java.JavaRDD; -import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.api.java.function.MapFunction; -import org.apache.spark.sql.Encoders; -import org.apache.spark.sql.SaveMode; -import org.apache.spark.sql.SparkSession; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.databind.ObjectMapper; - -import eu.dnetlib.dhp.actionmanager.bipmodel.BipScore; -import eu.dnetlib.dhp.actionmanager.bipmodel.score.deserializers.BipResultModel; -import eu.dnetlib.dhp.application.ArgumentApplicationParser; -import eu.dnetlib.dhp.common.HdfsSupport; -import eu.dnetlib.dhp.schema.common.ModelConstants; -import eu.dnetlib.dhp.schema.oaf.Instance; -import eu.dnetlib.dhp.schema.oaf.KeyValue; -import eu.dnetlib.dhp.schema.oaf.Measure; -import eu.dnetlib.dhp.schema.oaf.Result; -import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions; -import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils; -import eu.dnetlib.dhp.utils.DHPUtils; - -public class PrepareBipFinder implements Serializable { - - private static final Logger log = LoggerFactory.getLogger(PrepareBipFinder.class); - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - - public static void main(String[] args) throws Exception { - - String jsonConfiguration = IOUtils - .toString( - PrepareBipFinder.class - .getResourceAsStream( - "/eu/dnetlib/dhp/actionmanager/createunresolvedentities/prepare_parameters.json")); - - final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); - - parser.parseArgument(args); - - Boolean isSparkSessionManaged = Optional - .ofNullable(parser.get("isSparkSessionManaged")) - .map(Boolean::valueOf) - .orElse(Boolean.TRUE); - - log.info("isSparkSessionManaged: {}", isSparkSessionManaged); - - final String sourcePath = parser.get("sourcePath"); - log.info("sourcePath {}: ", sourcePath); - - final String outputPath = parser.get("outputPath"); - log.info("outputPath {}: ", outputPath); - - SparkConf conf = new SparkConf(); - - runWithSparkSession( - conf, - isSparkSessionManaged, - spark -> { - HdfsSupport.remove(outputPath, spark.sparkContext().hadoopConfiguration()); - prepareResults(spark, sourcePath, outputPath); - }); - } - - private static void prepareResults(SparkSession spark, String inputPath, String outputPath) { - - final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); - - JavaRDD bipDeserializeJavaRDD = sc - .textFile(inputPath) - .map(item -> OBJECT_MAPPER.readValue(item, BipResultModel.class)); - - spark - .createDataset(bipDeserializeJavaRDD.flatMap(entry -> entry.keySet().stream().map(key -> { - BipScore bs = new BipScore(); - bs.setId(key); - bs.setScoreList(entry.get(key)); - - return bs; - }).collect(Collectors.toList()).iterator()).rdd(), Encoders.bean(BipScore.class)) - .map((MapFunction) v -> { - Result r = new Result(); - final String cleanedPid = CleaningFunctions.normalizePidValue(DOI, v.getId()); - - r.setId(DHPUtils.generateUnresolvedIdentifier(v.getId(), DOI)); - Instance inst = new Instance(); - inst.setMeasures(getMeasure(v)); - - inst - .setPid( - Arrays - .asList( - OafMapperUtils - .structuredProperty( - cleanedPid, - OafMapperUtils - .qualifier( - DOI, DOI_CLASSNAME, - ModelConstants.DNET_PID_TYPES, - ModelConstants.DNET_PID_TYPES), - null))); - r.setInstance(Arrays.asList(inst)); - r - .setDataInfo( - OafMapperUtils - .dataInfo( - false, null, true, - false, - OafMapperUtils - .qualifier( - ModelConstants.PROVENANCE_ENRICH, - null, - ModelConstants.DNET_PROVENANCE_ACTIONS, - ModelConstants.DNET_PROVENANCE_ACTIONS), - null)); - return r; - }, Encoders.bean(Result.class)) - .write() - .mode(SaveMode.Overwrite) - .option("compression", "gzip") - .json(outputPath + "/bip"); - } - - private static List getMeasure(BipScore value) { - return value - .getScoreList() - .stream() - .map(score -> { - Measure m = new Measure(); - m.setId(score.getId()); - m - .setUnit( - score - .getUnit() - .stream() - .map(unit -> { - KeyValue kv = new KeyValue(); - kv.setValue(unit.getValue()); - kv.setKey(unit.getKey()); - kv - .setDataInfo( - OafMapperUtils - .dataInfo( - false, - UPDATE_DATA_INFO_TYPE, - true, - false, - OafMapperUtils - .qualifier( - UPDATE_MEASURE_BIP_CLASS_ID, - UPDATE_CLASS_NAME, - ModelConstants.DNET_PROVENANCE_ACTIONS, - ModelConstants.DNET_PROVENANCE_ACTIONS), - "")); - return kv; - }) - .collect(Collectors.toList())); - return m; - }) - .collect(Collectors.toList()); - } -} diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/oozie_app/workflow.xml index a2935a71d..a5388f28b 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/oozie_app/workflow.xml @@ -5,11 +5,6 @@ fosPath the input path of the resources to be extended - - - - - outputPath the path where to store the actionset @@ -77,35 +72,10 @@ - - - - - - - - - - - - - - - - - - - - - - - - - yarn @@ -125,6 +95,7 @@ --sourcePath${fosPath} --outputPath${workingDir}/input/fos + --delimiter${delimiter} diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareTest.java index ccb0ebbff..da7bcd3de 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareTest.java @@ -67,92 +67,6 @@ public class PrepareTest { spark.stop(); } - @Test - void bipPrepareTest() throws Exception { - final String sourcePath = getClass() - .getResource("/eu/dnetlib/dhp/actionmanager/createunresolvedentities/bip/bip.json") - .getPath(); - - PrepareBipFinder - .main( - new String[] { - "--isSparkSessionManaged", Boolean.FALSE.toString(), - "--sourcePath", sourcePath, - "--outputPath", workingDir.toString() + "/work" - - }); - - final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); - - JavaRDD tmp = sc - .textFile(workingDir.toString() + "/work/bip") - .map(item -> OBJECT_MAPPER.readValue(item, Result.class)); - - Assertions.assertEquals(86, tmp.count()); - - String doi1 = "unresolved::10.0000/096020199389707::doi"; - - Assertions.assertEquals(1, tmp.filter(r -> r.getId().equals(doi1)).count()); - Assertions.assertEquals(1, tmp.filter(r -> r.getId().equals(doi1)).collect().get(0).getInstance().size()); - Assertions - .assertEquals( - 3, tmp.filter(r -> r.getId().equals(doi1)).collect().get(0).getInstance().get(0).getMeasures().size()); - Assertions - .assertEquals( - "6.34596412687e-09", tmp - .filter(r -> r.getId().equals(doi1)) - .collect() - .get(0) - .getInstance() - .get(0) - .getMeasures() - .stream() - .filter(sl -> sl.getId().equals("influence")) - .collect(Collectors.toList()) - .get(0) - .getUnit() - .get(0) - .getValue()); - Assertions - .assertEquals( - "0.641151896994", tmp - .filter(r -> r.getId().equals(doi1)) - .collect() - .get(0) - .getInstance() - .get(0) - .getMeasures() - .stream() - .filter(sl -> sl.getId().equals("popularity_alt")) - .collect(Collectors.toList()) - .get(0) - .getUnit() - .get(0) - .getValue()); - Assertions - .assertEquals( - "2.33375102921e-09", tmp - .filter(r -> r.getId().equals(doi1)) - .collect() - .get(0) - .getInstance() - .get(0) - .getMeasures() - .stream() - .filter(sl -> sl.getId().equals("popularity")) - .collect(Collectors.toList()) - .get(0) - .getUnit() - .get(0) - .getValue()); - - final String doi2 = "unresolved::10.3390/s18072310::doi"; - - Assertions.assertEquals(1, tmp.filter(r -> r.getId().equals(doi2)).count()); - Assertions.assertEquals(1, tmp.filter(r -> r.getId().equals(doi2)).collect().get(0).getInstance().size()); - - } - @Test void fosPrepareTest() throws Exception { final String sourcePath = getClass() @@ -338,57 +252,4 @@ public class PrepareTest { } -// @Test -// void test3() throws Exception { -// final String sourcePath = "/Users/miriam.baglioni/Downloads/doi_fos_results_20_12_2021.csv.gz"; -// -// final String outputPath = workingDir.toString() + "/fos.json"; -// GetFOSSparkJob -// .main( -// new String[] { -// "--isSparkSessionManaged", Boolean.FALSE.toString(), -// "--sourcePath", sourcePath, -// -// "-outputPath", outputPath -// -// }); -// -// final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); -// -// JavaRDD tmp = sc -// .textFile(outputPath) -// .map(item -> OBJECT_MAPPER.readValue(item, FOSDataModel.class)); -// -// tmp.foreach(t -> Assertions.assertTrue(t.getDoi() != null)); -// tmp.foreach(t -> Assertions.assertTrue(t.getLevel1() != null)); -// tmp.foreach(t -> Assertions.assertTrue(t.getLevel2() != null)); -// tmp.foreach(t -> Assertions.assertTrue(t.getLevel3() != null)); -// -// } -// -// @Test -// void test4() throws Exception { -// final String sourcePath = "/Users/miriam.baglioni/Downloads/doi_sdg_results_20_12_21.csv.gz"; -// -// final String outputPath = workingDir.toString() + "/sdg.json"; -// GetSDGSparkJob -// .main( -// new String[] { -// "--isSparkSessionManaged", Boolean.FALSE.toString(), -// "--sourcePath", sourcePath, -// -// "-outputPath", outputPath -// -// }); -// -// final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); -// -// JavaRDD tmp = sc -// .textFile(outputPath) -// .map(item -> OBJECT_MAPPER.readValue(item, SDGDataModel.class)); -// -// tmp.foreach(t -> Assertions.assertTrue(t.getDoi() != null)); -// tmp.foreach(t -> Assertions.assertTrue(t.getSbj() != null)); -// -// } } diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/ProduceTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/ProduceTest.java index fce6c1e97..ce116688a 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/ProduceTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/ProduceTest.java @@ -340,18 +340,7 @@ public class ProduceTest { } private JavaRDD getResultJavaRDD() throws Exception { - final String bipPath = getClass() - .getResource("/eu/dnetlib/dhp/actionmanager/createunresolvedentities/bip/bip.json") - .getPath(); - PrepareBipFinder - .main( - new String[] { - "--isSparkSessionManaged", Boolean.FALSE.toString(), - "--sourcePath", bipPath, - "--outputPath", workingDir.toString() + "/work" - - }); final String fosPath = getClass() .getResource("/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos.json") .getPath(); @@ -449,18 +438,7 @@ public class ProduceTest { } private JavaRDD getResultJavaRDDPlusSDG() throws Exception { - final String bipPath = getClass() - .getResource("/eu/dnetlib/dhp/actionmanager/createunresolvedentities/bip/bip.json") - .getPath(); - PrepareBipFinder - .main( - new String[] { - "--isSparkSessionManaged", Boolean.FALSE.toString(), - "--sourcePath", bipPath, - "--outputPath", workingDir.toString() + "/work" - - }); final String fosPath = getClass() .getResource("/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos.json") .getPath(); @@ -517,14 +495,6 @@ public class ProduceTest { .filter(row -> row.getSubject() != null) .count()); - Assertions - .assertEquals( - 85, - tmp - .filter(row -> !row.getId().equals(doi)) - .filter(r -> r.getInstance() != null && r.getInstance().size() > 0) - .count()); - } @Test From a460ebe215ebe1f535905d0d3121a84bcd087c2b Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 10 Oct 2023 15:50:11 +0200 Subject: [PATCH 054/148] [UnresolvedEntities] updated action name --- .../createunresolvedentities/oozie_app/workflow.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/oozie_app/workflow.xml index a5388f28b..c8e9547dc 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/oozie_app/workflow.xml @@ -184,7 +184,7 @@ yarn cluster - Saves the result produced for bip and fos by grouping results with the same id + Save the unresolved entities grouping results with the same id eu.dnetlib.dhp.actionmanager.createunresolvedentities.SparkSaveUnresolved dhp-aggregation-${projectVersion}.jar From 554551682d0501fa7664fab0a9bee8be6b0309d3 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Wed, 11 Oct 2023 16:09:19 +0200 Subject: [PATCH 055/148] [raw graph] adopting the new COAR based vocabularies for the resource typing --- .../common/vocabulary/VocabularyGroup.java | 21 ++++++++++++ .../dhp/schema/oaf/utils/OafMapperUtils.java | 10 +++++- .../raw/AbstractMdRecordToOafMapper.java | 31 ++++++++++++++++++ .../dhp/oa/graph/raw/OafToOafMapper.java | 32 ++++++++++++++++--- .../dhp/oa/graph/raw/OdfToOafMapper.java | 27 ++++++++++++++++ .../oa/graph/raw/OriginalTypeComparator.java | 32 +++++++++++++++++++ .../dnetlib/dhp/oa/graph/raw/MappersTest.java | 21 ++++++++++++ .../dnetlib/dhp/oa/graph/clean/synonyms.txt | 7 +++- .../eu/dnetlib/dhp/oa/graph/clean/terms.txt | 4 ++- .../dnetlib/dhp/oa/graph/raw/oaf_record.xml | 4 ++- pom.xml | 2 +- 11 files changed, 182 insertions(+), 9 deletions(-) create mode 100644 dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OriginalTypeComparator.java diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/vocabulary/VocabularyGroup.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/vocabulary/VocabularyGroup.java index fc7175270..4c1feac45 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/vocabulary/VocabularyGroup.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/vocabulary/VocabularyGroup.java @@ -135,6 +135,27 @@ public class VocabularyGroup implements Serializable { return vocs.get(vocId.toLowerCase()).getSynonymAsQualifier(syn); } + public Qualifier lookupTermBySynonym(final String vocId, final String syn) { + if (StringUtils.isBlank(vocId)) { + return OafMapperUtils.unknown("", ""); + } + + final Vocabulary vocabulary = vocs.get(vocId.toLowerCase()); + + return Optional + .ofNullable(vocabulary.getTerm(syn)) + .map( + term -> OafMapperUtils + .qualifier(term.getId(), term.getName(), vocabulary.getId(), vocabulary.getName())) + .orElse( + Optional + .ofNullable(vocabulary.getTermBySynonym(syn)) + .map( + term -> OafMapperUtils + .qualifier(term.getId(), term.getName(), vocabulary.getId(), vocabulary.getName())) + .orElse(null)); + } + /** * getSynonymAsQualifierCaseSensitive * diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/OafMapperUtils.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/OafMapperUtils.java index c58096d35..1c557c805 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/OafMapperUtils.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/OafMapperUtils.java @@ -14,7 +14,6 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import eu.dnetlib.dhp.schema.common.AccessRightComparator; -import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.oaf.*; @@ -141,6 +140,15 @@ public class OafMapperUtils { .collect(Collectors.toList()); } + public static InstanceTypeMapping instanceTypeMapping(String originalType, Qualifier term) { + final InstanceTypeMapping m = new InstanceTypeMapping(); + m.setVocabularyName(term.getSchemeid()); + m.setOriginalType(originalType); + m.setTypeCode(term.getClassid()); + m.setTypeLabel(term.getClassname()); + return m; + } + public static Qualifier unknown(final String schemeid, final String schemename) { return qualifier(UNKNOWN, "Unknown", schemeid, schemename); } diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java index b37e6a755..2fed7d627 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java @@ -41,6 +41,11 @@ public abstract class AbstractMdRecordToOafMapper { protected static final String DATACITE_SCHEMA_KERNEL_4_SLASH = "http://datacite.org/schema/kernel-4/"; protected static final String DATACITE_SCHEMA_KERNEL_3 = "http://datacite.org/schema/kernel-3"; protected static final String DATACITE_SCHEMA_KERNEL_3_SLASH = "http://datacite.org/schema/kernel-3/"; + + protected static final String OPENAIRE_COAR_RESOURCE_TYPES_3_1 = "openaire::coar_resource_types_3_1"; + + public static final String OPENAIRE_USER_RESOURCE_TYPES = "openaire::user_resource_types"; + protected static final Qualifier ORCID_PID_TYPE = qualifier( ModelConstants.ORCID_PENDING, ModelConstants.ORCID_CLASSNAME, @@ -516,6 +521,32 @@ public abstract class AbstractMdRecordToOafMapper { protected abstract Field prepareDatasetStorageDate(Document doc, DataInfo info); + protected abstract String findOriginalType(Document doc); + + protected List prepareInstanceTypeMapping(Document doc) { + return Optional.ofNullable(findOriginalType(doc)) + .map(originalType -> { + final List mappings = Lists.newArrayList(); + + if (vocs.vocabularyExists(OPENAIRE_COAR_RESOURCE_TYPES_3_1)) { + + // TODO verify what the vocabs return when a synonym is not defined + Qualifier coarTerm = vocs.lookupTermBySynonym(OPENAIRE_COAR_RESOURCE_TYPES_3_1, originalType); + mappings.add(OafMapperUtils.instanceTypeMapping(originalType, coarTerm)); + + if (vocs.vocabularyExists(OPENAIRE_USER_RESOURCE_TYPES)) { + + // TODO verify what the vocabs return when a synonym is not defined + Qualifier userTerm = vocs.lookupTermBySynonym(OPENAIRE_USER_RESOURCE_TYPES, coarTerm.getClassid()); + mappings.add(OafMapperUtils.instanceTypeMapping(originalType, userTerm)); + } + } + + return mappings; + }) + .orElse(new ArrayList<>()); + } + private Journal prepareJournal(final Document doc, final DataInfo info) { final Node n = doc.selectSingleNode("//oaf:journal"); if (n != null) { diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java index a9f9367af..5cdb434f1 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java @@ -5,12 +5,10 @@ import static eu.dnetlib.dhp.schema.common.ModelConstants.*; import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.*; import java.net.URLDecoder; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; +import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils; import org.apache.commons.lang3.StringUtils; import org.dom4j.Document; import org.dom4j.Element; @@ -25,6 +23,8 @@ import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions; import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; import eu.dnetlib.dhp.schema.oaf.utils.ModelHardLimits; +import static org.apache.commons.lang3.StringUtils.contains; + public class OafToOafMapper extends AbstractMdRecordToOafMapper { public OafToOafMapper(final VocabularyGroup vocs, final boolean invisible, final boolean shouldHashId, @@ -139,6 +139,8 @@ public class OafToOafMapper extends AbstractMdRecordToOafMapper { final List alternateIdentifier = prepareResultPids(doc, info); final List pid = IdentifierFactory.getPids(alternateIdentifier, collectedfrom); + instance.setInstanceTypeMapping(prepareInstanceTypeMapping(doc)); + final Set pids = new HashSet<>(pid); instance @@ -187,6 +189,28 @@ public class OafToOafMapper extends AbstractMdRecordToOafMapper { return Lists.newArrayList(instance); } + /** + * The Dublin Core element dc:type can be repeated, but we need to base our mapping on a single value + * So this method tries to give precedence to the COAR resource type, when available. Otherwise, it looks for the + * openaire's info:eu-repo type, and as last resort picks the 1st type text available + * + * http://purl.org/coar/resource_type/c_5794 + * info:eu-repo/semantics/article + * Conference article + * + * @param doc the input document + * @return the chosen resource type + */ + @Override + protected String findOriginalType(Document doc) { + return (String) doc.selectNodes("//dc:type") + .stream() + .map(o -> "" + ((Node) o).getText().trim()) + .sorted(new OriginalTypeComparator()) + .findFirst() + .orElse(null); + } + @Override protected List> prepareSources(final Document doc, final DataInfo info) { return prepareListFields(doc, "//dc:source", info); diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java index bbd1e7ab1..c01775327 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java @@ -9,6 +9,7 @@ import java.net.URLDecoder; import java.util.*; import java.util.stream.Collectors; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.dom4j.Document; import org.dom4j.Element; @@ -139,6 +140,8 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper { final List alternateIdentifier = prepareResultPids(doc, info); final List pid = IdentifierFactory.getPids(alternateIdentifier, collectedfrom); + instance.setInstanceTypeMapping(prepareInstanceTypeMapping(doc)); + final Set pids = new HashSet<>(pid); instance @@ -217,6 +220,30 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper { } } + @Override + protected List prepareInstanceTypeMapping(Document doc) { + return null; + } + + /** + * The Datacite element + * + * journal article + * + * @param doc the input document + * @return the chosen resource type + */ + @Override + protected String findOriginalType(Document doc) { + final Element resourceType = (Element) doc.selectSingleNode( + "//metadata/*[local-name() = 'resource']/*[local-name() = 'resourceType']"); + + final String resourceTypeURI = resourceType.attributeValue("anyURI"); + final String resourceTypeTxt = resourceType.getText(); + + return ObjectUtils.firstNonNull(resourceTypeURI, resourceTypeTxt); + } + @Override protected List> prepareSources(final Document doc, final DataInfo info) { return new ArrayList<>(); // Not present in ODF ??? diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OriginalTypeComparator.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OriginalTypeComparator.java new file mode 100644 index 000000000..2eeead32e --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OriginalTypeComparator.java @@ -0,0 +1,32 @@ +package eu.dnetlib.dhp.oa.graph.raw; + +import java.util.Comparator; + +import static org.apache.commons.lang3.StringUtils.contains; +import static org.apache.commons.lang3.StringUtils.startsWith; + +public class OriginalTypeComparator implements Comparator { + + @Override + public int compare(String t1, String t2) { + + if (t1.equals(t2)) { + return 0; + } + if (startsWith(t1, "http") && contains(t1, "coar") && contains(t1, "resource_type")) { + return -1; + } + if (startsWith(t2, "http") && contains(t2, "coar") && contains(t2, "resource_type")) { + return 1; + } + if (startsWith(t1, "info:eu-repo/semantics")) { + return -1; + } + if (startsWith(t2, "info:eu-repo/semantics")) { + return 1; + } + + return t1.compareTo(t2); + } + +} diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java index b506d3a62..5df2b7a3b 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java @@ -116,6 +116,27 @@ class MappersTest { assertNotNull(instance.getPid()); assertTrue(instance.getPid().isEmpty()); + assertNotNull(instance.getInstanceTypeMapping()); + assertEquals(2, instance.getInstanceTypeMapping().size()); + + Optional coarType = instance.getInstanceTypeMapping() + .stream() + .filter(itm -> AbstractMdRecordToOafMapper.OPENAIRE_COAR_RESOURCE_TYPES_3_1.equals(itm.getVocabularyName())) + .findFirst(); + + assertTrue(coarType.isPresent()); + assertEquals("http://purl.org/coar/resource_type/c_5794", coarType.get().getTypeCode()); + assertEquals("conference paper", coarType.get().getTypeLabel()); + + Optional userType = instance.getInstanceTypeMapping() + .stream() + .filter(itm -> AbstractMdRecordToOafMapper.OPENAIRE_USER_RESOURCE_TYPES.equals(itm.getVocabularyName())) + .findFirst(); + + assertTrue(userType.isPresent()); + assertEquals("Article", userType.get().getTypeCode()); + assertEquals("Article", userType.get().getTypeLabel()); + assertFalse(instance.getAlternateIdentifier().isEmpty()); assertEquals("doi", instance.getAlternateIdentifier().get(0).getQualifier().getClassid()); assertEquals("10.3897/oneeco.2.e13718", instance.getAlternateIdentifier().get(0).getValue()); diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/synonyms.txt b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/synonyms.txt index 409dfd5dc..b3fa94b20 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/synonyms.txt +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/synonyms.txt @@ -1245,4 +1245,9 @@ dnet:relation_relClass @=@ IsRelatedTo @=@ isRelatedTo dnet:relation_subRelType @=@ relationship @=@ publicationDataset dnet:provenanceActions @=@ iis @=@ erroneous label to be cleaned FOS @=@ 0101 mathematics @=@ FOS: Mathematics -FOS @=@ 0102 computer and information sciences @=@ FOS: Computer and information sciences \ No newline at end of file +FOS @=@ 0102 computer and information sciences @=@ FOS: Computer and information sciences +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ Proceedings paper +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ Conference article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ http://purl.org/eprint/type/ConferencePaper +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ Conference article +openaire::user_resource_types @=@ Article @=@ http://purl.org/coar/resource_type/c_5794 \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/terms.txt b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/terms.txt index 24ce42fc2..a38c0e987 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/terms.txt +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/terms.txt @@ -1121,4 +1121,6 @@ dnet:relation_subRelType @=@ dnet:relation_subRelType @=@ supplement @=@ supplem dnet:relation_subRelType @=@ dnet:relation_subRelType @=@ version @=@ version FOS @=@ Fields of Science and Technology classification @=@ 0101 mathematics @=@ 0101 mathematics FOS @=@ Fields of Science and Technology classification @=@ 0102 computer and information sciences @=@ 0102 computer and information sciences -FOS @=@ Fields of Science and Technology classification @=@ 0103 physical sciences @=@ 0103 physical sciences \ No newline at end of file +FOS @=@ Fields of Science and Technology classification @=@ 0103 physical sciences @=@ 0103 physical sciences +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ conference paper +openaire::user_resource_types @=@ openaire::user_resource_types @=@ Article @=@ Article \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/oaf_record.xml b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/oaf_record.xml index 277578185..492fc9a7a 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/oaf_record.xml +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/oaf_record.xml @@ -47,7 +47,9 @@ provisioning services regulating services supporting services - Research Article + conference paper + http://purl.org/coar/resource_type/c_5794 + info:eu-repo/semantics/article 0001 2017-01-01 diff --git a/pom.xml b/pom.xml index 9cd82a343..0f1ec78af 100644 --- a/pom.xml +++ b/pom.xml @@ -888,7 +888,7 @@ 3.3.3 3.4.2 [2.12,3.0) - [3.17.1] + [4.17.2-SNAPSHOT] [4.0.3] [6.0.5] [3.1.6] From 89184d5b4f1af674af33fff2a46961804b8da751 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Wed, 11 Oct 2023 18:17:35 +0200 Subject: [PATCH 056/148] used the API instead of the IS for bulktagging and propagation for community through organization. Added a new propagation step for communities through projects. Still using the API and not the IS --- .../eu/dnetlib/dhp/PropagationConstant.java | 3 + .../eu/dnetlib/dhp/api/QueryCommunityAPI.java | 38 ++-- .../main/java/eu/dnetlib/dhp/api/Utils.java | 64 +++--- .../dnetlib/dhp/bulktag/SparkBulkTagJob.java | 70 ++++--- .../dhp/bulktag/community/ResultTagger.java | 28 +-- .../PrepareResultCommunitySet.java | 9 +- ...kResultToCommunityFromOrganizationJob.java | 12 +- .../PrepareResultCommunitySet.java | 121 ++++++++++++ .../ResultProjectList.java | 26 +++ .../SparkResultToCommunityFromProject.java | 156 +++++++++++++++ .../dhp/bulktag/input_bulkTag_parameters.json | 28 +-- .../dhp/bulktag/oozie_app/workflow.xml | 108 +--------- .../resources/eu/dnetlib/dhp/bulktag/query.xq | 62 ------ .../input_communitytoresult_parameters.json | 12 -- .../oozie_app/workflow.xml | 7 +- .../dnetlib/dhp/bulktag/BulkTagJobTest.java | 187 +++++++++--------- .../dhp/bulktag/QueryCommunityAPITest.java | 19 +- .../ResultToCommunityJobTest.java | 2 +- .../eosc/dataset/{dataset_10.json => dataset} | 0 .../bulktag/eosc/dataset/otherresearchproduct | 0 .../dhp/bulktag/eosc/dataset/publication | 0 .../dnetlib/dhp/bulktag/eosc/dataset/software | 0 .../bulktag/sample/dataset/no_updates/dataset | 10 + .../dataset/no_updates/dataset_10.json.gz | Bin 6736 -> 0 bytes .../sample/dataset/no_updates/datasource | 0 .../dataset/no_updates/otherresearchproduct | 0 .../sample/dataset/no_updates/publication | 0 .../sample/dataset/no_updates/software | 0 .../{dataset_10.json => dataset} | 0 .../datasource | 0 .../otherresearchproduct | 10 + .../publication | 0 .../update_datasourcewithconstraints/software | 0 .../contextnoprovenance/dataset | 10 + .../contextnoprovenance/dataset_10.json.gz | Bin 6888 -> 0 bytes .../contextnoprovenance/datasource | 0 .../contextnoprovenance/otherresearchproduct | 0 .../contextnoprovenance/publication | 0 .../contextnoprovenance/software | 0 .../dataset/update_subject/nocontext/dataset | 10 + .../nocontext/dataset_10.json.gz | Bin 6755 -> 0 bytes .../update_subject/nocontext/datasource | 0 .../nocontext/otherresearchproduct | 0 .../update_subject/nocontext/publication | 0 .../dataset/update_subject/nocontext/software | 0 .../{dataset_10.json => dataset} | 6 +- .../update_subject_datasource/datasource | 0 .../otherresearchproduct | 0 .../update_subject_datasource/publication | 0 .../update_subject_datasource/software | 0 .../update_zenodocommunity/dataset | 0 .../update_zenodocommunity/datasource | 0 .../otherresearchproduct | 10 + .../otherresearchproduct_10.json.gz | Bin 8300 -> 0 bytes .../update_zenodocommunity/publication | 0 .../update_zenodocommunity/software | 0 .../publication/update_datasource/dataset | 0 .../publication/update_datasource/datasource | 0 .../update_datasource/otherresearchproduct | 0 .../publication/update_datasource/publication | 10 + .../update_datasource/publication_10.json.gz | Bin 5257 -> 0 bytes .../publication/update_datasource/software | 0 .../dhp/bulktag/sample/software/dataset | 0 .../sample/software/otherresearchproduct | 0 .../dhp/bulktag/sample/software/publication | 0 .../dhp/bulktag/sample/software/software | 10 + .../sample/software/software_10.json.gz | Bin 6727 -> 0 bytes .../eu/dnetlib/dhp/eosctag/galaxy/publication | 0 .../dnetlib/dhp/eosctag/jupyter/publication | 0 .../dnetlib/dhp/eosctag/twitter/publication | 0 70 files changed, 622 insertions(+), 406 deletions(-) create mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java create mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultProjectList.java create mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java delete mode 100644 dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/query.xq rename dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/eosc/dataset/{dataset_10.json => dataset} (100%) create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/eosc/dataset/otherresearchproduct create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/eosc/dataset/publication create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/eosc/dataset/software create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates/dataset delete mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates/dataset_10.json.gz create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates/datasource create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates/otherresearchproduct create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates/publication create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates/software rename dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints/{dataset_10.json => dataset} (100%) create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints/datasource create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints/otherresearchproduct create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints/publication create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints/software create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/contextnoprovenance/dataset delete mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/contextnoprovenance/dataset_10.json.gz create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/contextnoprovenance/datasource create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/contextnoprovenance/otherresearchproduct create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/contextnoprovenance/publication create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/contextnoprovenance/software create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/nocontext/dataset delete mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/nocontext/dataset_10.json.gz create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/nocontext/datasource create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/nocontext/otherresearchproduct create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/nocontext/publication create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/nocontext/software rename dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject_datasource/{dataset_10.json => dataset} (93%) create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject_datasource/datasource create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject_datasource/otherresearchproduct create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject_datasource/publication create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject_datasource/software create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/otherresearchproduct/update_zenodocommunity/dataset create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/otherresearchproduct/update_zenodocommunity/datasource create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/otherresearchproduct/update_zenodocommunity/otherresearchproduct delete mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/otherresearchproduct/update_zenodocommunity/otherresearchproduct_10.json.gz create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/otherresearchproduct/update_zenodocommunity/publication create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/otherresearchproduct/update_zenodocommunity/software create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/publication/update_datasource/dataset create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/publication/update_datasource/datasource create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/publication/update_datasource/otherresearchproduct create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/publication/update_datasource/publication delete mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/publication/update_datasource/publication_10.json.gz create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/publication/update_datasource/software create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/software/dataset create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/software/otherresearchproduct create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/software/publication create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/software/software delete mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/software/software_10.json.gz create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/eosctag/galaxy/publication create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/eosctag/jupyter/publication create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/eosctag/twitter/publication diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java index 053300696..7c4186663 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java @@ -71,6 +71,9 @@ public class PropagationConstant { public static final String PROPAGATION_RESULT_COMMUNITY_ORGANIZATION_CLASS_ID = "result:community:organization"; public static final String PROPAGATION_RESULT_COMMUNITY_ORGANIZATION_CLASS_NAME = " Propagation of result belonging to community through organization"; + public static final String PROPAGATION_RESULT_COMMUNITY_PROJECT_CLASS_ID = "result:community:project"; + public static final String PROPAGATION_RESULT_COMMUNITY_PROJECT_CLASS_NAME = " Propagation of result belonging to community through project"; + public static final String PROPAGATION_ORCID_TO_RESULT_FROM_SEM_REL_CLASS_ID = "authorpid:result"; public static final String PROPAGATION_ORCID_TO_RESULT_FROM_SEM_REL_CLASS_NAME = "Propagation of authors pid to result through semantic relations"; diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java index cc615ba46..262ca0290 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java @@ -4,22 +4,18 @@ package eu.dnetlib.dhp.api; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; -import java.io.OutputStream; import java.net.HttpURLConnection; -import java.net.MalformedURLException; import java.net.URL; -import org.apache.http.HttpHeaders; import org.jetbrains.annotations.NotNull; -import com.google.gson.Gson; - /** * @author miriam.baglioni * @Date 06/10/23 */ public class QueryCommunityAPI { - private static final String baseUrl = "https://services.openaire.eu/openaire/"; + private static final String PRODUCTION_BASE_URL = "https://services.openaire.eu/openaire/"; + private static final String BETA_BASE_URL = "https://beta.services.openaire.eu/openaire/"; private static String get(String geturl) throws IOException { URL url = new URL(geturl); @@ -36,25 +32,35 @@ public class QueryCommunityAPI { return body; } - public static String communities() throws IOException { - return get(baseUrl + "community/communities"); + public static String communities(boolean production) throws IOException { + if (production) + return get(PRODUCTION_BASE_URL + "community/communities"); + return get(BETA_BASE_URL + "community/communities"); } - public static String community(String id) throws IOException { - return get(baseUrl + "community/" + id); + public static String community(String id, boolean production) throws IOException { + if (production) + return get(PRODUCTION_BASE_URL + "community/" + id); + return get(BETA_BASE_URL + "community/" + id); } - public static String communityDatasource(String id) throws IOException { - return get(baseUrl + "community/" + id + "/contentproviders"); + public static String communityDatasource(String id, boolean production) throws IOException { + if (production) + return get(PRODUCTION_BASE_URL + "community/" + id + "/contentproviders"); + return (BETA_BASE_URL + "community/" + id + "/contentproviders"); } - public static String communityPropagationOrganization(String id) throws IOException { - return get(baseUrl + "community/" + id + "/propagationOrganizations"); + public static String communityPropagationOrganization(String id, boolean production) throws IOException { + if (production) + return get(PRODUCTION_BASE_URL + "community/" + id + "/propagationOrganizations"); + return get(BETA_BASE_URL + "community/" + id + "/propagationOrganizations"); } - public static String communityProjects(String id, String page, String size) throws IOException { - return get(baseUrl + "community/" + id + "/projects/" + page + "/" + size); + public static String communityProjects(String id, String page, String size, boolean production) throws IOException { + if (production) + return get(PRODUCTION_BASE_URL + "community/" + id + "/projects/" + page + "/" + size); + return get(BETA_BASE_URL + "community/" + id + "/projects/" + page + "/" + size); } @NotNull diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java index c1aaa14c4..a0eacb774 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java @@ -30,14 +30,14 @@ public class Utils implements Serializable { private static final ObjectMapper MAPPER = new ObjectMapper(); private static final VerbResolver resolver = VerbResolverFactory.newInstance(); - public static CommunityConfiguration getCommunityConfiguration() throws IOException { + public static CommunityConfiguration getCommunityConfiguration(boolean production) throws IOException { final Map communities = Maps.newHashMap(); List validCommunities = new ArrayList<>(); - getValidCommunities() + getValidCommunities(production) .forEach(community -> { try { CommunityModel cm = MAPPER - .readValue(QueryCommunityAPI.community(community.getId()), CommunityModel.class); + .readValue(QueryCommunityAPI.community(community.getId(), production), CommunityModel.class); validCommunities.add(getCommunity(cm)); } catch (IOException e) { throw new RuntimeException(e); @@ -46,10 +46,11 @@ public class Utils implements Serializable { validCommunities.forEach(community -> { try { DatasourceList dl = MAPPER - .readValue(QueryCommunityAPI.communityDatasource(community.getId()), DatasourceList.class); + .readValue( + QueryCommunityAPI.communityDatasource(community.getId(), production), DatasourceList.class); community.setProviders(dl.stream().map(d -> { -// if(d.getEnabled() == null || Boolean.FALSE.equals(d.getEnabled())) -// return null; + if (d.getEnabled() == null || Boolean.FALSE.equals(d.getEnabled())) + return null; Provider p = new Provider(); p.setOpenaireId("10|" + d.getOpenaireId()); p.setSelectionConstraints(d.getSelectioncriteria()); @@ -80,18 +81,20 @@ public class Utils implements Serializable { c.setSubjects(cm.getSubjects()); c.getSubjects().addAll(cm.getFos()); c.getSubjects().addAll(cm.getSdg()); - c.setConstraints(cm.getAdvancedConstraints()); - if (c.getConstraints() != null) + if (cm.getAdvancedConstraints() != null) { + c.setConstraints(cm.getAdvancedConstraints()); c.getConstraints().setSelection(resolver); - c.setRemoveConstraints(cm.getRemoveConstraints()); - if (c.getRemoveConstraints() != null) + } + if (cm.getRemoveConstraints() != null) { + c.setRemoveConstraints(cm.getRemoveConstraints()); c.getRemoveConstraints().setSelection(resolver); + } return c; } - public static List getValidCommunities() throws IOException { + public static List getValidCommunities(boolean production) throws IOException { return MAPPER - .readValue(QueryCommunityAPI.communities(), CommunitySummary.class) + .readValue(QueryCommunityAPI.communities(production), CommunitySummary.class) .stream() .filter( community -> !community.getStatus().equals("hidden") && @@ -99,17 +102,26 @@ public class Utils implements Serializable { .collect(Collectors.toList()); } - public static CommunityEntityMap getCommunityOrganization() throws IOException { + /** + * it returns for each organization the list of associated communities + */ + public static CommunityEntityMap getCommunityOrganization(boolean production) throws IOException { CommunityEntityMap organizationMap = new CommunityEntityMap(); - getValidCommunities() + getValidCommunities(production) .forEach(community -> { String id = community.getId(); try { List associatedOrgs = MAPPER - .readValue(QueryCommunityAPI.communityPropagationOrganization(id), OrganizationList.class); - if (associatedOrgs.size() > 0) { - organizationMap.put(id, associatedOrgs); - } + .readValue( + QueryCommunityAPI.communityPropagationOrganization(id, production), OrganizationList.class); + associatedOrgs.forEach(o -> { + if (!organizationMap + .keySet() + .contains( + "20|" + o)) + organizationMap.put("20|" + o, new ArrayList<>()); + organizationMap.get("20|" + o).add(community.getId()); + }); } catch (IOException e) { throw new RuntimeException(e); } @@ -117,26 +129,28 @@ public class Utils implements Serializable { return organizationMap; } - public static CommunityEntityMap getCommunityProjects() throws IOException { + public static CommunityEntityMap getCommunityProjects(boolean production) throws IOException { CommunityEntityMap projectMap = new CommunityEntityMap(); - getValidCommunities() + getValidCommunities(production) .forEach(community -> { int page = -1; int size = 100; ContentModel cm = new ContentModel(); - List projectList = new ArrayList<>(); do { page++; try { cm = MAPPER .readValue( QueryCommunityAPI - .communityProjects(community.getId(), String.valueOf(page), String.valueOf(size)), + .communityProjects( + community.getId(), String.valueOf(page), String.valueOf(size), production), ContentModel.class); if (cm.getContent().size() > 0) { - - cm.getContent().forEach(p -> projectList.add("40|" + p.getOpenaireId())); - projectMap.put(community.getId(), projectList); + cm.getContent().forEach(p -> { + if (!projectMap.keySet().contains("40|" + p.getOpenaireId())) + projectMap.put("40|" + p.getOpenaireId(), new ArrayList<>()); + projectMap.get("40|" + p.getOpenaireId()).add(community.getId()); + }); } } catch (IOException e) { throw new RuntimeException(e); diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java index 0d98e4958..b24ee129a 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java @@ -23,6 +23,8 @@ import com.google.gson.Gson; import eu.dnetlib.dhp.api.Utils; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.bulktag.community.*; +import eu.dnetlib.dhp.schema.common.EntityType; +import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.oaf.Datasource; import eu.dnetlib.dhp.schema.oaf.Result; @@ -53,50 +55,38 @@ public class SparkBulkTagJob { .orElse(Boolean.TRUE); log.info("isSparkSessionManaged: {}", isSparkSessionManaged); - Boolean isTest = Optional - .ofNullable(parser.get("isTest")) - .map(Boolean::valueOf) - .orElse(Boolean.FALSE); - log.info("isTest: {} ", isTest); - final String inputPath = parser.get("sourcePath"); log.info("inputPath: {}", inputPath); final String outputPath = parser.get("outputPath"); log.info("outputPath: {}", outputPath); + final boolean production = Boolean.valueOf(parser.get("production")); + log.info("production: {}", production); + ProtoMap protoMappingParams = new Gson().fromJson(parser.get("pathMap"), ProtoMap.class); log.info("pathMap: {}", new Gson().toJson(protoMappingParams)); - final String resultClassName = parser.get("resultTableName"); - log.info("resultTableName: {}", resultClassName); - - final Boolean saveGraph = Optional - .ofNullable(parser.get("saveGraph")) - .map(Boolean::valueOf) - .orElse(Boolean.TRUE); - log.info("saveGraph: {}", saveGraph); - - Class resultClazz = (Class) Class.forName(resultClassName); - SparkConf conf = new SparkConf(); CommunityConfiguration cc; - String taggingConf = parser.get("taggingConf"); + String taggingConf = Optional + .ofNullable(parser.get("taggingConf")) + .map(String::valueOf) + .orElse(null); - if (isTest) { + if (taggingConf != null) { cc = CommunityConfigurationFactory.newInstance(taggingConf); } else { - cc = Utils.getCommunityConfiguration();// QueryInformationSystem.getCommunityConfiguration(parser.get("isLookUpUrl")); + cc = Utils.getCommunityConfiguration(production); } runWithSparkSession( conf, isSparkSessionManaged, spark -> { - removeOutputDir(spark, outputPath); extendCommunityConfigurationForEOSC(spark, inputPath, cc); - execBulkTag(spark, inputPath, outputPath, protoMappingParams, resultClazz, cc); + execBulkTag(spark, inputPath, outputPath, protoMappingParams, cc); }); } @@ -141,22 +131,30 @@ public class SparkBulkTagJob { String inputPath, String outputPath, ProtoMap protoMappingParams, - Class resultClazz, CommunityConfiguration communityConfiguration) { - ResultTagger resultTagger = new ResultTagger(); - readPath(spark, inputPath, resultClazz) - .map(patchResult(), Encoders.bean(resultClazz)) - .filter(Objects::nonNull) - .map( - (MapFunction) value -> resultTagger - .enrichContextCriteria( - value, communityConfiguration, protoMappingParams), - Encoders.bean(resultClazz)) - .write() - .mode(SaveMode.Overwrite) - .option("compression", "gzip") - .json(outputPath); + ModelSupport.entityTypes + .keySet() + .parallelStream() + .filter(e -> ModelSupport.isResult(e)) + .forEach(e -> { + removeOutputDir(spark, outputPath + e.name()); + ResultTagger resultTagger = new ResultTagger(); + Class resultClazz = ModelSupport.entityTypes.get(e); + readPath(spark, inputPath + e.name(), resultClazz) + .map(patchResult(), Encoders.bean(resultClazz)) + .filter(Objects::nonNull) + .map( + (MapFunction) value -> resultTagger + .enrichContextCriteria( + value, communityConfiguration, protoMappingParams), + Encoders.bean(resultClazz)) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(outputPath + e.name()); + }); + } public static Dataset readPath( diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/ResultTagger.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/ResultTagger.java index 5f62c10f4..98c46cf00 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/ResultTagger.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/ResultTagger.java @@ -82,19 +82,23 @@ public class ResultTagger implements Serializable { // communities contains all the communities to be not added to the context final Set removeCommunities = new HashSet<>(); + // if (conf.getRemoveConstraintsMap().keySet().size() > 0) conf .getRemoveConstraintsMap() .keySet() - .forEach(communityId -> { - if (conf.getRemoveConstraintsMap().get(communityId).getCriteria() != null && - conf - .getRemoveConstraintsMap() - .get(communityId) - .getCriteria() - .stream() - .anyMatch(crit -> crit.verifyCriteria(param))) - removeCommunities.add(communityId); - }); + .forEach( + communityId -> { + // log.info("Remove constraints for " + communityId); + if (conf.getRemoveConstraintsMap().keySet().contains(communityId) && + conf.getRemoveConstraintsMap().get(communityId).getCriteria() != null && + conf + .getRemoveConstraintsMap() + .get(communityId) + .getCriteria() + .stream() + .anyMatch(crit -> crit.verifyCriteria(param))) + removeCommunities.add(communityId); + }); // communities contains all the communities to be added as context for the result final Set communities = new HashSet<>(); @@ -124,10 +128,10 @@ public class ResultTagger implements Serializable { if (Objects.nonNull(result.getInstance())) { for (Instance i : result.getInstance()) { if (Objects.nonNull(i.getCollectedfrom()) && Objects.nonNull(i.getCollectedfrom().getKey())) { - collfrom.add(StringUtils.substringAfter(i.getCollectedfrom().getKey(), "|")); + collfrom.add(i.getCollectedfrom().getKey()); } if (Objects.nonNull(i.getHostedby()) && Objects.nonNull(i.getHostedby().getKey())) { - hostdby.add(StringUtils.substringAfter(i.getHostedby().getKey(), "|")); + hostdby.add(i.getHostedby().getKey()); } } diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunitySet.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunitySet.java index e0670b80f..e32e94a4b 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunitySet.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunitySet.java @@ -48,11 +48,10 @@ public class PrepareResultCommunitySet { final String outputPath = parser.get("outputPath"); log.info("outputPath: {}", outputPath); -// final CommunityEntityMap organizationMap = new Gson() -// .fromJson( -// parser.get("organizationtoresultcommunitymap"), -// CommunityEntityMap.class); - final CommunityEntityMap organizationMap = Utils.getCommunityOrganization(); + final boolean production = Boolean.valueOf(parser.get("production")); + log.info("production: {}", production); + + final CommunityEntityMap organizationMap = Utils.getCommunityOrganization(production); log.info("organizationMap: {}", new Gson().toJson(organizationMap)); SparkConf conf = new SparkConf(); diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/SparkResultToCommunityFromOrganizationJob.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/SparkResultToCommunityFromOrganizationJob.java index 50df08f8c..43f425b68 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/SparkResultToCommunityFromOrganizationJob.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/SparkResultToCommunityFromOrganizationJob.java @@ -56,12 +56,6 @@ public class SparkResultToCommunityFromOrganizationJob { final String resultClassName = parser.get("resultTableName"); log.info("resultTableName: {}", resultClassName); - final Boolean saveGraph = Optional - .ofNullable(parser.get("saveGraph")) - .map(Boolean::valueOf) - .orElse(Boolean.TRUE); - log.info("saveGraph: {}", saveGraph); - @SuppressWarnings("unchecked") Class resultClazz = (Class) Class.forName(resultClassName); @@ -73,9 +67,9 @@ public class SparkResultToCommunityFromOrganizationJob { isSparkSessionManaged, spark -> { removeOutputDir(spark, outputPath); - if (saveGraph) { - execPropagation(spark, inputPath, outputPath, resultClazz, possibleupdatespath); - } + + execPropagation(spark, inputPath, outputPath, resultClazz, possibleupdatespath); + }); } diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java new file mode 100644 index 000000000..b0fbf8056 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java @@ -0,0 +1,121 @@ + +package eu.dnetlib.dhp.resulttocommunityfromproject; + +import static eu.dnetlib.dhp.PropagationConstant.*; +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession; + +import java.util.*; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.io.compress.GzipCodec; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.api.java.function.MapGroupsFunction; +import org.apache.spark.sql.*; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.gson.Gson; + +import eu.dnetlib.dhp.api.Utils; +import eu.dnetlib.dhp.api.model.CommunityEntityMap; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.resulttocommunityfromorganization.ResultCommunityList; +import eu.dnetlib.dhp.resulttocommunityfromorganization.ResultOrganizations; +import eu.dnetlib.dhp.schema.common.ModelConstants; +import eu.dnetlib.dhp.schema.oaf.Relation; +import scala.Tuple2; + +public class PrepareResultCommunitySet { + + private static final Logger log = LoggerFactory.getLogger(PrepareResultCommunitySet.class); + + public static void main(String[] args) throws Exception { + String jsonConfiguration = IOUtils + .toString( + PrepareResultCommunitySet.class + .getResourceAsStream( + "/eu/dnetlib/dhp/resulttocommunityfromproject/input_preparecommunitytoresult_parameters.json")); + + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + parser.parseArgument(args); + + Boolean isSparkSessionManaged = isSparkSessionManaged(parser); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + String inputPath = parser.get("sourcePath"); + log.info("inputPath: {}", inputPath); + + final String outputPath = parser.get("outputPath"); + log.info("outputPath: {}", outputPath); + + final boolean production = Boolean.valueOf(parser.get("outputPath")); + log.info("production: {}", production); + + final CommunityEntityMap projectsMap = Utils.getCommunityProjects(production); + log.info("projectsMap: {}", new Gson().toJson(projectsMap)); + + SparkConf conf = new SparkConf(); + conf.set("hive.metastore.uris", parser.get("hive_metastore_uris")); + + runWithSparkHiveSession( + conf, + isSparkSessionManaged, + spark -> { + removeOutputDir(spark, outputPath); + prepareInfo(spark, inputPath, outputPath, projectsMap); + }); + } + + private static void prepareInfo( + SparkSession spark, + String inputPath, + String outputPath, + CommunityEntityMap projectMap) { + + final StructType structureSchema = new StructType() + .add( + "dataInfo", new StructType() + .add("deletedbyinference", DataTypes.BooleanType) + .add("invisible", DataTypes.BooleanType)) + .add("source", DataTypes.StringType) + .add("target", DataTypes.StringType) + .add("relClass", DataTypes.StringType); + + spark + .read() + .schema(structureSchema) + .json(inputPath) + .filter( + "dataInfo.deletedbyinference != true " + + "and relClass == '" + ModelConstants.IS_PRODUCED_BY + "'") + .select( + new Column("source").as("resultId"), + new Column("target").as("projectId")) + .groupByKey((MapFunction) r -> (String) r.getAs("source"), Encoders.STRING()) + .mapGroups((MapGroupsFunction) (k, v) -> { + ResultProjectList rpl = new ResultProjectList(); + rpl.setResultId(k); + ArrayList cl = new ArrayList<>(); + cl.addAll(projectMap.get(v.next().getAs("target"))); + v.forEachRemaining(r -> { + projectMap + .get(r.getAs("target")) + .forEach(c -> { + if (!cl.contains(c)) + cl.add(c); + }); + + }); + rpl.setCommunityList(cl); + return rpl; + }, Encoders.bean(ResultProjectList.class)) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(outputPath); + } + +} diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultProjectList.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultProjectList.java new file mode 100644 index 000000000..44798a1f3 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultProjectList.java @@ -0,0 +1,26 @@ + +package eu.dnetlib.dhp.resulttocommunityfromproject; + +import java.io.Serializable; +import java.util.ArrayList; + +public class ResultProjectList implements Serializable { + private String resultId; + private ArrayList communityList; + + public String getResultId() { + return resultId; + } + + public void setResultId(String resultId) { + this.resultId = resultId; + } + + public ArrayList getCommunityList() { + return communityList; + } + + public void setCommunityList(ArrayList communityList) { + this.communityList = communityList; + } +} diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java new file mode 100644 index 000000000..6d4779ea1 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java @@ -0,0 +1,156 @@ + +package eu.dnetlib.dhp.resulttocommunityfromproject; + +import static eu.dnetlib.dhp.PropagationConstant.*; +import static eu.dnetlib.dhp.PropagationConstant.PROPAGATION_RESULT_COMMUNITY_ORGANIZATION_CLASS_NAME; +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.apache.commons.io.IOUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SaveMode; +import org.apache.spark.sql.SparkSession; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.resulttocommunityfromorganization.ResultCommunityList; +import eu.dnetlib.dhp.resulttocommunityfromorganization.SparkResultToCommunityFromOrganizationJob; +import eu.dnetlib.dhp.schema.common.ModelConstants; +import eu.dnetlib.dhp.schema.common.ModelSupport; +import eu.dnetlib.dhp.schema.oaf.Context; +import eu.dnetlib.dhp.schema.oaf.Result; +import scala.Tuple2; + +/** + * @author miriam.baglioni + * @Date 11/10/23 + */ +public class SparkResultToCommunityFromProject implements Serializable { + private static final Logger log = LoggerFactory.getLogger(SparkResultToCommunityFromProject.class); + + public static void main(String[] args) throws Exception { + String jsonConfiguration = IOUtils + .toString( + SparkResultToCommunityFromProject.class + .getResourceAsStream( + "/eu/dnetlib/dhp/resulttocommunityfromproject/input_communitytoresult_parameters.json")); + + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + + parser.parseArgument(args); + + Boolean isSparkSessionManaged = isSparkSessionManaged(parser); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + String inputPath = parser.get("sourcePath"); + log.info("inputPath: {}", inputPath); + + final String outputPath = parser.get("outputPath"); + log.info("outputPath: {}", outputPath); + + final String possibleupdatespath = parser.get("preparedInfoPath"); + log.info("preparedInfoPath: {}", possibleupdatespath); + + final String resultClassName = parser.get("resultTableName"); + log.info("resultTableName: {}", resultClassName); + + @SuppressWarnings("unchecked") + Class resultClazz = (Class) Class.forName(resultClassName); + + SparkConf conf = new SparkConf(); + conf.set("hive.metastore.uris", parser.get("hive_metastore_uris")); + + runWithSparkHiveSession( + conf, + isSparkSessionManaged, + spark -> { +// removeOutputDir(spark, outputPath); + + execPropagation(spark, inputPath, outputPath, possibleupdatespath); + + }); + } + + private static void execPropagation( + SparkSession spark, + String inputPath, + String outputPath, + + String possibleUpdatesPath) { + + Dataset possibleUpdates = readPath(spark, possibleUpdatesPath, ResultProjectList.class); + + ModelSupport.entityTypes + .keySet() + .parallelStream() + .forEach(e -> { + if (ModelSupport.isResult(e)) { + removeOutputDir(spark, outputPath + e.name()); + Class resultClazz = ModelSupport.entityTypes.get(e); + Dataset result = readPath(spark, inputPath + e.name(), resultClazz); + + result + .joinWith( + possibleUpdates, + result.col("id").equalTo(possibleUpdates.col("resultId")), + "left_outer") + .map(resultCommunityFn(), Encoders.bean(resultClazz)) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(outputPath); + } + }); + + } + + private static MapFunction, R> resultCommunityFn() { + return value -> { + R ret = value._1(); + Optional rcl = Optional.ofNullable(value._2()); + if (rcl.isPresent()) { + ArrayList communitySet = rcl.get().getCommunityList(); + List contextList = ret + .getContext() + .stream() + .map(Context::getId) + .collect(Collectors.toList()); + + @SuppressWarnings("unchecked") + R res = (R) ret.getClass().newInstance(); + + res.setId(ret.getId()); + List propagatedContexts = new ArrayList<>(); + for (String cId : communitySet) { + if (!contextList.contains(cId)) { + Context newContext = new Context(); + newContext.setId(cId); + newContext + .setDataInfo( + Arrays + .asList( + getDataInfo( + PROPAGATION_DATA_INFO_TYPE, + PROPAGATION_RESULT_COMMUNITY_PROJECT_CLASS_ID, + PROPAGATION_RESULT_COMMUNITY_PROJECT_CLASS_NAME, + ModelConstants.DNET_PROVENANCE_ACTIONS))); + propagatedContexts.add(newContext); + } + } + res.setContext(propagatedContexts); + ret.mergeFrom(res); + } + return ret; + }; + } +} diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/input_bulkTag_parameters.json b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/input_bulkTag_parameters.json index a8be7c32e..dbe2d088f 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/input_bulkTag_parameters.json +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/input_bulkTag_parameters.json @@ -1,10 +1,5 @@ [ - { - "paramName":"is", - "paramLongName":"isLookUpUrl", - "paramDescription": "URL of the isLookUp Service", - "paramRequired": true - }, + { "paramName":"s", "paramLongName":"sourcePath", @@ -17,12 +12,7 @@ "paramDescription": "the json path associated to each selection field", "paramRequired": true }, - { - "paramName":"tn", - "paramLongName":"resultTableName", - "paramDescription": "the name of the result table we are currently working on", - "paramRequired": true - }, + { "paramName": "out", "paramLongName": "outputPath", @@ -35,17 +25,19 @@ "paramDescription": "true if the spark session is managed, false otherwise", "paramRequired": false }, - { - "paramName": "test", - "paramLongName": "isTest", - "paramDescription": "Parameter intended for testing purposes only. True if the reun is relatesd to a test and so the taggingConf parameter should be loaded", - "paramRequired": false - }, + { "paramName": "tg", "paramLongName": "taggingConf", "paramDescription": "this parameter is intended for testing purposes only. It is a possible tagging configuration obtained via the XQUERY. Intended to be removed", "paramRequired": false + }, + + { + "paramName": "p", + "paramLongName": "production", + "paramDescription": "this parameter is intended for testing purposes only. It is a possible tagging configuration obtained via the XQUERY. Intended to be removed", + "paramRequired": true } ] \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml index b868e4c72..56eed4703 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml @@ -4,10 +4,6 @@ sourcePath the source path - - isLookUpUrl - the isLookup service endpoint - pathMap the json path associated to each selection field @@ -44,7 +40,7 @@ - + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] @@ -102,16 +98,9 @@ - + - - - - - - - - + yarn-cluster cluster @@ -128,98 +117,15 @@ --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --sourcePath${sourcePath}/publication - --resultTableNameeu.dnetlib.dhp.schema.oaf.Publication - --outputPath${outputPath}/publication + --sourcePath${sourcePath}/ + --outputPath${outputPath}/ --pathMap${pathMap} - --isLookUpUrl${isLookUpUrl} + --production${production} - + - - - yarn-cluster - cluster - bulkTagging-dataset - eu.dnetlib.dhp.bulktag.SparkBulkTagJob - dhp-enrichment-${projectVersion}.jar - - --num-executors=${sparkExecutorNumber} - --executor-memory=${sparkExecutorMemory} - --executor-cores=${sparkExecutorCores} - --driver-memory=${sparkDriverMemory} - --conf spark.extraListeners=${spark2ExtraListeners} - --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} - --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} - --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - - --sourcePath${sourcePath}/dataset - --resultTableNameeu.dnetlib.dhp.schema.oaf.Dataset - --outputPath${outputPath}/dataset - --pathMap${pathMap} - --isLookUpUrl${isLookUpUrl} - - - - - - - - yarn-cluster - cluster - bulkTagging-orp - eu.dnetlib.dhp.bulktag.SparkBulkTagJob - dhp-enrichment-${projectVersion}.jar - - --num-executors=${sparkExecutorNumber} - --executor-memory=${sparkExecutorMemory} - --executor-cores=${sparkExecutorCores} - --driver-memory=${sparkDriverMemory} - --conf spark.extraListeners=${spark2ExtraListeners} - --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} - --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} - --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - - --sourcePath${sourcePath}/otherresearchproduct - --resultTableNameeu.dnetlib.dhp.schema.oaf.OtherResearchProduct - --outputPath${outputPath}/otherresearchproduct - --pathMap${pathMap} - --isLookUpUrl${isLookUpUrl} - - - - - - - - yarn-cluster - cluster - bulkTagging-software - eu.dnetlib.dhp.bulktag.SparkBulkTagJob - dhp-enrichment-${projectVersion}.jar - - --num-executors=${sparkExecutorNumber} - --executor-memory=${sparkExecutorMemory} - --executor-cores=${sparkExecutorCores} - --driver-memory=${sparkDriverMemory} - --conf spark.extraListeners=${spark2ExtraListeners} - --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} - --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} - --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - - --sourcePath${sourcePath}/software - --resultTableNameeu.dnetlib.dhp.schema.oaf.Software - --outputPath${outputPath}/software - --pathMap${pathMap} - --isLookUpUrl${isLookUpUrl} - - - - - - diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/query.xq b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/query.xq deleted file mode 100644 index 052037efb..000000000 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/query.xq +++ /dev/null @@ -1,62 +0,0 @@ -for $x in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType') -let $subj := $x//CONFIGURATION/context/param[./@name='subject']/text() -let $datasources := $x//CONFIGURATION/context/category[./@id=concat($x//CONFIGURATION/context/@id,'::contentproviders')]/concept -let $organizations := $x//CONFIGURATION/context/category[./@id=concat($x//CONFIGURATION/context/@id,'::resultorganizations')]/concept -let $communities := $x//CONFIGURATION/context/category[./@id=concat($x//CONFIGURATION/context/@id,'::zenodocommunities')]/concept -let $fos := $x//CONFIGURATION/context/param[./@name='fos']/text() -let $sdg := $x//CONFIGURATION/context/param[./@name='sdg']/text() -let $zenodo := $x//param[./@name='zenodoCommunity']/text() -where $x//CONFIGURATION/context[./@type='community' or ./@type='ri'] and $x//context/param[./@name = 'status']/text() != 'hidden' -return - -{ $x//CONFIGURATION/context/@id} - -{$x//CONFIGURATION/context/param[./@name='removeConstraints']/text() } - - -{$x//CONFIGURATION/context/param[./@name='advancedConstraints']/text() } - - - {for $y in tokenize($subj,',') - return - {$y}} - {for $y in tokenize($fos,',') - return - {$y}} - {for $y in tokenize($sdg,',') - return - {$y}} - - - {for $d in $datasources - where $d/param[./@name='enabled']/text()='true' - return - - - {$d//param[./@name='openaireId']/text()} - - - {$d/param[./@name='selcriteria']/text()} - - } - - -{for $zc in $zenodo -return - - -{$zc} - -} -{for $zc in $communities -return - - -{$zc/param[./@name='zenodoid']/text()} - - -{$zc/param[./@name='selcriteria']/text()} - -} - - \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/input_communitytoresult_parameters.json b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/input_communitytoresult_parameters.json index eebc1a0ca..b6eb309a5 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/input_communitytoresult_parameters.json +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/input_communitytoresult_parameters.json @@ -11,18 +11,6 @@ "paramDescription": "the hive metastore uris", "paramRequired": true }, - { - "paramName":"sg", - "paramLongName":"saveGraph", - "paramDescription": "true if the new version of the graph must be saved", - "paramRequired": false - }, - { - "paramName":"test", - "paramLongName":"isTest", - "paramDescription": "true if it is executing a test", - "paramRequired": false - }, { "paramName": "out", "paramLongName": "outputPath", diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml index d9805ab7b..b25822ad0 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml @@ -4,10 +4,7 @@ sourcePath the source path - - - - + outputPath the output path @@ -106,7 +103,7 @@ --sourcePath${sourcePath}/relation --outputPath${workingDir}/preparedInfo/resultCommunityList --hive_metastore_uris${hive_metastore_uris} - --organizationtoresultcommunitymap${organizationtoresultcommunitymap} + diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java index 11dad9055..7cbbcaafb 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java @@ -6,6 +6,7 @@ import static eu.dnetlib.dhp.bulktag.community.TaggingConstants.ZENODO_COMMUNITY import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.HashMap; import java.util.List; import org.apache.commons.io.FileUtils; @@ -98,14 +99,11 @@ public class BulkTagJobTest { SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", - getClass().getResource("/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates").getPath(), + getClass().getResource("/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates/").getPath(), "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Dataset", - "-outputPath", workingDir.toString() + "/dataset", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + "-outputPath", workingDir.toString() + "/", "-pathMap", pathMap }); @@ -133,19 +131,16 @@ public class BulkTagJobTest { @Test void bulktagBySubjectNoPreviousContextTest() throws Exception { final String sourcePath = getClass() - .getResource("/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/nocontext") + .getResource("/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/nocontext/") .getPath(); final String pathMap = BulkTagJobTest.pathMap; SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Dataset", - "-outputPath", workingDir.toString() + "/dataset", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + "-outputPath", workingDir.toString() + "/", "-pathMap", pathMap }); @@ -230,19 +225,19 @@ public class BulkTagJobTest { void bulktagBySubjectPreviousContextNoProvenanceTest() throws Exception { final String sourcePath = getClass() .getResource( - "/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/contextnoprovenance") + "/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/contextnoprovenance/") .getPath(); final String pathMap = BulkTagJobTest.pathMap; SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Dataset", - "-outputPath", workingDir.toString() + "/dataset", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -311,18 +306,18 @@ public class BulkTagJobTest { @Test void bulktagByDatasourceTest() throws Exception { final String sourcePath = getClass() - .getResource("/eu/dnetlib/dhp/bulktag/sample/publication/update_datasource") + .getResource("/eu/dnetlib/dhp/bulktag/sample/publication/update_datasource/") .getPath(); SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Publication", - "-outputPath", workingDir.toString() + "/publication", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -384,25 +379,25 @@ public class BulkTagJobTest { void bulktagByZenodoCommunityTest() throws Exception { final String sourcePath = getClass() .getResource( - "/eu/dnetlib/dhp/bulktag/sample/otherresearchproduct/update_zenodocommunity") + "/eu/dnetlib/dhp/bulktag/sample/otherresearchproduct/update_zenodocommunity/") .getPath(); SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.OtherResearchProduct", - "-outputPath", workingDir.toString() + "/orp", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); JavaRDD tmp = sc - .textFile(workingDir.toString() + "/orp") + .textFile(workingDir.toString() + "/otherresearchproduct") .map(item -> OBJECT_MAPPER.readValue(item, OtherResearchProduct.class)); Assertions.assertEquals(10, tmp.count()); @@ -505,18 +500,18 @@ public class BulkTagJobTest { @Test void bulktagBySubjectDatasourceTest() throws Exception { final String sourcePath = getClass() - .getResource("/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject_datasource") + .getResource("/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject_datasource/") .getPath(); SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Dataset", - "-outputPath", workingDir.toString() + "/dataset", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -636,14 +631,14 @@ public class BulkTagJobTest { SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", - getClass().getResource("/eu/dnetlib/dhp/bulktag/sample/software/software_10.json.gz").getPath(), + getClass().getResource("/eu/dnetlib/dhp/bulktag/sample/software/").getPath(), "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Software", - "-outputPath", workingDir.toString() + "/software", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -732,18 +727,18 @@ public class BulkTagJobTest { final String sourcePath = getClass() .getResource( - "/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints") + "/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints/") .getPath(); SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Dataset", - "-outputPath", workingDir.toString() + "/dataset", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -774,19 +769,19 @@ public class BulkTagJobTest { void bulkTagOtherJupyter() throws Exception { final String sourcePath = getClass() .getResource( - "/eu/dnetlib/dhp/eosctag/jupyter/otherresearchproduct") + "/eu/dnetlib/dhp/eosctag/jupyter/") .getPath(); SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.OtherResearchProduct", - "-outputPath", workingDir.toString() + "/otherresearchproduct", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -829,18 +824,18 @@ public class BulkTagJobTest { public void bulkTagDatasetJupyter() throws Exception { final String sourcePath = getClass() .getResource( - "/eu/dnetlib/dhp/eosctag/jupyter/dataset") + "/eu/dnetlib/dhp/eosctag/jupyter/") .getPath(); SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Dataset", - "-outputPath", workingDir.toString() + "/dataset", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -878,18 +873,18 @@ public class BulkTagJobTest { final String sourcePath = getClass() .getResource( - "/eu/dnetlib/dhp/eosctag/jupyter/software") + "/eu/dnetlib/dhp/eosctag/jupyter/") .getPath(); SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Software", - "-outputPath", workingDir.toString() + "/software", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -1096,18 +1091,18 @@ public class BulkTagJobTest { void galaxyOtherTest() throws Exception { final String sourcePath = getClass() .getResource( - "/eu/dnetlib/dhp/eosctag/galaxy/otherresearchproduct") + "/eu/dnetlib/dhp/eosctag/galaxy/") .getPath(); SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.OtherResearchProduct", - "-outputPath", workingDir.toString() + "/otherresearchproduct", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -1214,18 +1209,18 @@ public class BulkTagJobTest { void galaxySoftwareTest() throws Exception { final String sourcePath = getClass() .getResource( - "/eu/dnetlib/dhp/eosctag/galaxy/software") + "/eu/dnetlib/dhp/eosctag/galaxy/") .getPath(); SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Software", - "-outputPath", workingDir.toString() + "/software", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -1333,19 +1328,19 @@ public class BulkTagJobTest { void twitterDatasetTest() throws Exception { final String sourcePath = getClass() .getResource( - "/eu/dnetlib/dhp/eosctag/twitter/dataset") + "/eu/dnetlib/dhp/eosctag/twitter/") .getPath(); SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Dataset", - "-outputPath", workingDir.toString() + "/dataset", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -1373,19 +1368,19 @@ public class BulkTagJobTest { void twitterOtherTest() throws Exception { final String sourcePath = getClass() .getResource( - "/eu/dnetlib/dhp/eosctag/twitter/otherresearchproduct") + "/eu/dnetlib/dhp/eosctag/twitter/") .getPath(); SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.OtherResearchProduct", - "-outputPath", workingDir.toString() + "/otherresearchproduct", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -1418,19 +1413,19 @@ public class BulkTagJobTest { void twitterSoftwareTest() throws Exception { final String sourcePath = getClass() .getResource( - "/eu/dnetlib/dhp/eosctag/twitter/software") + "/eu/dnetlib/dhp/eosctag/twitter/") .getPath(); SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Software", - "-outputPath", workingDir.toString() + "/software", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -1455,19 +1450,19 @@ public class BulkTagJobTest { void EoscContextTagTest() throws Exception { final String sourcePath = getClass() .getResource( - "/eu/dnetlib/dhp/bulktag/eosc/dataset/dataset_10.json") + "/eu/dnetlib/dhp/bulktag/eosc/dataset/") .getPath(); SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Dataset", - "-outputPath", workingDir.toString() + "/dataset", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -1533,16 +1528,16 @@ public class BulkTagJobTest { SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", getClass() - .getResource("/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints") + .getResource("/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints/") .getPath(), "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Dataset", - "-outputPath", workingDir.toString() + "/dataset", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); @@ -1574,14 +1569,14 @@ public class BulkTagJobTest { SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", - getClass().getResource("/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates").getPath(), + getClass().getResource("/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates/").getPath(), "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Dataset", - "-outputPath", workingDir.toString() + "/dataset", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-production", Boolean.TRUE.toString(), "-pathMap", pathMap }); diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/QueryCommunityAPITest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/QueryCommunityAPITest.java index b0043d1e8..a0083dab8 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/QueryCommunityAPITest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/QueryCommunityAPITest.java @@ -26,7 +26,7 @@ public class QueryCommunityAPITest { @Test void communityList() throws Exception { - String body = QueryCommunityAPI.communities(); + String body = QueryCommunityAPI.communities(true); new ObjectMapper() .readValue(body, CommunitySummary.class) .forEach(p -> { @@ -41,7 +41,7 @@ public class QueryCommunityAPITest { @Test void community() throws Exception { String id = "dh-ch"; - String body = QueryCommunityAPI.community(id); + String body = QueryCommunityAPI.community(id, true); System.out .println( new ObjectMapper() @@ -53,7 +53,7 @@ public class QueryCommunityAPITest { @Test void communityDatasource() throws Exception { String id = "dh-ch"; - String body = QueryCommunityAPI.communityDatasource(id); + String body = QueryCommunityAPI.communityDatasource(id, true); new ObjectMapper() .readValue(body, DatasourceList.class) .forEach(ds -> { @@ -68,7 +68,7 @@ public class QueryCommunityAPITest { @Test void validCommunities() throws Exception { - CommunityConfiguration cc = Utils.getCommunityConfiguration(); + CommunityConfiguration cc = Utils.getCommunityConfiguration(true); System.out.println(cc.getCommunities().keySet()); Community community = cc.getCommunities().get("aurora"); Assertions.assertEquals(0, community.getSubjects().size()); @@ -84,11 +84,20 @@ public class QueryCommunityAPITest { Assertions .assertEquals( 35, community.getProviders().stream().filter(p -> p.getSelectionConstraints() == null).count()); + + } + + @Test + void eutopiaCommunityConfiguration() throws Exception { + CommunityConfiguration cc = Utils.getCommunityConfiguration(true); + System.out.println(cc.getCommunities().keySet()); + Community community = cc.getCommunities().get("eutopia"); + community.getProviders().forEach(p -> System.out.println(p.getOpenaireId())); } @Test void getCommunityProjects() throws Exception { - CommunityEntityMap projectMap = Utils.getCommunityProjects(); + CommunityEntityMap projectMap = Utils.getCommunityProjects(true); Assertions.assertFalse(projectMap.containsKey("mes")); Assertions.assertEquals(33, projectMap.size()); Assertions diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromorganization/ResultToCommunityJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromorganization/ResultToCommunityJobTest.java index 4dd8b976c..da3be8b35 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromorganization/ResultToCommunityJobTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromorganization/ResultToCommunityJobTest.java @@ -78,7 +78,7 @@ public class ResultToCommunityJobTest { .getResource("/eu/dnetlib/dhp/resulttocommunityfromorganization/sample") .getPath(), "-hive_metastore_uris", "", - "-saveGraph", "true", + "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Dataset", "-outputPath", workingDir.toString() + "/dataset", "-preparedInfoPath", preparedInfoPath diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/eosc/dataset/dataset_10.json b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/eosc/dataset/dataset similarity index 100% rename from dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/eosc/dataset/dataset_10.json rename to dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/eosc/dataset/dataset diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/eosc/dataset/otherresearchproduct b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/eosc/dataset/otherresearchproduct new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/eosc/dataset/publication b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/eosc/dataset/publication new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/eosc/dataset/software b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/eosc/dataset/software new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates/dataset b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates/dataset new file mode 100644 index 000000000..b3ac15d4c --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates/dataset @@ -0,0 +1,10 @@ +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055868909,"id":"50|od______3989::02dd5d2c222191b0b9bd4f33c8e96529","originalId":["od______3989::02dd5d2c222191b0b9bd4f33c8e96529"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[{"value":"10.4185/RLCS-2018-1243","qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofcollection":"2020-03-03T13:05:26.091Z","dateoftransformation":"2020-03-03T13:06:53.161Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:26.091Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/562","datestamp":"2018-01-23T15:06:07Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Gallardo-Camacho, Jorge","name":"Jorge","surname":"Gallardo-Camacho","rank":1,"pid":null,"affiliation":null},{"fullname":"Trujillo Fernández, José Ramón","name":"José Ramón","surname":"Trujillo Fernández","rank":2,"pid":null,"affiliation":null},{"fullname":"Jorge Alonso, Ana","name":"Ana","surname":"Jorge Alonso","rank":3,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Comunicación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"yihadismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"viralidad","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"vídeo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"propaganda","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"terrorismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"terrorism","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"jihadism","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"virality","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"video","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"5903 Ideologías Políticas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"6310.13 Terrorismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"El individualismo como estrategia","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Individualism as an improvised strategy","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Se analiza como el Yihadismo distribuye su mensaje propaganístico en You Tube.\nUniversidad Camilo José Cela","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"MediaLab - UCJC","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/562"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055868954,"id":"50|od______3989::05d8c751462f9bb8d2b06956dfbc5c7b","originalId":["od______3989::05d8c751462f9bb8d2b06956dfbc5c7b"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:26.079Z","dateoftransformation":"2020-03-03T13:06:53.239Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:26.079Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/227","datestamp":"2017-11-15T16:29:55Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Rico Pérez, Marta","name":"Marta","surname":"Rico Pérez","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Actividad Física y del Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Eventos deportivos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Retos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"I Congreso Nuevos Retos en los Eventos Deportivos","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"I Congreso Nuevos Retos en los Eventos Deportivos, celebrado en la Universidad Camilo José Cela.\n\n\n\nDerivado del proyecto de investigación \"La situación de los eventos deportivos en España a través de su estructuras y de sus herramientas\", tiene como finalidad conocer la situación de los eventos deportivos en España. Se realizó un estudio descriptivo holístico en el que se investigaron elementos culturales, el perfil de los recursos humanos, la estructura de los comités, entre otras.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2015-06-20","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/227"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2015-06-20","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055869097,"id":"50|od______3989::0f89464c4ac4c398fe0c71433b175a62","originalId":["od______3989::0f89464c4ac4c398fe0c71433b175a62"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:26.069Z","dateoftransformation":"2020-03-03T13:06:53.54Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:26.069Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/802","datestamp":"2018-11-14T15:30:47Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Ciencias de la Actividad Física y del Deporte","name":"Ciencias La Actividad Física Y. Del Deporte","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Actividad Física y del Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Actividades deportivas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Impacto físico","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Proyectos de investigación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"2411.06 Fisiología del Ejercicio","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Investigaciones en Ciencias de la Actividad Física y del Deporte CCAFD - UCJC","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Blanca Romero, investigadora y docente de Ciencias de la Actividad Física y del Deporte en la Universidad Camilo José Cela, describe dos de las líneas de investigación que se han desarrollado en el departamento CCAFD de UCJC: el impacto físico de las diferentes actividades deportivas y los métodos de recuperación más eficaces tras el ejercicio físico.\nBlanca también destaca la importancia en el ámbito de la salud los estudios en Ciencias de la Actividad Física y del Deporte.\nCiencias de la Actividad Física y del Deporte en la Universidad Camilo José Cela","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2016-03-21","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Universidad Camilo José Cela, Ciencias de la Actividad Física y del Deporte","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/802"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2016-03-21","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055869215,"id":"50|od______3989::1729c3988199b95d1d566851af7d3c55","originalId":["od______3989::1729c3988199b95d1d566851af7d3c55"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:25.653Z","dateoftransformation":"2020-03-03T13:06:53.764Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:25.653Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/801","datestamp":"2018-11-14T15:19:38Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Ciencias de la Actividad Física y del Deporte","name":"Ciencias La Actividad Física Y. Del Deporte","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Actividad Física y del Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Prevención","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Hidratación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deportistas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Electrolitos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deportes de resistencia","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"2411.06 Fisiología del Ejercicio","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"La importancia de la hidratación en competiciones resistencia, por Juan del Coso","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Juan del Coso, investigador y profesor en el Grado en Ciencias de la Actividad Física y del Deporte UCJC, presenta el proyecto de investigación que ha realizado el departamento CCAFD de UCJC: la prevención de desequilibrios de agua y electrolitos que se dan en los deportes de resistencia para que los deportistas realicen su actividad de una manera más segura.\nMás información: http://www.ucjc.edu/2016/03/investiga...\nUniversidad Camilo José Cela, Ciencias de la Actividad Física y del Deporte","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2016-03-21","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Grado en Ciencias de la Actividad Física y del Deporte UCJC","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/801"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2016-03-21","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055869487,"id":"50|od______3989::2e3f34ce90520fae350a7e1148d7dcea","originalId":["od______3989::2e3f34ce90520fae350a7e1148d7dcea"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.912Z","dateoftransformation":"2020-03-03T13:06:54.218Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.912Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/800","datestamp":"2018-11-14T13:43:21Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Actividad Física y del Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Cafeína","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Efectos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Alto rendimiento","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Competiciones","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Beneficios","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"2411.06 Fisiología del Ejercicio","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Investigación - Efectos positivos y negativos del uso de la cafeína en el deporte","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Proyecto de investigación que se están desarrollando en la Universidad Camilo José Cela (UCJC) sobre la cafeína y sus efectos cuando se realiza deporte. https://www.ucjc.edu/2016/08/la-bbc-s...\n\nJuan Del Coso, director del Laboratorio de Fisiología del Ejercicio de la Universidad Camilo José Cela, nos describe cómo está siendo la investigación que desarrolla con deportistas de alto rendimiento y el consumo de cafeína.\n\n\"Nuestra investigación se enmarca dentro del campo de la fisiología del ejercicio y el objetivo que tenemos es investigar los efectos que tiene la cafeína en los deportistas, principalmente saber si es una ayuda ergogénica en la mayor parte de los deportes, pero también conocer qué perjuicios existen para aquellos deportistas que la utilizan para incrementar el rendimiento deportivo\", explica Del Coso.\n\nY es que los datos son muy llamativos: tres de cada cuatro deportistas de alto rendimiento utilizan la cafeína antes de la competición y muchas veces se utiliza sin saber los perjuicios que puede tener para el deportista. \"Lo que estamos investigando es cómo una sustancia, que te puede hacer rendir mejor, cómo puede afectar negativamente cuando termina una competición\".\n\nPero esta investigación no sólo puede extraer conclusiones interesantes y útiles para los deportistas profesionales, \"investigando el deporte de alto rendimiento podemos conocer los beneficios y perjuicios de la cafeína que también afectarían al deportista amateur y hacer un deporte en general más seguro\".\n\nPara el desarrollo de este proyecto de investigación están colaborando el Laboratorio de Fisiología del Ejercicio de la Universidad Camilo José Cela y la Agencia para la Protección de la Salud en el Deporte.\nPara el desarrollo de este proyecto de investigación están colaborando el Laboratorio de Fisiología del Ejercicio de la Universidad Camilo José Cela y la Agencia para la Protección de la Salud en el Deporte.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2017-10-06","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Universidad Camilo José Cela, Vicerrectorado de Innovación","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/800"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2017-10-06","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055869498,"id":"50|od______3989::2f4f3c820c450bd08dac08d07cc82dcf","originalId":["od______3989::2f4f3c820c450bd08dac08d07cc82dcf"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.806Z","dateoftransformation":"2020-03-03T13:06:54.229Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.806Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/797","datestamp":"2018-11-14T12:12:23Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Cuesta Cano, Laura","name":"Laura","surname":"Cuesta Cano","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Comunicación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Marketing digital","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Medios sociales","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Marcas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Empresas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Emprendedores","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"II Liga de Debate CICAE - UCJC","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"6114.13 Marketing","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Laura Cuesta - Liga de Debate CICAE - UCJC","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Laura Cuesta, profesora en la Universidad Camilo José Cela (UCJC) y Especialista en Marketing Digital y Medios Sociales para marcas y emprendedores. \n\nEn esta ocasión, Laura Cuesta, nos explica el significado de redes sociales y nos habla sobre el uso que le dan los usuarios.\n\nII Liga de Debate CICAE - UCJC http://ligacicae.ucjc.edu/\n\nLa Universidad Camilo José Cela (UCJC) y la Asociación de Colegios Privados e Independientes –Círculo de Calidad Educativa (CICAE) – organizan la segunda edición de esta gran liga de debate académico. El formato es al estilo inglés, similar al de la Oxford Union, donde algunos jóvenes se reunían a debatir sobre temas de la más candente actualidad y polemizaban con la intención de formarse en tan valoradas habilidades.\n\nLa Liga de Debate CICAE-UCJC es una ocasión única para los alumnos de ponerse en contacto con herramientas tan útiles como la oratoria y la comunicación efectiva. Pero, sobre ","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-10-23","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Universidad Camilo José Cela, Vicerrectorado de Innovación","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/797"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-10-23","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055870582,"id":"50|od______3989::752fd0b2bbac1ea1cc50e52fd46eb663","originalId":["od______3989::752fd0b2bbac1ea1cc50e52fd46eb663"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.561Z","dateoftransformation":"2020-03-03T13:06:55.716Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.561Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/798","datestamp":"2018-11-14T12:57:26Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Psicología","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Alcohol","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Dependencia","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Proyectos de investigación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Alcoholismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Realidad virtual","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Psicología conductual","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Investigación - Intervención basada en realidad virtual sobre la dependencia del alcohol","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Proyecto de investigación que se están desarrollando en la Universidad Camilo José Cela (UCJC) con tecnología aplicada al abandono de la dependencia del alcohol. \nRosa Jurado, profesora e investigadora de UCJC https://www.ucjc.edu/, explica cómo se conjuga el alcoholismo con la realidad virtual. \"Mi investigación consiste en desarrollar una técnica de intervención o rehabilitación que tiene como propósito fomentar la capacidad de inhibición para que las personas que tienen dependencia del alcohol sean capaces de controlar su conducta de aproximación hacia aquello que tiene que ver con el consumo de alcohol\".\n\nEn esta investigación, liderada por la Universidad Camilo José Cela, están participando, el Instituto de Investigación Biomédica del Hospital 12 de Octubre y la Universidad Politécnica de Madrid, con el CeDint. \"En nuestro equipo de investigación pensamos que, además del deseo de consumo, existen una carencia clara de capacidad de control inhibitorio en este tipo de personas\", destaca Rosa Jurado.\n\nLa realidad virtual sirve, según explica Gabriel Rubio, jefe de Psiquiatría del Hospital 12 de Octubre, \"para dar un paso más, para ver cómo reacciona un sujeto en un ambiente que no es el real, pero se parece mucho al real\". El objetivo final de estas acciones es \"mejorar la capacidad del individuo para poner en marcha mecanismos de inhibición conductual, para que el sujeto sea capaz de decir que no\".\n\nY no acaba ahí el proceso. La fase final del tratamiento se enfoca en que el paciente \"sea capaz de reconstruir su vida\" a través de programas de entrenamiento de habilidades sociales, asertividad, familias... \"Juntando todas las áreas de intervención, la abstinencia se mantiene mucho más tiempo y las recaídas disminuyen\".","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2017-10-05","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Universidad Camilo José Cela, Vicerrectorado de Innovación","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/798"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2017-10-05","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055870750,"id":"50|od______3989::7fcbe3a03280663cddebfd3cb9203177","originalId":["od______3989::7fcbe3a03280663cddebfd3cb9203177"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:25.652Z","dateoftransformation":"2020-03-03T13:06:55.95Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:25.652Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/796","datestamp":"2018-11-05T13:36:54Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Comunicación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Redes sociales","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Comunicación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Información","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Desinformación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"III Liga de Debate CICAE","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Instantaneidad","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Información periodística","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Fake news","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"5910.02 Medios de Comunicación de Masas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Jorge Gallardo - Liga de Debate CICAE","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Jorge Gallardo, profesor de Derecho en la Universidad Camilo José Cela (UCJC), Doctor en Comunicación Audiovisual y MBA en Empresas Audiovisuales. Subdirector de Espejo Público en Antena 3 Noticias. \n\nEn esta ocasión, Jorge Gallardo habla sobre las redes sociales, cómo han transformado la manera en la que nos comunicamos y nos informamos. \n\nIII Liga de Debate CICAE - UCJC http://ligacicae.ucjc.edu/\n\nLa Universidad Camilo José Cela (UCJC) y la Asociación de Colegios Privados e Independientes –Círculo de Calidad Educativa (CICAE) – organizan la tercera edición de esta gran liga de debate académico. El formato es al estilo inglés, similar al de la Oxford Union, donde algunos jóvenes se reunían a debatir sobre temas de la más candente actualidad y polemizaban con la intención de formarse en tan valoradas habilidades.\n\nLa Liga de Debate CICAE-UCJC es una ocasión única para los alumnos de ponerse en contacto con herramientas tan útiles como la oratoria y la comunicación efectiva. Pero, sobre","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-10-23","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/796"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-10-23","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055872001,"id":"50|od______3989::d791339867bec6d3eb2104deeb4e4961","originalId":["od______3989::d791339867bec6d3eb2104deeb4e4961"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.586Z","dateoftransformation":"2020-03-03T13:06:57.721Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.586Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/645","datestamp":"2018-04-12T07:59:27Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Transferencia de Conocimiento e Innovación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Innovación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Investigación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Ciencia","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Tecnología","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"UCJC Open Science Day 2018","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"El UCJC Open Science Day tiene como objetivo mostrar las actividades de investigación llevadas a cabo por investigadores de la Universidad Camilo José Cela (UCJC) a todos los miembros de la UCJC.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-04-10","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/645"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-04-10","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055872022,"id":"50|od______3989::d90d3a1f64ad264b5ebed8a35b280343","originalId":["od______3989::d90d3a1f64ad264b5ebed8a35b280343"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.912Z","dateoftransformation":"2020-03-03T13:06:57.747Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.912Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/795","datestamp":"2018-11-06T15:43:10Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Comunicación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Unión Europea","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Desinformación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Política","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Periodismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Ciudadanos europeos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Democracias","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Estado de derecho","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Derechos humanos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"5910.02 Medios de Comunicación de Masas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Ramón Luis Varcárcel - La Unión Europea ante el reto de la desinformación","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Conferencia de Ramón Luis Valcárcel, vicepresidente del Parlamento Europeo, en el Campus de Almagro de la Universidad Camilo José Cela (UCJC) bajo el título \"La Unión Europea ante el reto de la desinformación\", en la que ha alertado sobre el alcance de la desinformación en las democracias y sobre cómo pone en peligro los derechos de los ciudadanos europeos. También ha participado el rector de la UCJC, Samuel Martín-Barbero.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-10-11","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/795"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-10-11","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates/dataset_10.json.gz b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates/dataset_10.json.gz deleted file mode 100644 index bd29d59ae948f260084f492e11d2439e22d96d1e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6736 zcmV-W8n5LaiwFp=uZ>;+17u-zVRL14UokK)YIARH0PS7flG`?xzMrQ+scNcbs?=(d z67_f2X*-!t*@??`rYbw0N`WLKp+f=+NOq!RUA1|JeF9(l!WUDy%dKwn6#FE54giXj z`ajbiGB$Qy8MOou2j@FD0C7GX{M1WmLC?fO_U1qJxVYdsk5bl~FKC*xK76xaO0i`3 zMZ!{6uq4{(UzmvNvV|)1qBrk(gIUkk#NXFSUNAv*$sYVr|4rk9OMwl?Da~^Zt@QFu z&a<^t#XMFr&%dYXw{ykTGUtU<8>p^}3YxLI6wc!n%b0#}B3Lo6+dk24=esiYiPw~R z{kq?Sc8hWi>S6`YSYFU<-JAR2H1xu7I-btF8FU(33cdf9$v67{hO_BxKKFto36mfW zg212okr&OPWV9F#<0+es!vM7<6<=~e(=&L$KYI79{!h?~Oj8yEFbgF!c*IY=Z`lT9 zk!vOrNfkbAKF8-lV;3|nv9piZdix9NiadhW3z`yk^wsI-r%ub8rMeYW7ch_>L2Eb5L(_t3BmpNtE2263>iH9> zvnq=9pNA)Z4nBo{JN0onDC6^Zz>*Ta23hov-<8l7qMI;LSOY~t=U3I62i_p`yn&BW z9f#wqeqbV@TCuVkNm4i8|^rY%tY znV0ICD)1v9`YkvuJDPE5Lwkz_Pr1cdkM{mv%P&=V&eK$qPniIwm>r(E4|{p)L-LP}A9>(K)^FQaqt6nh^3l!H&?G$#ooDJ7>eD+P~f%#96P zG8?|43C}a>U}zJD3$8%-2~8Xft-`Q;mjM7Gk^hJ`?v4l z&H9;4x9iRZ2~hM`rg1Fl9z7z)atf%!3$JPV-xCeO1MZkI4?v4fFZGj|FFDJ zv|vkq!}~@?xgh3=`<>eR;jjaU)+sO8H~R&e(jOFY5dWSbch*FefG?zifE-YZ=PV%> z-y7Bp4&Khet9Q~x6Jz60o1*JYG%!hi;j7>X(#l|!{h7!QU)k>`pp zS4CY&)G)yZjCiKc*-nA@H5a)ZNdQ;MWAJEuqsm5&W5s9zZVWtP29oKZu`Qh+N1G!N zABp66H1Licw)Yr(d!RLC{_@%5FAgO8at+hUiLP*k80JaeWHPxX#=2MFbq)OG;Iiih zql30T0m`glClBF zb9{+afpd$U+EK_<#ueG5cZt;W;G()&?crxNCDGiuSjH=u%^1y)RZuy`!(Zc4VfQ!H z?@IefmefRy9E1dS+nf;-cERI%e&j#yKI_$9MdL7py^4@yhpE{`vY(KRNs#iX$10uH{IE$uypX{%9O5W>GXvg2)@s!f~>Q;xL{>y<0evw^jcS zawP9Tx0}rFNw*7zv-e52GxD@ew+n(v?-Jc^a^(9*e)!TK&x6@K4Bs!^?m50rKHI)k zNOpTlRnZ`sRe#nrGgF%V_Ooh`+HPanIhxs>XlCeMF7!M6r|_B&iy5+kUs3`C6Py}z z`?76r+C7F}3yDx++x$YhxJ9dYJZFW2pj`-@U+Pr6=!~37vBY;@@|uLkfde!u;uGu=g|CBuMft_1T$fk1-?S*4#ZNx>$t*x-BXAGdBKQc7_8a;LP~)` zlnKW^TGd}jvO+GyE-nL_fHwk?H%BB+y>4AhI+yyZ+sMZPN&@n^|&y_hio`5OwoZb z1@1<5IZWX^vI|qVFvSCgDQ4bma&4GmF`bRZqj*H)Q4C)#m={m{(Qp{~laT6T;x~pV zZmZtSFva-3VTy3{KBE-lPDClD-jzuKFiJ7PqyR&2K6=kd0d9O9?UQ@&q}#{t>OI-3 z#DDu)*ZO(@Rhrf6K*-esVNL+cxpScr7aI9+XO`wZ@D3bSZ=$$k9W8UrGjP8!M=tbm zac4SO1>>3B7|k#5j5=W21SS~yJ}k0e!iTpGb23d&E052aih0c4xamr4s~$8`Fug&4 zzZC*GJw5sKQB4kafv-Q1J={I;^$Dyp0s7}MV@mdGUz zOeliqOZ(iMq}z9b(JMTE6(m|wj6~&BDR?=x@i&Iypf!@ObjZjWFFXRRcaA^v{rSCW zE#Vq`cu^zt%2gS33&Tg*_H7}plo)YZGi9a9wr^l~Eil^^RK%zfHpeQ5ST#6!C9JMN z#)xm<>P{3Cz5qQL3}cT| ze>RI|Vd5v@csvb#x|k%xI1FzIVcuH3gCWdtFb;?J3}H^j?=ObAlc)U{rhj7$)1S?U zQy0T@F-#Z3{8*-z{SXb(z+FhDbs+x669rmxNVJZG^Glrp0p4aEp`zvUzqsmFGUe{r zX(xsRv$3lj#)e*IIZzyv!ohPZlj6yKbk@9@tQw0h43ow(TeE^&vPEN%^$AoZpO-pj zosv_T-vF#5Q*_ZJ?oOuIgL$7!0k5v``~}P5qY3bTR-GNbXHXvZy}+u##_$L)x%n6| zQYKc^p2yYg-At5#_Sf}`9exjeW#5%N<&ols{y^c49%N#T`MsiMBve#Krjx5D9Rh7r zUA!)&7i&hw7;5KbO~t_4rJy9Ef1L^cX#De&DJeEV#kschjj`YwK(u40v4o2lpa9j0S-} zeP5)9TAubv4>noGm86H^d^DT~eygO1zk>&YAs?f6(rLuVnGo_~7yq+QIVe_@xpeos zV>CEM!$&-CxbSVqKscofwtXjDaH`$d?Lwcvb|AExB_f5TAM`EZI$Ng;2)65(nh`z+ zMVnB3!UUeT<<3Dl3WI~^4itv7{h;s>vGY68GUx=(8zR_}n$L(~nO8^8W4j`&kJ;b6 zN_HfME*iz%Nes_!W)tKLQxI(5G;=VByCp#Hz=rIHgBX6d&a}zW+*SHpZhJ8zbRnJ4BoJ6B^I8vP$G|On zGC~g;o5~{I{$ehj-o7C@$NfE>;Nh|+YyDxm^*t^GvMdf(f0}WD1@kh~VpYmBspWfL zl=`0Q!-fzpkx><6$3&q_;`x|@>Dj(54*FhauFx&Y=$z@C?I>~qGjFDoP&87to3s=b zJ6%d=ls|AC2Xqf)uy7IcHBB>}VI-#!Q=l+>HZnfwrc9usT7qP61#XoX3(M#RFhm9m zt5UmHRFghthL?4IjVg)#cO@VNKb@C)(LmS_nvG(pC=?`454k9spF&Ala5O(#62w(F z+6w40D5iyjk~e`U0> zn+i~{0p3?&Ohj*-%7%St=aAcBfyly8H(a{DwJH1X6-BUg&lJhnxm0~y%luB!#6b6U zqN2ubtg|SX$~Zh68))|TRZpw`85#zPb~ru+_?hi5Y1M|_-3BSZbmFwk4V{k*+?39Z zN;hQtD<}*;X*C>Ljv;wLJ#$;ZtKPmIcJt?zNUmj~hu7}*1^6887Q09xt3f%KoV6w^ z!?&7(5e=W4I1#ioRw5hCXuA|UfnY!^5niZJ)2w8rza8Y(XKC*N9b-4K&+J4I%T&8t ztAU5~y_lVcl9OeFTLyo_6jrdfzO#ao7dn5AS;5rla6ZIdxtQ?-Qqd+yzIWt}old(( zMf*2{WDZS7Mf=#XbyT#DP|>_jQ_y z)LSN}RLDNO;FhX!(|Pr0O(koT?m6DrgYw2sWme)Is7vzEdeHMJRo{YN7fX^DSLonq z6%MA8VdoM?OX`Rjt&fe^rCn*6x|iY6aEC$%P!I`Aq;gKFT`11Z$P>QQi8~&#h~mvA zXQwBRZ@t;XVe|g68TC=>ruYTL3yCncnK0bF9-jacSrbr=Wan=5bQSnm0rKqGkOn79>(Fsi@dPNcj-Rx=Q%2L9OVe!UDF# ziRmx}eQw`oc(oCEoD#EVr>Q=;J(w5RC%`$~zRmjp0rZh7!$!fTzWf0yB&Y`o@Fh}c zJ`6MuJv!h10;X%oNs5)i@K{W{Tv?C`mhHAjG+!mABGHT>7?Gvz^M|!Ex*LlL*nM1km26}xC8P`U*u(#Bop|o z1w#)5K!ajtgdPz0!0vY0^n$yWCS?wHX&^0EQQ4_mEK|CsyJ* z1((jz7V`+6R%eg0dE9EH_L~~FR=;kN4~@ZAG(_+?Och2(TN?LcxBrp|qP9_J6scW2 zF{Pl(;CsPK8;Y6Gm-W$9TiYXA>{woBzuVoou-yZ)fu=IKsg3%obM$hn0a&tb( z#a!CW@)FCHbS$E_Q-9&+rMnfSZ6K=T1Nr51mkyW?xGA1zE~`tcKs;wQwspZ3#LRV-C?m@%6oK%VYZ7Z`U@{{m}Y|slV_$esOl6oNxd@A0ZDq>a?P<` zx@a3+N`Hib^Mb{q@*7kWRlpFz?vSUDNKs{pxW@XB1@m0ylzd*|OY|N&XYbZ2THWk) zb>gHtSB=TD0-Xcl&da5y&Cq(vuc#A;;{@}*JmrUa!X!nQVaEi*D=jI_LnyXyW6g}v z>TQljOod`77ezzgCYO+AKtLy}eWx9LL8m}jLwU<<;9|uiOJ1pOnxZhq;D2Vg}ujm6SiVw2+gc`!*rwfx)<{*WWZP4aesaQ?> z+Uyh1I73E?<{sRhjLMoPk`y@RR}yF-(C(RlhZbdFw`TSb`-IDE`!+GSq~vchU+aB6 zKfuCY!lZ`oaC6xDGraw)0q9ejD4z7Io}RKt05RGKbWP~|R4`D!EBX4W(Pd~ThR@L_ zG+~-Ck(l%>Oj;mNb^ToiC!OtwTD}Ol7h~{y7Gc4&mSOU4OTg2nRkoE5rf@CW7r(j*u(J)t>SnoC1dn<5JaIy{WU zOeceq=y~v0Fhrzs?NWX%aBI0Df#t+B$8`FyS3RAf>GBXnO(&S-yp%>@O?N`UkkPlI zBx5mdaAnmyUOiyz1ysKtyvW}(eM629w)CSRp~YO;{@L;KHFg!Ci(nA0w(soCcrb>g zW+W{r^t$R%N(#Kc(XyU28EYX;jxpOP_DJjSRd26hP2mAcF1dCGn9coFk5GKx#rCZs zcqzG5)Y8*5X-d#tqOc~dY)3)_d(WA&x#9aDYKGgGn8*&A|GWfOr*m9Mq@|%Z+r9y~ z@=HFnk6_RU29+>F0y>t#tf$?OL0S(g_yY64ZS=FCLv;HNpPT?HT$CGzS+iV-@4@p3 zg-)lAIrLGY>6k+wF1C(2)Dh;;wA0KXHw8N8kYf%#By(sIde<_C7IDOe)Efp^_rr4p< zuCv1ZEcnN8{pzsj%EWJzx5RA~hH&P4123qu_vqX- z&5R_P58@ILc7_c3xl~I=KBGv*O5`<3y{-q#@fy1b^WAxk`|#J4L1O`mAvxskB*s2@ zB=u#AP-TDEpOoMZ0JGAh_WI-mi&RNTo4;Z*uNP(1xRy~T3pcS9=@?Bn_OjZfQ?3)aH zVI=zfIeI#s60Be<*6MI6!CXo(mlDjS1pCM)SeFmwk3{TqJhKnka2(I96FjqVr+H@1 zBRig%c%5W?%gMvd7~pgccjfr+)&}UrT6(4QmaLP+q8&N!;!1P=p-66`BCyPwR3e&qg zAX)|DF}xy~H;}{}S#)r;J1#T`Ia!q%x+{lHhXZ&b`u zJeaQ|){R$1Lh<;bKKoeISHs-#1;()_x@Zn}XHT3;(PY8yTTN=MrcW?TfQRh4Iq{4q zWuH-%(oEB3jUnO9DduJtnypLiB2LnbE|t6wQ6i$}5+72)I~yJ-f7Sav28(yC)MxBh zc>57kdUiMGl+ArSsJN@^o66XS-P(+7Y+ZA>9}iVysWnSEb}`jX(au=*fNp$wsXZVs z@&1(UTfEVyzD6gJrax7#0!s_&lNU5A;g_dSaQjXieL_{llyr2Fdku%X9P09Yr&C84 mX+%97UBuBvK2*90Mg_Y~7jYih(M4YW>Hh(z`8FgaPyqm7df$-% diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates/datasource b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates/datasource new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates/otherresearchproduct b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates/otherresearchproduct new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates/publication b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates/publication new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates/software b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates/software new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints/dataset_10.json b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints/dataset similarity index 100% rename from dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints/dataset_10.json rename to dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints/dataset diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints/datasource b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints/datasource new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints/otherresearchproduct b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints/otherresearchproduct new file mode 100644 index 000000000..ae580eab9 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints/otherresearchproduct @@ -0,0 +1,10 @@ +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055422336,"id":"50|od______2017::0750a4d0782265873d669520f5e33c07","originalId":["od______2017::0750a4d0782265873d669520f5e33c07"],"collectedfrom":[{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null}],"pid":[],"dateofcollection":"2018-05-08T17:28:57.843Z","dateoftransformation":"2019-11-30T04:19:48.897Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2018-05-08T17:28:57.843Z","altered":true,"baseURL":"http://repositorio.ipv.pt/oai/openaire","identifier":"oai:repositorio.ipv.pt:10400.19/4572","datestamp":"2017-05-05T02:01:39Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Godinho, Maria do Céu","name":"Maria Do Céu","surname":"Godinho","rank":1,"pid":null,"affiliation":null},{"fullname":"Costa, Cristina Amaro","name":"Cristina Amaro","surname":"Costa","rank":2,"pid":null,"affiliation":null}],"resulttype":{"classid":"other","classname":"other","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"por","classname":"Portuguese","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Pesticidas","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:result_subject","schemename":"dnet:result_subject"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Proteção integrada","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:result_subject","schemename":"dnet:result_subject"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"O uso dos pesticidas em Portugal. Que caminhos?","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"A agricultura está inevitavelmente associada a um conjunto de variáveis no ecossistema, onde estão incluídos as pragas e doenças nas culturas e presença de outras plantas que competem pelo alimento, água e luz. Estes agentes são responsáveis por perdas no rendimento que, normalmente, não são aceites pelo agricultor. E é, por isso, que se recorre aos pesticidas: é preciso proteger as culturas do ataque dos seus inimigos.\nTambém é verdade que a agricultura é a atividade desenvolvida pelo Homem que mais se aproxima dos sistemas naturais e tem, assim, um papel central na produção de diversos serviços do ecossistema, como a biodiversidade agrícola, qualidade do solo e água, estabilidade climática e manutenção da paisagem. Ora, o uso dos pesticidas introduz efeitos negativos em todo este sistema: degradam o solo, reduzem a qualidade e disponibilidade de água, causam poluição e perda de habitats selvagens e de biodiversidade. Aqui reside a encruzilhada na proteção das culturas.\ninfo:eu-repo/semantics/publishedVersion","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2016-07-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":{"value":"Quercus","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[{"id":"http://zenodo.org/communities/chicago-covid-19"}],"externalReference":[],"instance":[{"license":null,"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0020","classname":"Other ORP type","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null},"url":["http://hdl.handle.net/10400.19/4572"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null},"dateofacceptance":{"value":"2016-07-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"contactperson":[],"contactgroup":[],"tool":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055423621,"id":"50|od______2017::1bd97baef19dbd2db3203b112bb83bc5","originalId":["od______2017::1bd97baef19dbd2db3203b112bb83bc5"],"collectedfrom":[{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null}],"pid":[],"dateofcollection":"2019-02-25T04:30:56.994Z","dateoftransformation":"2019-11-30T04:19:58.463Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2019-02-25T04:30:56.994Z","altered":true,"baseURL":"http://repositorio.ipv.pt/oai/openaire","identifier":"oai:repositorio.ipv.pt:10400.19/5415","datestamp":"2019-02-23T03:01:23Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Brandão, Daniela","name":"Daniela","surname":"Brandão","rank":1,"pid":null,"affiliation":null},{"fullname":"Duarte, Natália","name":"Natália","surname":"Duarte","rank":2,"pid":null,"affiliation":null},{"fullname":"Araújo, Lia","name":"Lia","surname":"Araújo","rank":3,"pid":null,"affiliation":null},{"fullname":"Alves, Sara","name":"Sara","surname":"Alves","rank":4,"pid":null,"affiliation":null},{"fullname":"Teixeira, Laetitia","name":"Laetitia","surname":"Teixeira","rank":5,"pid":null,"affiliation":null},{"fullname":"Ribeiro, Oscar","name":"Oscar","surname":"Ribeiro","rank":6,"pid":null,"affiliation":null}],"resulttype":{"classid":"other","classname":"other","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"eng","classname":"English","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Quality of life","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:result_subject","schemename":"dnet:result_subject"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Centenarians","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:result_subject","schemename":"dnet:result_subject"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Centenarians´ quality of life and the role of health and living context","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Introduction: The interest of the scienti􏰀c community to study the centenarian population has been growing due to the signi􏰀cant expansion of life expectancy and human longevity. Evidences about their quality of life (QoL) are, nevertheless, still limited. Different conceptualizations of QoL, the overvaluation of the negative aspects of advanced age and the heterogeneity of the oldest old are some issues that could explain the dif􏰀culties in this assessment. Objectives: This study aims to analyze the perceived QoL in a sample of centenarians and analyze if there are significant life context (community vs institutional) and health (objective and subjective perception) differences.\nMethods: A subsample of 59 Portuguese centenarians who participated in the PT100_Oporto Centenarian Study, conducted in 2013 was considered. Descriptive and chi-square analysis were conducted.\nResults: Most centenarians perceived their QoL in a very positive manner (50.8% rated it as “good”, 6.8% as “very good” and 3.4% as “excellent”); 28.8% perceived it as “acceptable” and 10.2% as “bad”. No significant statistical differences were found for the considered variables.\nConclusions: Centenarians seem to present high levels of perceived QoL, which can be associated with a great ability to adapt to adverse situations that occur through their lives. Neither contextual and health aspects seem to in􏰂uence the perception of QoL in this particular sample study. Nonetheless, more research is \nneeded to understand the speci􏰀cities of the QoL in the centenarian population, namely the role of psychological and social variables.\ninfo:eu-repo/semantics/publishedVersion","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2014-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[{"id":"http://zenodo.org/communities/edenis"},{"id":"http://zenodo.org/communities/aginfra"}, {"id":"http://zenodo.org/communities/primefish"}],"externalReference":[],"instance":[{"license":null,"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0020","classname":"Other ORP type","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null},"url":["http://hdl.handle.net/10400.19/5415"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null},"dateofacceptance":{"value":"2014-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"contactperson":[],"contactgroup":[],"tool":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055423836,"id":"50|od______2017::1e400f1747487fd15998735c41a55c72","originalId":["od______2017::1e400f1747487fd15998735c41a55c72"],"collectedfrom":[{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null}],"pid":[],"dateofcollection":"2019-02-25T04:30:56.996Z","dateoftransformation":"2019-11-30T04:19:59.556Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2019-02-25T04:30:56.996Z","altered":true,"baseURL":"http://repositorio.ipv.pt/oai/openaire","identifier":"oai:repositorio.ipv.pt:10400.19/5422","datestamp":"2019-02-23T03:01:29Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Brandão, Daniela","name":"Daniela","surname":"Brandão","rank":1,"pid":null,"affiliation":null},{"fullname":"Ribeiro, Oscar","name":"Oscar","surname":"Ribeiro","rank":2,"pid":null,"affiliation":null},{"fullname":"Araújo, Lia","name":"Lia","surname":"Araújo","rank":3,"pid":null,"affiliation":null},{"fullname":"Paul, Constança","name":"Constança","surname":"Paul","rank":4,"pid":null,"affiliation":null}],"resulttype":{"classid":"other","classname":"other","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"eng","classname":"English","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Oldest old","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:result_subject","schemename":"dnet:result_subject"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Census","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:result_subject","schemename":"dnet:result_subject"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"What about the Portuguese oldest old? A global overview using census data","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Background\nThe older population is itself aging, and achieving an advanced age is becoming more common worldwide. In Portugal, individuals aged 80+ represent 5.6 % of the total population and 26.5 % of the popu- lation were aged 65 and over in 2011. Having a national profile on this population will give important information to develop interven- tion programs and identify the areas requiring most attention. Ob- jective: This study aims to provide a profile of the Portuguese oldest old, as given by the last national census data.\nMethods\nThe characteristics of all residents aged 80+ (N = 532,219) were ana- lysed considering socio-demographic information (gender, marital status, education, type of residence, place of birth, income) and the existence of difficulties in functional, sensorial and cognitive activities due to health problems or ageing.\nResults\nThe majority of the most aged are females (64.5 %), widowed (53.9 %), and present low educational levels (46.1 % never attended school and 31.6 % do not know how to read/write). Own pensions constitute the main source of income (96.3 %) and the majority live in private households (88.8 %), with 43.2 % currently living in the place where they were born. The majority (73.0 %) reported major difficulties in at least one functional activity – bathing/dressing, walk- ing/climbing stairs, seeing, hearing, memory/concentration, under- standing others/being understood.\nConclusions\nThe high percentage of oldest old living in private households and the presence of functional limitations point to the importance of in- formal care and community care services to support this population. Further studies paying attention to their needs and utilisation of ser- vices are required.\ninfo:eu-repo/semantics/publishedVersion","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2016-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[{"dataInfo":[{"deletedbyinference":false,"inferenceprovenance":"propagation","inferred":true,"invisible":false,"provenanceaction":{"classid":"propagation:community:productsthroughsemrel","classname":" Propagation of result belonging to community through semantic relation","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.85"}],"id":"ee"},{"dataInfo":[{"deletedbyinference":false,"inferenceprovenance":"propagation","inferred":true,"invisible":false,"provenanceaction":{"classid":"propagation:community:productsthroughsemrel","classname":" Propagation of result belonging to community through semantic relation","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.85"}],"id":"fam"},{"dataInfo":[{"deletedbyinference":false,"inferenceprovenance":"propagation","inferred":true,"invisible":false,"provenanceaction":{"classid":"propagation:community:productsthroughsemrel","classname":" Propagation of result belonging to community through semantic relation","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.85"}],"id":"covid-19"},{"id":"http://zenodo.org/communities/c2smart"}],"externalReference":[],"instance":[{"license":null,"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0020","classname":"Other ORP type","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null},"url":["http://hdl.handle.net/10400.19/5422"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null},"dateofacceptance":{"value":"2016-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"contactperson":[],"contactgroup":[],"tool":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055424072,"id":"50|od______2017::210281c5bc1c739a11ccceeeca806396","originalId":["od______2017::210281c5bc1c739a11ccceeeca806396"],"collectedfrom":[{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null}],"pid":[],"dateofcollection":"2018-11-29T04:22:08.265Z","dateoftransformation":"2019-11-30T04:20:00.603Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2018-11-29T04:22:08.265Z","altered":true,"baseURL":"http://repositorio.ipv.pt/oai/openaire","identifier":"oai:repositorio.ipv.pt:10400.19/1089","datestamp":"2018-11-26T15:31:28Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Avelar-Rosa, Bruno","name":"Bruno","surname":"Avelar-Rosa","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"other","classname":"other","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"por","classname":"Portuguese","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Treino de Jovens","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:result_subject","schemename":"dnet:result_subject"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Iniciação Desportiva","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:result_subject","schemename":"dnet:result_subject"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Metodologia","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:result_subject","schemename":"dnet:result_subject"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Didática","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:result_subject","schemename":"dnet:result_subject"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Treino de Jovens: Princípios Orientadores","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Documento que recolhe os traços gerais da intervenção do treinador de jovens, considerando as possíveis orientações pedagógicas, os eixos de evolução do praticante, os diferentes modelos didáticos de iniciação, os momentos idóneos para a intervenção nas diferentes qualidades motoras e a relação entre o treinador de jovens e os restantes agentes implicados no processo (destacando-se, naturalmente, os pais).","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2010-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":{"value":"Confederação do Desporto de Portugal","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[{"dataInfo":[{"deletedbyinference":false,"inferenceprovenance":"propagation","inferred":true,"invisible":false,"provenanceaction":{"classid":"propagation:community:productsthroughsemrel","classname":" Propagation of result belonging to community through semantic relation","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.85"}],"id":"beopen"},{"dataInfo":[{"deletedbyinference":false,"inferenceprovenance":"propagation","inferred":true,"invisible":false,"provenanceaction":{"classid":"propagation:community:productsthroughsemrel","classname":" Propagation of result belonging to community through semantic relation","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.85"}],"id":"fam"},{"dataInfo":[{"deletedbyinference":false,"inferenceprovenance":"propagation","inferred":true,"invisible":false,"provenanceaction":{"classid":"propagation:community:productsthroughsemrel","classname":" Propagation of result belonging to community through semantic relation","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.85"}],"id":"covid-19"},{"id":"http://zenodo.org/communities/primefish"}, {"id":"http://zenodo.org/communities/c2smart"}] ,"externalReference":[],"instance":[{"license":null,"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0020","classname":"Other ORP type","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null},"url":["http://hdl.handle.net/10400.19/1089"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null},"dateofacceptance":{"value":"2010-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"contactperson":[],"contactgroup":[],"tool":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055424163,"id":"50|od______2017::225a8382b6a3db76fba97c2dc146f937","originalId":["od______2017::225a8382b6a3db76fba97c2dc146f937"],"collectedfrom":[{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null}],"pid":[],"dateofcollection":"2018-05-08T17:26:56.251Z","dateoftransformation":"2019-11-30T04:20:01.127Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2018-05-08T17:26:56.251Z","altered":true,"baseURL":"http://repositorio.ipv.pt/oai/openaire","identifier":"oai:repositorio.ipv.pt:10400.19/3290","datestamp":"2016-07-14T02:01:33Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Millenium","name":null,"surname":null,"rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"other","classname":"other","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"por","classname":"Portuguese","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[],"title":[{"value":"Referências bibliográficas ao estilo APA","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[],"dateofacceptance":{"value":"2016-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[{"id":"primefish"}],"externalReference":[],"instance":[{"license":null,"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0020","classname":"Other ORP type","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null},"url":["http://hdl.handle.net/10400.19/3290"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null},"dateofacceptance":{"value":"2016-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"contactperson":[],"contactgroup":[],"tool":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055424299,"id":"50|od______2017::241d59362b075000e89d276e7bb6badc","originalId":["od______2017::241d59362b075000e89d276e7bb6badc"],"collectedfrom":[{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null}],"pid":[],"dateofcollection":"2019-09-06T04:39:29.25Z","dateoftransformation":"2019-11-30T04:20:01.631Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2019-09-06T04:39:29.25Z","altered":true,"baseURL":"http://repositorio.ipv.pt/oai/openaire","identifier":"oai:repositorio.ipv.pt:10400.19/1190","datestamp":"2019-09-04T02:00:41Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Menezes, Luís","name":"Luís","surname":"Menezes","rank":1,"pid":null,"affiliation":null},{"fullname":"Ribeiro, António","name":"António","surname":"Ribeiro","rank":2,"pid":null,"affiliation":null},{"fullname":"Martins, Ana Patrícia","name":"Ana Patrícia","surname":"Martins","rank":3,"pid":null,"affiliation":null},{"fullname":"Rodrigues, Cátia","name":"Cátia","surname":"Rodrigues","rank":4,"pid":null,"affiliation":null},{"fullname":"Gomes, Helena","name":"Helena","surname":"Gomes","rank":5,"pid":null,"affiliation":null},{"fullname":"Novo, Sónia","name":"Sónia","surname":"Novo","rank":6,"pid":null,"affiliation":null}],"resulttype":{"classid":"other","classname":"other","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"por","classname":"Portuguese","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Matemática","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:result_subject","schemename":"dnet:result_subject"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Capacidades tranversais","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:result_subject","schemename":"dnet:result_subject"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Penso, comunico, resolvo","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Esta brochura destina-se a alunos do ensino básico (1.º a 9.º anos), tendo como objetivo fornecer informação sobre as capacidades transversais que o novo Programa de Matemática português (2007) veio destacar.\nEscola Superior de Educação de Viseu","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2010-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":{"value":"ESEV","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[{"id":"http://zenodo.org/communities/fake"}],"externalReference":[],"instance":[{"license":null,"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0020","classname":"Other ORP type","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null},"url":["http://hdl.handle.net/10400.19/1190"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null},"dateofacceptance":{"value":"2010-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"contactperson":[],"contactgroup":[],"tool":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055424626,"id":"50|od______2017::28fe553056801d4a764edac6bec334ec","originalId":["od______2017::28fe553056801d4a764edac6bec334ec"],"collectedfrom":[{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null}],"pid":[],"dateofcollection":"2019-02-25T04:30:56.998Z","dateoftransformation":"2019-11-30T04:20:03.799Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2019-02-25T04:30:56.998Z","altered":true,"baseURL":"http://repositorio.ipv.pt/oai/openaire","identifier":"oai:repositorio.ipv.pt:10400.19/5432","datestamp":"2019-02-23T03:01:24Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"da Costa Teixeira, Laetitia","name":"Laetitia","surname":"Da Costa Teixeira","rank":1,"pid":null,"affiliation":null},{"fullname":"Araújo, Lia","name":"Lia","surname":"Araújo","rank":2,"pid":null,"affiliation":null},{"fullname":"Ribeiro, Oscar","name":"Oscar","surname":"Ribeiro","rank":3,"pid":null,"affiliation":null},{"fullname":"Jopp, Daniela","name":"Daniela","surname":"Jopp","rank":4,"pid":null,"affiliation":null},{"fullname":"Paul, Constança","name":"Constança","surname":"Paul","rank":5,"pid":null,"affiliation":null}],"resulttype":{"classid":"other","classname":"other","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"eng","classname":"English","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Europe","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:result_subject","schemename":"dnet:result_subject"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Centenarians","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:result_subject","schemename":"dnet:result_subject"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Centenarians in Europe","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"In the last decade, the number of centenarians world- wide has increased. Following this tendency, the number of centenarian studies has also exponentially augmented. Given the complexity to conduct research with centenarians, elementary information on this specific age group remains unknown at a European level. This paper compares basic characteristics of centenarians from 32 European coun- tries based on Census 2011. Results revealed that France is the country with higher ratio of centenarians, followed by Italy and Greece; on the other hand Croatia, Romania and Bulgaria present the lower ratio. The distribution accord- ing to gender is similar for all countries, with higher ratio of women reaching 100 years old. Different patterns of education level were found on countries where this infor- mation was reliable: Portugal and Greece are the countries with lower levels of education, with more than half without formal education; Finland, UK and Iceland are the countries with higher levels of education, with all centenarians having at least lower secondary education. Analysing the residence situation (living in the community or living in the institu- tion), Romania, Bulgaria, Lithuania, Greece and Latvia are the countries that present higher percentages of centenarians living in the community; Iceland is the country with lower percentage of centenarians living in the community. A posi- tive correlation between the number of centenarians and the percentage of Total Health Expenditure of GDP is observed. This study provides important information about the current profile of European centenarians, which is compared to gen- eral information on the centenarian population from EUA and Asia.\ninfo:eu-repo/semantics/publishedVersion","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2017-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":null,"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0020","classname":"Other ORP type","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null},"url":["http://hdl.handle.net/10400.19/5432"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null},"dateofacceptance":{"value":"2017-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"contactperson":[],"contactgroup":[],"tool":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055424730,"id":"50|od______2017::2a777b5c11f070ece546d75162047179","originalId":["od______2017::2a777b5c11f070ece546d75162047179"],"collectedfrom":[{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null}],"pid":[],"dateofcollection":"2019-02-25T04:30:56.998Z","dateoftransformation":"2019-11-30T04:20:04.285Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2019-02-25T04:30:56.998Z","altered":true,"baseURL":"http://repositorio.ipv.pt/oai/openaire","identifier":"oai:repositorio.ipv.pt:10400.19/5428","datestamp":"2019-02-23T03:01:30Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"da Costa Teixeira, Laetitia","name":"Laetitia","surname":"Da Costa Teixeira","rank":1,"pid":null,"affiliation":null},{"fullname":"Araújo, Lia","name":"Lia","surname":"Araújo","rank":2,"pid":null,"affiliation":null},{"fullname":"Paul, Constança","name":"Constança","surname":"Paul","rank":3,"pid":null,"affiliation":null},{"fullname":"Ribeiro, Oscar","name":"Oscar","surname":"Ribeiro","rank":4,"pid":null,"affiliation":null}],"resulttype":{"classid":"other","classname":"other","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"eng","classname":"English","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Survival","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:result_subject","schemename":"dnet:result_subject"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"100 years old","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:result_subject","schemename":"dnet:result_subject"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Survival after 100 years of age in Portuguese centenarians","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"The group of individuals aged 80 and over is growing faster than other segment of the population, and within this group the number of centenarians has risen exponentially worldwide. This phenomena lead to an increasing number of centenarian studies, improving the knowledge about this population, validating the idea of heterogeneity that char- acterizes this group. However, the factors influencing their aging survival remains understudied. The present study aims to identify factors associated with survival after 100 years old in a sample of Portuguese centenarians. After being included in a population-based study on centenarians (PT100–Oporto Centenarian Study), each participant was assessed by a tel- ephone interview every 6 months in order to assess his/her health condition. Follow-up was considered as the time (in months) between 100th birthday and death or the last tel- ephone contact. Survival analysis was performed to iden- tify factors associated with survival after 100 years old. Sociological and medical factors were considered. 140 cen- tenarians were assessed and 115 (82.1%) died. The median survival time after 100 years old was 38 months. Survival after age 100 was associated with acute disease (last month), self-perception of physical condition and drinking status. This study intends do identify factors that predict longevity in long-lived individuals. Findings recognize the importance of factors related with health status, self-perception of physi- cal condition and life style (no alcohol consume) for survival. Further studies should explore the predictive value of these factors, as well as psychosocial aspects, considering living longer but also the quality of time beyond one hundred.\ninfo:eu-repo/semantics/publishedVersion","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":null,"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0020","classname":"Other ORP type","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null},"url":["http://hdl.handle.net/10400.19/5428"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null},"dateofacceptance":{"value":"2018-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"contactperson":[],"contactgroup":[],"tool":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055426968,"id":"50|od______2017::48eb86b38e6e2cfb94bf0a4663848c10","originalId":["od______2017::48eb86b38e6e2cfb94bf0a4663848c10"],"collectedfrom":[{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null}],"pid":[],"dateofcollection":"2019-02-25T04:30:56.995Z","dateoftransformation":"2019-11-30T04:20:13.611Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2019-02-25T04:30:56.995Z","altered":true,"baseURL":"http://repositorio.ipv.pt/oai/openaire","identifier":"oai:repositorio.ipv.pt:10400.19/5416","datestamp":"2019-02-23T03:01:21Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Teixeira, Laetitia","name":"Laetitia","surname":"Teixeira","rank":1,"pid":null,"affiliation":null},{"fullname":"Araújo, Lia","name":"Lia","surname":"Araújo","rank":2,"pid":null,"affiliation":null},{"fullname":"Duarte, Natália","name":"Natália","surname":"Duarte","rank":3,"pid":null,"affiliation":null},{"fullname":"Brandão, Daniela","name":"Daniela","surname":"Brandão","rank":4,"pid":null,"affiliation":null},{"fullname":"Azevedo, Maria João","name":"Maria João","surname":"Azevedo","rank":5,"pid":null,"affiliation":null},{"fullname":"Ribeiro, Oscar","name":"Oscar","surname":"Ribeiro","rank":6,"pid":null,"affiliation":null}],"resulttype":{"classid":"other","classname":"other","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"eng","classname":"English","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Cognition","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:result_subject","schemename":"dnet:result_subject"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Survival","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:result_subject","schemename":"dnet:result_subject"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Functional capacity","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:result_subject","schemename":"dnet:result_subject"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Cognition and functionality: how do they related to time lived after 100 years of age?","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Introduction: The number of centenarians has greatly increased in Portugal in the last decades. Therefore, the analysis of survival time after becoming a centenarian and related factors constitute an import- ant issue for the quality of care provision. The objective of this study is to identify health related factors associated with the time lived after being 100 years of age. Methods: Data come from the population-based study PT100 (Oporto Centenarian Study) and considers information gathered through face-to-face interviews with centenarians and their proxies during 2013. Study eligibility criteria included being 100 and more years old and living in the Oporto Metropolitan Area. Survival analyses were performed in order to identify factors associated with survival after the 100. Functional status (e.g., walking, being bedrid- den) and specific health conditions (e.g. presence/absence of cognitive impairment) were considered as potential factors. Results: The sam- ple comprises 140 centenarians with approximately 14.5 months as median survival time after their 100th anniversary. Centenarians who were bedridden presented a higher probability to live after 100 years when compared with functionally independent centenarians. Presence of cognitive impairment was not associated with the time lived after 100 years old. Conclusions: Given the increasing number of centenarians in Portugal, different studies based on this population need to be consid- ered. The present study, focused on the life after 100 years old, provides new information about this topic contributing to the formulation of new scientific questions for this population.\ninfo:eu-repo/semantics/publishedVersion","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2014-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":null,"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0020","classname":"Other ORP type","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null},"url":["http://hdl.handle.net/10400.19/5416"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null},"dateofacceptance":{"value":"2014-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"contactperson":[],"contactgroup":[],"tool":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055427049,"id":"50|od______2017::4a1bcdd965b879fbac86a7d3b947a6cc","originalId":["od______2017::4a1bcdd965b879fbac86a7d3b947a6cc"],"collectedfrom":[{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null}],"pid":[],"dateofcollection":"2018-05-08T17:24:51.637Z","dateoftransformation":"2019-11-30T04:20:13.727Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2018-05-08T17:24:51.637Z","altered":true,"baseURL":"http://repositorio.ipv.pt/oai/openaire","identifier":"oai:repositorio.ipv.pt:10400.19/995","datestamp":"2014-07-15T01:00:31Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Oliveira, Ana Maria","name":"Ana Maria","surname":"Oliveira","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"other","classname":"other","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"por","classname":"Portuguese","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Cognição e Linguagem","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:result_subject","schemename":"dnet:result_subject"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Processamento da Linguagem","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:result_subject","schemename":"dnet:result_subject"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Processar a Linguagem através dos Sentidos: entre a doçura e a amargura","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[],"dateofacceptance":{"value":"2010-07-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":{"value":"Instituto de Literatura Comparada Margarida Losa – Faculdade de Letras da Universidade do Porto","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":null,"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0010","classname":"Lecture","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null},"url":["http://hdl.handle.net/10400.19/995"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::2017","value":"Repositório Científico do Instituto Politécnico de Viseu","dataInfo":null},"dateofacceptance":{"value":"2010-07-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"contactperson":[],"contactgroup":[],"tool":[]} diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints/publication b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints/publication new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints/software b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints/software new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/contextnoprovenance/dataset b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/contextnoprovenance/dataset new file mode 100644 index 000000000..d03739195 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/contextnoprovenance/dataset @@ -0,0 +1,10 @@ +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055868909,"id":"50|od______3989::02dd5d2c222191b0b9bd4f33c8e96529","originalId":["od______3989::02dd5d2c222191b0b9bd4f33c8e96529"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[{"value":"10.4185/RLCS-2018-1243","qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofcollection":"2020-03-03T13:05:26.091Z","dateoftransformation":"2020-03-03T13:06:53.161Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:26.091Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/562","datestamp":"2018-01-23T15:06:07Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Gallardo-Camacho, Jorge","name":"Jorge","surname":"Gallardo-Camacho","rank":1,"pid":null,"affiliation":null},{"fullname":"Trujillo Fernández, José Ramón","name":"José Ramón","surname":"Trujillo Fernández","rank":2,"pid":null,"affiliation":null},{"fullname":"Jorge Alonso, Ana","name":"Ana","surname":"Jorge Alonso","rank":3,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"COVID-19","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"yihadismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"viralidad","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"vídeo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"propaganda","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"terrorismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"terrorism","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"jihadism","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"virality","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"video","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"5903 Ideologías Políticas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"6310.13 Terrorismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"El individualismo como estrategia","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Individualism as an improvised strategy","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Se analiza como el Yihadismo distribuye su mensaje propaganístico en You Tube.\nUniversidad Camilo José Cela","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"MediaLab - UCJC","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[{"dataInfo":[{"deletedbyinference":false,"inferenceprovenance":"propagation","inferred":true,"invisible":false,"provenanceaction":{"classid":"propagation:community:productsthroughsemrel","classname":" Propagation of result belonging to community through semantic relation","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.85"}],"id":"ee"},{"dataInfo":[{"deletedbyinference":false,"inferenceprovenance":"propagation","inferred":true,"invisible":false,"provenanceaction":{"classid":"propagation:community:productsthroughsemrel","classname":" Propagation of result belonging to community through semantic relation","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.85"}],"id":"fam"},{"dataInfo":[{"deletedbyinference":false,"inferenceprovenance":"propagation","inferred":true,"invisible":false,"provenanceaction":{"classid":"propagation:community:productsthroughsemrel","classname":" Propagation of result belonging to community through semantic relation","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.85"}],"id":"covid-19"}],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/562"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055868954,"id":"50|od______3989::05d8c751462f9bb8d2b06956dfbc5c7b","originalId":["od______3989::05d8c751462f9bb8d2b06956dfbc5c7b"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:26.079Z","dateoftransformation":"2020-03-03T13:06:53.239Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:26.079Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/227","datestamp":"2017-11-15T16:29:55Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Rico Pérez, Marta","name":"Marta","surname":"Rico Pérez","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"coronavirus disease-19","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"agriculture","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Retos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"I Congreso Nuevos Retos en los Eventos Deportivos","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"I Congreso Nuevos Retos en los Eventos Deportivos, celebrado en la Universidad Camilo José Cela.\n\n\n\nDerivado del proyecto de investigación \"La situación de los eventos deportivos en España a través de su estructuras y de sus herramientas\", tiene como finalidad conocer la situación de los eventos deportivos en España. Se realizó un estudio descriptivo holístico en el que se investigaron elementos culturales, el perfil de los recursos humanos, la estructura de los comités, entre otras.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2015-06-20","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/227"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2015-06-20","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055869097,"id":"50|od______3989::0f89464c4ac4c398fe0c71433b175a62","originalId":["od______3989::0f89464c4ac4c398fe0c71433b175a62"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:26.069Z","dateoftransformation":"2020-03-03T13:06:53.54Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:26.069Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/802","datestamp":"2018-11-14T15:30:47Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Ciencias de la Actividad Física y del Deporte","name":"Ciencias La Actividad Física Y. Del Deporte","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"fishery","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"sea","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Impacto físico","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Proyectos de investigación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"2411.06 Fisiología del Ejercicio","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Investigaciones en Ciencias de la Actividad Física y del Deporte CCAFD - UCJC","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Blanca Romero, investigadora y docente de Ciencias de la Actividad Física y del Deporte en la Universidad Camilo José Cela, describe dos de las líneas de investigación que se han desarrollado en el departamento CCAFD de UCJC: el impacto físico de las diferentes actividades deportivas y los métodos de recuperación más eficaces tras el ejercicio físico.\nBlanca también destaca la importancia en el ámbito de la salud los estudios en Ciencias de la Actividad Física y del Deporte.\nCiencias de la Actividad Física y del Deporte en la Universidad Camilo José Cela","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2016-03-21","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Universidad Camilo José Cela, Ciencias de la Actividad Física y del Deporte","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/802"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2016-03-21","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055869215,"id":"50|od______3989::1729c3988199b95d1d566851af7d3c55","originalId":["od______3989::1729c3988199b95d1d566851af7d3c55"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:25.653Z","dateoftransformation":"2020-03-03T13:06:53.764Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:25.653Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/801","datestamp":"2018-11-14T15:19:38Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Ciencias de la Actividad Física y del Deporte","name":"Ciencias La Actividad Física Y. Del Deporte","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Actividad Física y del Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Prevención","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Hidratación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deportistas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Electrolitos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deportes de resistencia","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"2411.06 Fisiología del Ejercicio","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"La importancia de la hidratación en competiciones resistencia, por Juan del Coso","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Juan del Coso, investigador y profesor en el Grado en Ciencias de la Actividad Física y del Deporte UCJC, presenta el proyecto de investigación que ha realizado el departamento CCAFD de UCJC: la prevención de desequilibrios de agua y electrolitos que se dan en los deportes de resistencia para que los deportistas realicen su actividad de una manera más segura.\nMás información: http://www.ucjc.edu/2016/03/investiga...\nUniversidad Camilo José Cela, Ciencias de la Actividad Física y del Deporte","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2016-03-21","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Grado en Ciencias de la Actividad Física y del Deporte UCJC","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/801"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2016-03-21","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055869487,"id":"50|od______3989::2e3f34ce90520fae350a7e1148d7dcea","originalId":["od______3989::2e3f34ce90520fae350a7e1148d7dcea"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.912Z","dateoftransformation":"2020-03-03T13:06:54.218Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.912Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/800","datestamp":"2018-11-14T13:43:21Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Actividad Física y del Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Cafeína","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Efectos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Alto rendimiento","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Competiciones","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Beneficios","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"2411.06 Fisiología del Ejercicio","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Investigación - Efectos positivos y negativos del uso de la cafeína en el deporte","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Proyecto de investigación que se están desarrollando en la Universidad Camilo José Cela (UCJC) sobre la cafeína y sus efectos cuando se realiza deporte. https://www.ucjc.edu/2016/08/la-bbc-s...\n\nJuan Del Coso, director del Laboratorio de Fisiología del Ejercicio de la Universidad Camilo José Cela, nos describe cómo está siendo la investigación que desarrolla con deportistas de alto rendimiento y el consumo de cafeína.\n\n\"Nuestra investigación se enmarca dentro del campo de la fisiología del ejercicio y el objetivo que tenemos es investigar los efectos que tiene la cafeína en los deportistas, principalmente saber si es una ayuda ergogénica en la mayor parte de los deportes, pero también conocer qué perjuicios existen para aquellos deportistas que la utilizan para incrementar el rendimiento deportivo\", explica Del Coso.\n\nY es que los datos son muy llamativos: tres de cada cuatro deportistas de alto rendimiento utilizan la cafeína antes de la competición y muchas veces se utiliza sin saber los perjuicios que puede tener para el deportista. \"Lo que estamos investigando es cómo una sustancia, que te puede hacer rendir mejor, cómo puede afectar negativamente cuando termina una competición\".\n\nPero esta investigación no sólo puede extraer conclusiones interesantes y útiles para los deportistas profesionales, \"investigando el deporte de alto rendimiento podemos conocer los beneficios y perjuicios de la cafeína que también afectarían al deportista amateur y hacer un deporte en general más seguro\".\n\nPara el desarrollo de este proyecto de investigación están colaborando el Laboratorio de Fisiología del Ejercicio de la Universidad Camilo José Cela y la Agencia para la Protección de la Salud en el Deporte.\nPara el desarrollo de este proyecto de investigación están colaborando el Laboratorio de Fisiología del Ejercicio de la Universidad Camilo José Cela y la Agencia para la Protección de la Salud en el Deporte.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2017-10-06","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Universidad Camilo José Cela, Vicerrectorado de Innovación","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/800"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2017-10-06","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055869498,"id":"50|od______3989::2f4f3c820c450bd08dac08d07cc82dcf","originalId":["od______3989::2f4f3c820c450bd08dac08d07cc82dcf"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.806Z","dateoftransformation":"2020-03-03T13:06:54.229Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.806Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/797","datestamp":"2018-11-14T12:12:23Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Cuesta Cano, Laura","name":"Laura","surname":"Cuesta Cano","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Comunicación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Marketing digital","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Medios sociales","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Marcas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Empresas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Emprendedores","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"II Liga de Debate CICAE - UCJC","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"6114.13 Marketing","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Laura Cuesta - Liga de Debate CICAE - UCJC","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Laura Cuesta, profesora en la Universidad Camilo José Cela (UCJC) y Especialista en Marketing Digital y Medios Sociales para marcas y emprendedores. \n\nEn esta ocasión, Laura Cuesta, nos explica el significado de redes sociales y nos habla sobre el uso que le dan los usuarios.\n\nII Liga de Debate CICAE - UCJC http://ligacicae.ucjc.edu/\n\nLa Universidad Camilo José Cela (UCJC) y la Asociación de Colegios Privados e Independientes –Círculo de Calidad Educativa (CICAE) – organizan la segunda edición de esta gran liga de debate académico. El formato es al estilo inglés, similar al de la Oxford Union, donde algunos jóvenes se reunían a debatir sobre temas de la más candente actualidad y polemizaban con la intención de formarse en tan valoradas habilidades.\n\nLa Liga de Debate CICAE-UCJC es una ocasión única para los alumnos de ponerse en contacto con herramientas tan útiles como la oratoria y la comunicación efectiva. Pero, sobretodo, una oportunidad de fomentar el pensamiento crítico y la socialización entre alumnos con espíritu emprendedor. La Liga de Debate CICAE - Universidad Camilo José Cela se diferencia del resto de torneos en que potenciamos que sea eminentemente formativa. Alumnos y profesores reciben formación en comunicación, oratoria y debate por parte de alguno de los expertos en debate españoles más reconocidos en la escena internacional.\n\nDatos de la III Liga de Debate CICAE - UCJC http://ligacicae.ucjc.edu/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-10-23","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Universidad Camilo José Cela, Vicerrectorado de Innovación","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/797"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-10-23","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055870582,"id":"50|od______3989::752fd0b2bbac1ea1cc50e52fd46eb663","originalId":["od______3989::752fd0b2bbac1ea1cc50e52fd46eb663"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.561Z","dateoftransformation":"2020-03-03T13:06:55.716Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.561Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/798","datestamp":"2018-11-14T12:57:26Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Psicología","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Alcohol","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Dependencia","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Proyectos de investigación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Alcoholismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Realidad virtual","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Psicología conductual","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Investigación - Intervención basada en realidad virtual sobre la dependencia del alcohol","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Proyecto de investigación que se están desarrollando en la Universidad Camilo José Cela (UCJC) con tecnología aplicada al abandono de la dependencia del alcohol. \nRosa Jurado, profesora e investigadora de UCJC https://www.ucjc.edu/, explica cómo se conjuga el alcoholismo con la realidad virtual. \"Mi investigación consiste en desarrollar una técnica de intervención o rehabilitación que tiene como propósito fomentar la capacidad de inhibición para que las personas que tienen dependencia del alcohol sean capaces de controlar su conducta de aproximación hacia aquello que tiene que ver con el consumo de alcohol\".\n\nEn esta investigación, liderada por la Universidad Camilo José Cela, están participando, el Instituto de Investigación Biomédica del Hospital 12 de Octubre y la Universidad Politécnica de Madrid, con el CeDint. \"En nuestro equipo de investigación pensamos que, además del deseo de consumo, existen una carencia clara de capacidad de control inhibitorio en este tipo de personas\", destaca Rosa Jurado.\n\nLa realidad virtual sirve, según explica Gabriel Rubio, jefe de Psiquiatría del Hospital 12 de Octubre, \"para dar un paso más, para ver cómo reacciona un sujeto en un ambiente que no es el real, pero se parece mucho al real\". El objetivo final de estas acciones es \"mejorar la capacidad del individuo para poner en marcha mecanismos de inhibición conductual, para que el sujeto sea capaz de decir que no\".\n\nY no acaba ahí el proceso. La fase final del tratamiento se enfoca en que el paciente \"sea capaz de reconstruir su vida\" a través de programas de entrenamiento de habilidades sociales, asertividad, familias... \"Juntando todas las áreas de intervención, la abstinencia se mantiene mucho más tiempo y las recaídas disminuyen\".","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2017-10-05","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Universidad Camilo José Cela, Vicerrectorado de Innovación","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/798"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2017-10-05","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055870750,"id":"50|od______3989::7fcbe3a03280663cddebfd3cb9203177","originalId":["od______3989::7fcbe3a03280663cddebfd3cb9203177"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:25.652Z","dateoftransformation":"2020-03-03T13:06:55.95Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:25.652Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/796","datestamp":"2018-11-05T13:36:54Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Comunicación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Redes sociales","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Comunicación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Información","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Desinformación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"III Liga de Debate CICAE","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Instantaneidad","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Información periodística","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Fake news","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"5910.02 Medios de Comunicación de Masas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Jorge Gallardo - Liga de Debate CICAE","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Jorge Gallardo, profesor de Derecho en la Universidad Camilo José Cela (UCJC), Doctor en Comunicación Audiovisual y MBA en Empresas Audiovisuales. Subdirector de Espejo Público en Antena 3 Noticias. \n\nEn esta ocasión, Jorge Gallardo habla sobre las redes sociales, cómo han transformado la manera en la que nos comunicamos y nos informamos. \n\nIII Liga de Debate CICAE - UCJC http://ligacicae.ucjc.edu/\n\nLa Universidad Camilo José Cela (UCJC) y la Asociación de Colegios Privados e Independientes –Círculo de Calidad Educativa (CICAE) – organizan la tercera edición de esta gran liga de debate académico. El formato es al estilo inglés, similar al de la Oxford Union, donde algunos jóvenes se reunían a debatir sobre temas de la más candente actualidad y polemizaban con la intención de formarse en tan valoradas habilidades.\n\nLa Liga de Debate CICAE-UCJC es una ocasión única para los alumnos de ponerse en contacto con herramientas tan útiles como la oratoria y la comunicación efectiva. Pero, sobretodo, una oportunidad de fomentar el pensamiento crítico y la socialización entre alumnos con espíritu emprendedor. La Liga de Debate CICAE - Universidad Camilo José Cela se diferencia del resto de torneos en que potenciamos que sea eminentemente formativa. Alumnos y profesores reciben formación en comunicación, oratoria y debate por parte de alguno de los expertos en debate españoles más reconocidos en la escena internacional.\n\nDatos de la III Liga de Debate CICAE - UCJC http://ligacicae.ucjc.edu/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-10-23","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/796"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-10-23","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055872001,"id":"50|od______3989::d791339867bec6d3eb2104deeb4e4961","originalId":["od______3989::d791339867bec6d3eb2104deeb4e4961"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.586Z","dateoftransformation":"2020-03-03T13:06:57.721Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.586Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/645","datestamp":"2018-04-12T07:59:27Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Transferencia de Conocimiento e Innovación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Innovación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Investigación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Ciencia","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Tecnología","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"UCJC Open Science Day 2018","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"El UCJC Open Science Day tiene como objetivo mostrar las actividades de investigación llevadas a cabo por investigadores de la Universidad Camilo José Cela (UCJC) a todos los miembros de la UCJC.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-04-10","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/645"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-04-10","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055872022,"id":"50|od______3989::d90d3a1f64ad264b5ebed8a35b280343","originalId":["od______3989::d90d3a1f64ad264b5ebed8a35b280343"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.912Z","dateoftransformation":"2020-03-03T13:06:57.747Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.912Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/795","datestamp":"2018-11-06T15:43:10Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Comunicación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Unión Europea","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Desinformación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Política","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Periodismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Ciudadanos europeos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Democracias","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Estado de derecho","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Derechos humanos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"5910.02 Medios de Comunicación de Masas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Ramón Luis Varcárcel - La Unión Europea ante el reto de la desinformación","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Conferencia de Ramón Luis Valcárcel, vicepresidente del Parlamento Europeo, en el Campus de Almagro de la Universidad Camilo José Cela (UCJC) bajo el título \"La Unión Europea ante el reto de la desinformación\", en la que ha alertado sobre el alcance de la desinformación en las democracias y sobre cómo pone en peligro los derechos de los ciudadanos europeos. También ha participado el rector de la UCJC, Samuel Martín-Barbero.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-10-11","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/795"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-10-11","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/contextnoprovenance/dataset_10.json.gz b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/contextnoprovenance/dataset_10.json.gz deleted file mode 100644 index ee62cd7913e67677f1d81e880f137d1545656608..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6888 zcmVxLlHK z`R}e$UCC6?oiw_82FOki0!pH7RwUA)q)MD@1Dj{qC+N$*=!-$_a_ieX#XiZNLsFLP z`p;BHO2XYnrfl=)@O+2A=i}k`y@VF@Tr6a7{(X;&OP=#6Wxe@=ra9}wHw&f|OLku* zEM)~tqK*EAiMUQ%s4_2l^PV@D^=wZ3eXZmr6I7?{!4LIk8W&s&tT;|-o^zt*3ZAjNpxL@N_rq!Eh2eBOoq028G}aV)|1Fa*_5TfL)7gCP1xXSnK^z2u zKl39mnnlTIF&xHIHXDZlYD+4<P{2QX56u2A(27h`76UK~B{L}E``%Y<1G309 z6N#h>OPkNJJgDrFrX@D^30rS}L|u_5P1ZkvD{Yul(WM3+KUj;LZI1>Q$)=MMb`lDx=ru7|+Auz#r=z z>|01}u=pobCn<00?t5$BPgou+zOGcaqUsVl(i5odc6n%8FpVVO1ZYJxXMcb36!NT! zV*Tgg$)AId;onYuTn@_kA|9}$gs(vs{o~gq)P?9KOcd5YQPBBy`R0K)2t9A$qg2P? zIIvo=-M*&{-t&(F5H>_rcoP&fV+BR4{0tz;*HqgL$~Zqh{^px+2G|>^idP((J&AtprzGu<>gr+G~i99-`8I4!6Pdcl#P+R7*mqcYzI{ht&}{po zXv&bYpMSuw<MJkO+qp-mVrxdPoMG;uJr3d8n&!dzEq6$AKvO_x+8)Zx%34zNa6u!wb2 zLsxKcFtiE7goA&sF;MdiM$aI4&2p6_&e z|ADW!E+ZJ}Ryc|orxVK}s+N+8#Pk*OZ7xd{+n$5|u7Z7A*iYIEYyOU<(A}f3PQnVC zXDs3LDUHYxx#Glk0Npus8RZ+7pMfKWq)L0>Q0)==exXz3x#G)JQ6~}>rtcg2#t%1Z z^LqW+0Z!|!iH`AkU&ZfsWj9Kj!w{R50yg2;{G^PFyjUq&E>}6rV9Y+yJb7McC2~QG zcalUbm14=ok`#I!W=bTsBmzaj$j@U)o>pymG>Do8I1PIUnDK_<4DIy(tLuG1GpG1A z6c0lZPtXOA-@vegT#EDs+vytzDlT$6Zv!YPk750^H-}q3N~{=&eaWz8P^YG&*wp!P zv^f&-kw}h51Mk?)BoCPC_oj%Z%%49$`|Lon&)2YsJ<%Di5yO=9WhRpw*c*EZ-qgTf zrLA~gFgmFFW1eQ$wH;R~v9{+gt2y!d=3kc(gjmU3BaZ?+Dh09MD^-f*NNc~VBps}% zNK!Tcd5*8}wQwbCHxejhD&vaA*1JNNd~{LW8v60q&`P4Y!+0F8VC~FkhJ2-J8Fu(< zTq1KN{5dpB9iJ|aF(L{SHQ%Y|2P!3)T^Z!qn;+L2Sa6y;n z2G%h6fdG`SX5~v7^X-p8Iho>eBe%K9?^3g9(m)Vz;in7hUfTkyq?Yv z(M`i~euxhEA@I1V%YFzqB0E2X^FutMA7bXsCO7&a7Sq{iJc>s&9>wt0f_d@89}S0* zKMARhFmbCN;;!u7^h1mv>W2tNAJPpm?t~j+>RlU81lJZ(o^Lu}+5{#T z`93U_VA_VggSoD%XZy1YreYp**KfMwp{hHL6igqpKWyPfPESuhep25Uaqg?%kUiYr zef1Hn6#@DeGGj{i>#;VGN*jcXVGG01WFBZf_i%0Xt!F2K@}(^|C+YURVDy@izjF1g zDEgCfsuXN7trv}4vGr4 z&Dp+%RE3q3T?~b^^YHS|+NW{VWYhJ%} zj>s-z>l~4ta75yPbC(^FZccQLNau)rOh;tkhc`JQ{YfxG58~9H&7xVD_(?b(PeY$B zCdn`k!#f<2cV_RPBQhL}!{GxRk(2R<^F;3CY2Op+-|C6BppT60LWj)WVRIs*diW?k^yc7Zs z^fJqV;+PZ;o;#ToPxrmEW^1x)EV|H58p~|W3U0|3jZW4lkd=H|YM*sVPGx=zu#Qa8 zMU%KcnPLy-eKG}XU7`F1%i&Ms;Qyq$0sKIxJf3@jRe_D+MOx~_L4{c>%mptW>;)eb}VMh-#vBvyfQ8N-Ms*BIb^^*>Px~WcHC(?^GBV!D? z^RlL5VD3^-l2O4RpXOTRY*{M27yS%Bfx+QvotAe_8s|>csn#(%5stm#@lG{^!ChgY zolDoTAbvwS{gqe{|H+8u&~&5+hpi($bcOWbJJQ4NtYbOS!yg#`J4p|t>5aFq12$X? zM=_guVc;z&8-^a8Fy9|dlSvZmdwI8!9`4NEJ<`K)G?@9pgGdjfLEuk66zQRsr+w0c z4S8`b>0vk@4d;R1D(T@LFaklBk8yO;!NAD55b|>8|FfkWB&*WMpMTJ=5jaMJV>GyV z!?|xe2f`^`u6#Jxd(;eeBUX2MC|-dvTxf?)f$8G}LGFO)g?3m#wnmE_ASXdp6}@Z4_7r=>kosb?{OiJWpS|j(~JvDn3tIrt5Ti`5a0U@553ft ziV!Z5Q59puM4?Rh_?Uv}*|seX+FnPl&^5~Fg6YWXC~^TaZ>ED#Gy<-huoM3D zlRn0GmvwxNDirT^R+WQ4bT_Zt{v@#qN;KLJu5c^3?MTj@0=9MFlJ&3jfZ|<0K45V zc2YJqPKD*(XzQ{|S2TzX)ETd!s7r?NO29sBe$c(JD4>iDeWBRvk|hRzWwf!I3Q({C zzB*t`L?4{WhJEP4A-CNEk%g{qxO9DMQ?~FGMX+?s6v^0yRDGMv{7%utK=*c{qQ-9Q zU{NrYaVQ)sXtwutPpkhq8U~7X*gph}Gut22st$d+4N`#V#A%ruIv*E!D4iRXZpij0 zP#FBB)o^GzhU5kL%$ozRd;5CW&7apIxt59UUb{CG;O}U+*hLCi4a&jftTkB~zSR`; zX!yH{6G2O3C9=_swo9=S2s*?P;e`w}%}Q4K+d*!}EIoQa!`N-?GdoelGS$Pa)xcwp zy%?Q`l9OeFTLzzD3M*J#zdS+7OC3MQtYGSJIKRhMIiK+(Qqd+yzIWt}old((Mf(q( zWDZS7Mf;;;>!@fQp`v-6rlPqi&{5GG741=}XtU{!RJ6rtF^s2y7mq?OO1x=8WB89Z ziQ&s6Ui9vuqTQ9flT@^+H-0D;E%;y$lGpOIO+}l`u6dArM@0+f@GlsCSSs3SsgF!f zsgQlx;FhZKAbIt%rjj*M_Z)BRQF&vhGAr>6)CKuy-Rb#^s;^*N7fX_uq0qt6DjZA) z!_Fo2medh5TJIaN3%k-XbuYuK;SPlkpdb>KNaY5lcA+>wCr|lO2kv;nB8o4YoS&YY z-TAVK!{)-4VN^JNjiE$PvN1i1q!6U`-q7@1obxhcLZ1 z%ylWNXDOj*rU!^}_btvzG`XhC;g|5KT=C2H%r;+GU3BtqY2!tnHtn|F-1Y4z&YK%%liNUw2>;qM!~8+{}wVNs0Rt~B~oZU3{(#- zy4d~*rfbMaij~6fSWLZKS&#~r?T$w@UnQm@(TpJIk)>_(hO9wf8MH5=nh$2Ects<= zk(OI&nj9%05m-Mujj1*Xo|^Zd1{SvKFxLm66>F`sNV5G&bJlhoA?QZLopNju|Aq=YkNYA9n0(d7Y{eiZTE<5peedSgQ3%@V*~vm(R6H}-!Ha~4b%xX z5ITLk%m#8(pko6$Hqhg-fhJx!4Q^rsO~PQ2cu^2VH1-+w<2dvfem5GkXgtPVcpDq& z&g`9J1BGM%L2RIKF!9G9kPT$zX`c-=?cK%(3Mcbm>;h9cg3&z*M$hq0a&te)`CQtK z@)FCHbS$E_Lx17nrF#{nZ6K=P1Nqf^mkyW?3{yPMTvV4`2 zafXZ(%{{m~7?m|oBq?x=uO!ewpx$!pO zA~E4vn6yBk>iW9~PCD8TwR{m6UW~!-S%d}8T84?w!h{L(QdmhUCNK3=fZ2K1A%CC%} zeFTF>(5Zy!640;=W<5O&8KiZmf-f-s+eViK4WirkSaJfWa8Yg;M$K|AzDJKE6gr(c z=FlG!O~)Mi{bK8wLmgobO*_pTa#NsV4msw~V={*(p?4#5Xc0$jNWEb&^~U32oFpt- zB*QqG1>Vq~U|^YBnL~GF?;dj~2nXZvVNcqHgIV~Yh(onF?GuM=?6xb!AumK>hZs@G z5r-Uc$VDdl4NemKp5f+^lEb1+EL^wf3>2uY>WZgBqFt69Y*ri?-2mgJJH-x_b_Xjw z&w_sp*RKwXu1p;0HsvzezRwGo)g3Nv;_@;5iV?xSaZB7*VF+iwH}HZgdXJ7x(~L-> z`5-P3VQ0vYpGvi4K{YeSK0bo{|&|V*%V3H~*Y4cYM=Jm3S8qYH7U||=MJm3C=jy5gN2?lGTWJo@f zNN1xrMgTi7?`AwN!*bc@+ZYPfiV9Mn6q}bCGm4L(SzQ={QND`{l*uY!nBB?17e=CA z9;2tzA;Ai!VyzAr63m4Jb0NW8NU%TH1nc6V{FaD)j%W6JHXO$@>jcki+-aVf8<8E) z%<;?~k!Kcoo_`b1ESb#wA^bU>L@XXBLlydgEo|@adOk}DxgPgMbeCPwiA?5&i7SB-Cu~dvc1=J)lA-|Umc#H2k<^ZuX7n3Y zIZN*4TBX`GmTSKaEHbl5?VK%`zy(BjT)zqK3q7Y(=QnYF6X!Si1Ncqwc&yug6E`9| zzlrmkJfhzuxbbC=WacG9>MzD4ngruf6tajVQ#uTzVCoG=cfIU!XZB9|O=kYX?&nVi zlhNb@`c14nZTn3ovuph(ctL;s$`9wG;oSF}-^BS%+y(vL;4INRjs(NUV~ev=L0@$z zlI||D1H;DLmZ|2LyFSq_4Em0%d*Y*Q98P%&Ya@#H6PTVWq`O4f@???8SYdc~2Slqt zoWT~!>_8H8WzoUWZokkVCRJ zv2JV?3B~J+`tD;WKZM2U!ANGzm)JsVyqf8F~f28(a4)OYMx`1%o3 zdUiMWl+AlQsJQF%o6OjT-P(+7Y+iG@A1_s7sx?bFHZj$MqMfnq1>N}SQhPvN;rl7u zcle@D{ftf`O?#?X1(p`lCogGM!Y|Jt;r6{a`iQECDe34U59$v0xzy$RPN$A8((rmX ix`?BT{9frI=oRcXUBr#ZjxO@%PyY{%{0-$MQ~?0JZc1qY diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/contextnoprovenance/datasource b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/contextnoprovenance/datasource new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/contextnoprovenance/otherresearchproduct b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/contextnoprovenance/otherresearchproduct new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/contextnoprovenance/publication b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/contextnoprovenance/publication new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/contextnoprovenance/software b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/contextnoprovenance/software new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/nocontext/dataset b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/nocontext/dataset new file mode 100644 index 000000000..54552697b --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/nocontext/dataset @@ -0,0 +1,10 @@ +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055868909,"id":"50|od______3989::02dd5d2c222191b0b9bd4f33c8e96529","originalId":["od______3989::02dd5d2c222191b0b9bd4f33c8e96529"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[{"value":"10.4185/RLCS-2018-1243","qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofcollection":"2020-03-03T13:05:26.091Z","dateoftransformation":"2020-03-03T13:06:53.161Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:26.091Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/562","datestamp":"2018-01-23T15:06:07Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Gallardo-Camacho, Jorge","name":"Jorge","surname":"Gallardo-Camacho","rank":1,"pid":null,"affiliation":null},{"fullname":"Trujillo Fernández, José Ramón","name":"José Ramón","surname":"Trujillo Fernández","rank":2,"pid":null,"affiliation":null},{"fullname":"Jorge Alonso, Ana","name":"Ana","surname":"Jorge Alonso","rank":3,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"COVID-19","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"yihadismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"viralidad","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"vídeo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"propaganda","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"terrorismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"terrorism","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"jihadism","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"virality","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"video","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"5903 Ideologías Políticas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"6310.13 Terrorismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"El individualismo como estrategia","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Individualism as an improvised strategy","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Se analiza como el Yihadismo distribuye su mensaje propaganístico en You Tube.\nUniversidad Camilo José Cela","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"MediaLab - UCJC","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/562"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055868954,"id":"50|od______3989::05d8c751462f9bb8d2b06956dfbc5c7b","originalId":["od______3989::05d8c751462f9bb8d2b06956dfbc5c7b"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:26.079Z","dateoftransformation":"2020-03-03T13:06:53.239Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:26.079Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/227","datestamp":"2017-11-15T16:29:55Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Rico Pérez, Marta","name":"Marta","surname":"Rico Pérez","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"coronavirus disease-19","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"agriculture","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Retos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"I Congreso Nuevos Retos en los Eventos Deportivos","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"I Congreso Nuevos Retos en los Eventos Deportivos, celebrado en la Universidad Camilo José Cela.\n\n\n\nDerivado del proyecto de investigación \"La situación de los eventos deportivos en España a través de su estructuras y de sus herramientas\", tiene como finalidad conocer la situación de los eventos deportivos en España. Se realizó un estudio descriptivo holístico en el que se investigaron elementos culturales, el perfil de los recursos humanos, la estructura de los comités, entre otras.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2015-06-20","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/227"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2015-06-20","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055869097,"id":"50|od______3989::0f89464c4ac4c398fe0c71433b175a62","originalId":["od______3989::0f89464c4ac4c398fe0c71433b175a62"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:26.069Z","dateoftransformation":"2020-03-03T13:06:53.54Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:26.069Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/802","datestamp":"2018-11-14T15:30:47Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Ciencias de la Actividad Física y del Deporte","name":"Ciencias La Actividad Física Y. Del Deporte","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"fishery","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"sea","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Impacto físico","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Proyectos de investigación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"2411.06 Fisiología del Ejercicio","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Investigaciones en Ciencias de la Actividad Física y del Deporte CCAFD - UCJC","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Blanca Romero, investigadora y docente de Ciencias de la Actividad Física y del Deporte en la Universidad Camilo José Cela, describe dos de las líneas de investigación que se han desarrollado en el departamento CCAFD de UCJC: el impacto físico de las diferentes actividades deportivas y los métodos de recuperación más eficaces tras el ejercicio físico.\nBlanca también destaca la importancia en el ámbito de la salud los estudios en Ciencias de la Actividad Física y del Deporte.\nCiencias de la Actividad Física y del Deporte en la Universidad Camilo José Cela","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2016-03-21","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Universidad Camilo José Cela, Ciencias de la Actividad Física y del Deporte","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/802"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2016-03-21","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055869215,"id":"50|od______3989::1729c3988199b95d1d566851af7d3c55","originalId":["od______3989::1729c3988199b95d1d566851af7d3c55"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:25.653Z","dateoftransformation":"2020-03-03T13:06:53.764Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:25.653Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/801","datestamp":"2018-11-14T15:19:38Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Ciencias de la Actividad Física y del Deporte","name":"Ciencias La Actividad Física Y. Del Deporte","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Actividad Física y del Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Prevención","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Hidratación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deportistas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Electrolitos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deportes de resistencia","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"2411.06 Fisiología del Ejercicio","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"La importancia de la hidratación en competiciones resistencia, por Juan del Coso","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Juan del Coso, investigador y profesor en el Grado en Ciencias de la Actividad Física y del Deporte UCJC, presenta el proyecto de investigación que ha realizado el departamento CCAFD de UCJC: la prevención de desequilibrios de agua y electrolitos que se dan en los deportes de resistencia para que los deportistas realicen su actividad de una manera más segura.\nMás información: http://www.ucjc.edu/2016/03/investiga...\nUniversidad Camilo José Cela, Ciencias de la Actividad Física y del Deporte","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2016-03-21","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Grado en Ciencias de la Actividad Física y del Deporte UCJC","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/801"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2016-03-21","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055869487,"id":"50|od______3989::2e3f34ce90520fae350a7e1148d7dcea","originalId":["od______3989::2e3f34ce90520fae350a7e1148d7dcea"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.912Z","dateoftransformation":"2020-03-03T13:06:54.218Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.912Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/800","datestamp":"2018-11-14T13:43:21Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Actividad Física y del Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Cafeína","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Efectos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Alto rendimiento","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Competiciones","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Beneficios","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"2411.06 Fisiología del Ejercicio","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Investigación - Efectos positivos y negativos del uso de la cafeína en el deporte","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Proyecto de investigación que se están desarrollando en la Universidad Camilo José Cela (UCJC) sobre la cafeína y sus efectos cuando se realiza deporte. https://www.ucjc.edu/2016/08/la-bbc-s...\n\nJuan Del Coso, director del Laboratorio de Fisiología del Ejercicio de la Universidad Camilo José Cela, nos describe cómo está siendo la investigación que desarrolla con deportistas de alto rendimiento y el consumo de cafeína.\n\n\"Nuestra investigación se enmarca dentro del campo de la fisiología del ejercicio y el objetivo que tenemos es investigar los efectos que tiene la cafeína en los deportistas, principalmente saber si es una ayuda ergogénica en la mayor parte de los deportes, pero también conocer qué perjuicios existen para aquellos deportistas que la utilizan para incrementar el rendimiento deportivo\", explica Del Coso.\n\nY es que los datos son muy llamativos: tres de cada cuatro deportistas de alto rendimiento utilizan la cafeína antes de la competición y muchas veces se utiliza sin saber los perjuicios que puede tener para el deportista. \"Lo que estamos investigando es cómo una sustancia, que te puede hacer rendir mejor, cómo puede afectar negativamente cuando termina una competición\".\n\nPero esta investigación no sólo puede extraer conclusiones interesantes y útiles para los deportistas profesionales, \"investigando el deporte de alto rendimiento podemos conocer los beneficios y perjuicios de la cafeína que también afectarían al deportista amateur y hacer un deporte en general más seguro\".\n\nPara el desarrollo de este proyecto de investigación están colaborando el Laboratorio de Fisiología del Ejercicio de la Universidad Camilo José Cela y la Agencia para la Protección de la Salud en el Deporte.\nPara el desarrollo de este proyecto de investigación están colaborando el Laboratorio de Fisiología del Ejercicio de la Universidad Camilo José Cela y la Agencia para la Protección de la Salud en el Deporte.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2017-10-06","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Universidad Camilo José Cela, Vicerrectorado de Innovación","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/800"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2017-10-06","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055869498,"id":"50|od______3989::2f4f3c820c450bd08dac08d07cc82dcf","originalId":["od______3989::2f4f3c820c450bd08dac08d07cc82dcf"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.806Z","dateoftransformation":"2020-03-03T13:06:54.229Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.806Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/797","datestamp":"2018-11-14T12:12:23Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Cuesta Cano, Laura","name":"Laura","surname":"Cuesta Cano","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Comunicación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Marketing digital","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Medios sociales","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Marcas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Empresas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Emprendedores","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"II Liga de Debate CICAE - UCJC","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"6114.13 Marketing","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Laura Cuesta - Liga de Debate CICAE - UCJC","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Laura Cuesta, profesora en la Universidad Camilo José Cela (UCJC) y Especialista en Marketing Digital y Medios Sociales para marcas y emprendedores. \n\nEn esta ocasión, Laura Cuesta, nos explica el significado de redes sociales y nos habla sobre el uso que le dan los usuarios.\n\nII Liga de Debate CICAE - UCJC http://ligacicae.ucjc.edu/\n\nLa Universidad Camilo José Cela (UCJC) y la Asociación de Colegios Privados e Independientes –Círculo de Calidad Educativa (CICAE) – organizan la segunda edición de esta gran liga de debate académico. El formato es al estilo inglés, similar al de la Oxford Union, donde algunos jóvenes se reunían a debatir sobre temas de la más candente actualidad y polemizaban con la intención de formarse en tan valoradas habilidades.\n\nLa Liga de Debate CICAE-UCJC es una ocasión única para los alumnos de ponerse en contacto con herramientas tan útiles como la oratoria y la comunicación efectiva. Pero, sobretodo, una oportunidad de fomentar el pensamiento crítico y la socialización entre alumnos con espíritu emprendedor. La Liga de Debate CICAE - Universidad Camilo José Cela se diferencia del resto de torneos en que potenciamos que sea eminentemente formativa. Alumnos y profesores reciben formación en comunicación, oratoria y debate por parte de alguno de los expertos en debate españoles más reconocidos en la escena internacional.\n\nDatos de la III Liga de Debate CICAE - UCJC http://ligacicae.ucjc.edu/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-10-23","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Universidad Camilo José Cela, Vicerrectorado de Innovación","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/797"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-10-23","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055870582,"id":"50|od______3989::752fd0b2bbac1ea1cc50e52fd46eb663","originalId":["od______3989::752fd0b2bbac1ea1cc50e52fd46eb663"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.561Z","dateoftransformation":"2020-03-03T13:06:55.716Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.561Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/798","datestamp":"2018-11-14T12:57:26Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Psicología","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Alcohol","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Dependencia","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Proyectos de investigación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Alcoholismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Realidad virtual","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Psicología conductual","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Investigación - Intervención basada en realidad virtual sobre la dependencia del alcohol","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Proyecto de investigación que se están desarrollando en la Universidad Camilo José Cela (UCJC) con tecnología aplicada al abandono de la dependencia del alcohol. \nRosa Jurado, profesora e investigadora de UCJC https://www.ucjc.edu/, explica cómo se conjuga el alcoholismo con la realidad virtual. \"Mi investigación consiste en desarrollar una técnica de intervención o rehabilitación que tiene como propósito fomentar la capacidad de inhibición para que las personas que tienen dependencia del alcohol sean capaces de controlar su conducta de aproximación hacia aquello que tiene que ver con el consumo de alcohol\".\n\nEn esta investigación, liderada por la Universidad Camilo José Cela, están participando, el Instituto de Investigación Biomédica del Hospital 12 de Octubre y la Universidad Politécnica de Madrid, con el CeDint. \"En nuestro equipo de investigación pensamos que, además del deseo de consumo, existen una carencia clara de capacidad de control inhibitorio en este tipo de personas\", destaca Rosa Jurado.\n\nLa realidad virtual sirve, según explica Gabriel Rubio, jefe de Psiquiatría del Hospital 12 de Octubre, \"para dar un paso más, para ver cómo reacciona un sujeto en un ambiente que no es el real, pero se parece mucho al real\". El objetivo final de estas acciones es \"mejorar la capacidad del individuo para poner en marcha mecanismos de inhibición conductual, para que el sujeto sea capaz de decir que no\".\n\nY no acaba ahí el proceso. La fase final del tratamiento se enfoca en que el paciente \"sea capaz de reconstruir su vida\" a través de programas de entrenamiento de habilidades sociales, asertividad, familias... \"Juntando todas las áreas de intervención, la abstinencia se mantiene mucho más tiempo y las recaídas disminuyen\".","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2017-10-05","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Universidad Camilo José Cela, Vicerrectorado de Innovación","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/798"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2017-10-05","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055870750,"id":"50|od______3989::7fcbe3a03280663cddebfd3cb9203177","originalId":["od______3989::7fcbe3a03280663cddebfd3cb9203177"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:25.652Z","dateoftransformation":"2020-03-03T13:06:55.95Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:25.652Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/796","datestamp":"2018-11-05T13:36:54Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Comunicación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Redes sociales","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Comunicación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Información","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Desinformación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"III Liga de Debate CICAE","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Instantaneidad","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Información periodística","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Fake news","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"5910.02 Medios de Comunicación de Masas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Jorge Gallardo - Liga de Debate CICAE","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Jorge Gallardo, profesor de Derecho en la Universidad Camilo José Cela (UCJC), Doctor en Comunicación Audiovisual y MBA en Empresas Audiovisuales. Subdirector de Espejo Público en Antena 3 Noticias. \n\nEn esta ocasión, Jorge Gallardo habla sobre las redes sociales, cómo han transformado la manera en la que nos comunicamos y nos informamos. \n\nIII Liga de Debate CICAE - UCJC http://ligacicae.ucjc.edu/\n\nLa Universidad Camilo José Cela (UCJC) y la Asociación de Colegios Privados e Independientes –Círculo de Calidad Educativa (CICAE) – organizan la tercera edición de esta gran liga de debate académico. El formato es al estilo inglés, similar al de la Oxford Union, donde algunos jóvenes se reunían a debatir sobre temas de la más candente actualidad y polemizaban con la intención de formarse en tan valoradas habilidades.\n\nLa Liga de Debate CICAE-UCJC es una ocasión única para los alumnos de ponerse en contacto con herramientas tan útiles como la oratoria y la comunicación efectiva. Pero, sobretodo, una oportunidad de fomentar el pensamiento crítico y la socialización entre alumnos con espíritu emprendedor. La Liga de Debate CICAE - Universidad Camilo José Cela se diferencia del resto de torneos en que potenciamos que sea eminentemente formativa. Alumnos y profesores reciben formación en comunicación, oratoria y debate por parte de alguno de los expertos en debate españoles más reconocidos en la escena internacional.\n\nDatos de la III Liga de Debate CICAE - UCJC http://ligacicae.ucjc.edu/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-10-23","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/796"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-10-23","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055872001,"id":"50|od______3989::d791339867bec6d3eb2104deeb4e4961","originalId":["od______3989::d791339867bec6d3eb2104deeb4e4961"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.586Z","dateoftransformation":"2020-03-03T13:06:57.721Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.586Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/645","datestamp":"2018-04-12T07:59:27Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Transferencia de Conocimiento e Innovación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Innovación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Investigación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Ciencia","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Tecnología","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"UCJC Open Science Day 2018","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"El UCJC Open Science Day tiene como objetivo mostrar las actividades de investigación llevadas a cabo por investigadores de la Universidad Camilo José Cela (UCJC) a todos los miembros de la UCJC.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-04-10","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/645"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-04-10","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055872022,"id":"50|od______3989::d90d3a1f64ad264b5ebed8a35b280343","originalId":["od______3989::d90d3a1f64ad264b5ebed8a35b280343"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.912Z","dateoftransformation":"2020-03-03T13:06:57.747Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.912Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/795","datestamp":"2018-11-06T15:43:10Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Comunicación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Unión Europea","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Desinformación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Política","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Periodismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Ciudadanos europeos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Democracias","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Estado de derecho","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Derechos humanos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"5910.02 Medios de Comunicación de Masas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Ramón Luis Varcárcel - La Unión Europea ante el reto de la desinformación","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Conferencia de Ramón Luis Valcárcel, vicepresidente del Parlamento Europeo, en el Campus de Almagro de la Universidad Camilo José Cela (UCJC) bajo el título \"La Unión Europea ante el reto de la desinformación\", en la que ha alertado sobre el alcance de la desinformación en las democracias y sobre cómo pone en peligro los derechos de los ciudadanos europeos. También ha participado el rector de la UCJC, Samuel Martín-Barbero.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-10-11","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/795"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-10-11","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/nocontext/dataset_10.json.gz b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/nocontext/dataset_10.json.gz deleted file mode 100644 index cf3c3aa7b6412355f6172a0b70a3929ebdcc2735..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6755 zcmV-p8l2@HiwFqS2%ug717u-zVRL14UokK)YIARH0PS7Pj^j4A-p^AI1_qb`TrNqr z{I{#PDwztplSX$ZKyG>vP!eskB8d(qRpR6}aPtiJ33}O!UJQDdt+#oK`y}@qlCor% z>n~F^Qo1=)=-3u19-i;;_k2A3r4!Mdo~Lusnf}t@=_Sv25VOv7PUDPq;g>m+l11Ae zA{MipMZsD=RXXwoPN+culI; zue%*6H!oJ8E|&9zWjRe&ovG`OeaH95!|}wKK&7!H-}!G5eW(7{pNuEdspCbFA9NK#ovTEy{dC}D+lgZGx1 zU8gs2$Nus2C$kqvp5u;>TyM~atUnYq=5x;EF0hf{d+6z3u0X`2@ACDET?5Vb>FcgF z-Mg7CqK9TH=Eh2toPoFk_Xe=pts)bXd@u;)zN|LT;fb4k-|E@VOv=NL}?e$O4M z9PB4Zt+DtPiW8LAb@z+0?^BkAlCMhDEvdYOhI9(0-7F7{bEc34oB*wWX6&oyPaw}S z&sTr#pZwMP6#lmA|tx9GF)<~c2CH~ z@s}s($6n7l1~I;i!eiCX&>~+7soL5c+MYr8GaAQKM&f8j6B;f>mwXN{U{Iz>P=3t{ zc}*602#9_UotA0M*tDU%`JBhxV60kucc=3#s0N%B!3FdbJ3apF%dgK*kKD=ief*wSyYlQyV0-Yp z_u$B zXlM%#7KSEaxG)nXi$e4Bqk!K>_yvun=GYd?U zzzqAY({uaerJX)Cx<#Gy992`>a5{?#Pa_VU7oq{ckWj!sEXyU$*@EA&-bg4O#=Vsk_W|_k*pw4bVe^2DR&t*?=l!P zAxlcW5e0c!1g!Te{VL^`OlG(tM`kqPu`n)*8H=%-uqfWsRY*gqQ(n67O594fm*ZNU z-o4}Ntcn17x+V5v+UdlSfXannX{6hV{xuVY3{A^He^<_a%FQ?Bh1H)!k*emA8z*6j z%~KX}`h*7Lh+J{vTYzruy0r4O%TL1*LQ<(baH#eGeZSBsvP|;DGOrS)6-@8}y_@lK zwpAeh%+t)YB!DaCA@pcFt;%?aL&<0k-5B(U2}q_K#3pri9ITJh@F{iV(2%0mx$;Zf9g(x*A?(r9m$U44fe|Zl*h@Tx?|rF zD*yC%rIjA8{-J~*#8PAmc@*G5NkQFCA!DQ~8T(yEac@b}C}us7=lBYP0>>6Jw4;!* z2uor~=L&)8!9}&B+QW}(QlPo>bP+CLG-EVD20_^|?mrC+iPc}1&!y@kSyYiOG7u8F z+xmzYu}dCS<0Joh`&rHQO3H>N=au9eP0}l$l)st`X^{lgIK05^n9>Zt{pahy{9^Hc zC_bb=xRwv;N8@nhyMv)Op9H};@&ac#@rTho2>oyrbZ+58-j@A8$cMZKwQe-IC$-M& zPacz6r{!syTIYGA&J}9i=*V@CT>qszoO+X~?>{cJ?l~?epKabsB(^=JGOrQJ%C8lr zOs8hQ|0-()w%ZtWmQr?@QWgp+QhLdur^IDHgF0J+S;L!ibRl^N##2aUfoKwl=PVc2 zH`FKuFe896Z98a*L+G5$M7qGmE%~Bgmm(vojo@Bz4F5TU*Wg!;8o#`$^;i!=8zH~D z2jSkP&?OI0DUgjmBI2EP!166UOYK|{WIpOROd zCzOC%3iBOOpdic$5^V*+($8SRwfPq%l)&8na`P6Sqq75Dn?(piFU`nW|CW&@*awIQ z@1fbNP8W=mrA+%x=D0>u6$nKN>urhkx~C95@`4e`&>ORPPl^XnW0aJdG)sz79IQSW1#)9Pznj$hFbXD^*8B~zx zpc;4sOd=T(psK8AcW55am*XEf!z0hJI&Ft9$NC{YWW%w3h!*%EaJ#9^ehAwmTR(*L zLp-1#V&Y6j*ZLvm4q4#!VNKY4xaEwH^cx>`1hUZ;E_-G-?$v@k{PV}LQd665!OrC0xnhBPhnvOgR z#-!S@GOyj{d$N;>|Ng7W^-T}5)U#Edj>7`s0e?fPv%Zgi$M-Q;s|b5ArqK=FjM+o0 z%~HV==tNT9=&K3jAv$hGrPLKX@O}H7;oUZFjazK8`ci`0GS%?&n}n@dB|Q7`3cXwZX0&_)x{<%n)6y2(?jf7MNAwpQf zFf^Han$I0vTV3PX3E=(0yf-89=3UC@fu6r~^(-m+lQJqLEHRB2jiFO#g&Zm6_%X%{ zuR!aa;$yx$zL%v%T!9bEEkes0vC2WE++XMfN;YqEVWdRA(TYhURkC>l&1()C2q7c- zhOjtRc8Fzxy;rHxHOLt7&0AH8oWc*FCv6-Q6>O@rc>}3(BPYoKu1LGkv{UR4v}lmv z;W4xugb(Nj8{(#fXUX3!af4OZJoJ``h1VX4`q;6xj>uLxB5}jH&5lSrCR#_Nbwoa- zBhqvI>l~5p$eW-CaqLbe!NiZ;$R7^JzDwt$s2}?NEsn@rv$xO@>Gy_y|DKM>(eUv+ zkz0A1_e8okdLrG)v_H0DeN zK!DY(4QJUb5N9~+mLlf%@MtTV1T(&?4BCd8W?4`ylfuGtE0f~Mu6I^1O_sGu7rIG9 zk*rwG4cVgB$hriwlFtj}vyRD3WH$h7$rNogiMx|2c3|EmQ^3*{-als(*ccwfB{v@iM#{vJnp3r^yqk#=nyEIs^nE7HCfX#hLAfeDk=u%E>cPonli|zxE2{(6cW!xKgDlg zaJXBiEQ!wSeEqg5#xU=>0vOw_Vl&K`t$xEWE02toH=EE-=QPsx`S~vib8cR?&6W6;J>0!|G-0@?P9x8d7Cq0;u7Y9iX{pp}T_1s2D4_`wM1Wi7~-bn=m zBj;%1zu@lOUYL zF!h7FMO?+)v<|^$JEppa&qC286rZpZ&)ag_L0Jleh35_whVxyg@DVZNJ5e&IFwAR` zvIW(j5yK)YkDiBSMphlOzj>ByNepc?io25-p54qQ$QXto*u1I7U`X#4%AEWSx3B(A zG7-pK(^{+e;>=iOSg1E!|k&Lb%d|$`C7-=2C}`4=I?QsoQ+7?p5RpRiX$k zn2NlPBIhvjCMpO;E#SHiOJT56sZ?C}y{==A?tu&zo`!ry<3vRm$!NeNC=B0?j1Rgl zBFHEgAlcg#w@S2yC3FoKB7=pM0oqHdNf+b0iz>cG8H)ag0+5177loQM5cZQ|qi8A$ z1&QN*E{fu(P*UU^&CjF+aV5649I6b8sbjN^XOCg=*-tCR z%QDpNHb?=c6UIfR>3lrJP3cUlbWJwDfx_UEQp3LGXp$G?({~QM>g?)aJANKSawQ_w zyta2Jz~^8)*+mLj1{&tR%> zI3HrItk3uXsc54k*Ew>AR;O*FqWzmrGK;3AqJ8YxS}IygsAx{Bsc3cxv{W=pMSD;x z+GKn!6>UD4_rtO0gah9RB4-@Y5dLvSA^aGH^Uf_)wA-?`l8QEVhWDkSd5?A>c_mNN zRJ76Lz=h;nDw;QiKd=9|RJ2*4woGO;6Vd1Et?6u1X! zgM2jZ^n6O?_t3AW3lizB(8AFu983kn&IGiU*b*}uZyTWvyV5XqFTtbX7KIj|kVY&L z(soMCLUDdhp74bV+;PeRiZ`2_&rZ&6z1hTK^Z3{dU3buP`(!(*vT(G6F{V~`#H{F! z{#XODM6fnueRm>Q-3Geb=r!~{Os_RL%T^4h=8Am)IjiMUxs$XeW)At0BN5`~5B1VxF6hq)BKE%y~N*RVGR-A#5vWlyQnV;NQd&mM%f^NP3GZC`| zYU`QgmlQ7~Lf>Y@aQBYudyM|?fBtv2c_+g{(-~%>(iWb>QitZg|$(54bi69kb`W(q_fDMgkEinfGKiYg`$6po^lM59n7ibH7lC59dn z*7t-o{|>=$eL6KRu0!!n0QE#O^a*d|2b=+w87(q(k$E72!A;~V&92}^4%Ji*QijF39IH_bC zdkU;J;VBw`=7pI(WYoxsk+=%MrDC*&Jb<@V(W6Wpw@Rtqy2g#wuj=GOZLlQ`5qcaZ zb1kDujr*~ie+dLp*(el>)Gi+BR8VE;d!d)s6th%a)<sy^#Hqb|jreyCfBv$v8B9PB8AvQ@*b! z3{r#=0afXZ(#XYz^7?m+kBq?x=uau&JK)L5ByfiOz zvo*7W*e6^ho41k1B_{t6*-Gu}xgI9|5(YKw4%hpoKf~LpXgtm%xM0>o$| zP_AwXg)`m(17|O3rMNUSHX*X0p~K5q$W$;WfjtjC1w%wC*EZzW9JiKB5?DrbaZH8(deu=8 znyxlM)O3VF&I_RhR&*y63>ke(N)i_023Jz9@u~(}O`y6}<3;|St{ZZEu%w?g2`%K( z)X%h^Z?LKWT>y=6xp`-9#)CF26eDR)q1I)MVv^(ijfVB4$XIisV~m+Xu|`UVuR1#g zD+&)-a=}%1fYIDvbqK}xU2fiLf|nEvNew+sk)~3*4HQX3n{D0zT-lXAw2NTS02)=Qn*>xWfl*I&LmFw-sNe^T|F%}| zf(p^iJA88lsPME{GmM&LU3?E7N65E2walT95>3k-`f#ze%%PSrhsLdD4%s2lGKVa4 z=pmUyBj34}IW!Lg)~8P28#}{cKa3(4%%gr7OgyLWjxeyyjm)9jvbT>pYe6T9sSameve*gi&7vcw@v9I}y#{)B_Xu4lNurDU;a5)0cbS_1{D zt32Xqk!Y4>2a6R8MmxZ`?o6>orP+~G2jNYSS)6^r9C_ab{MA#ZK#YtW|n96fIKtLaop>8X3=Qk_Tl4j6tHj@^;zJ# z&LCne7_h-)=yq=AncbSbeLORa2sOUneRZSW$a5bvLe;nx_tgythTOYLG;;<=uJ_Uz zP5sH#8(E^6C7RulX!cUItZL;?W@?>mb|v<)vo3}P{iGI%mH=VnCA9?$LzA^2Gzr4X zD$0fhq8(mnwK=UeXI|j>`jC-Rx+XYNw65nr7q)kIJ)gyd9LBvC-DMkeA`w}x<4Rz} z2@?~69S8_WViW+~au}W=5Xuo#kAA~SXUUyht5mthGUc~{NoEqrt+NFa*nkKR>o>u5 zp<{Jw{U+9LV*Mr`f!_qT$J*^Tu|2Z&n^?ce1Nu$8Yj5_5CQj6+?tD0)kvAL!J_}eh zrhPx~#!i25+sz)gW^bk6Wa8fMeEz668jK#%Z(`(W+HW$N9Q2#u0sY}i*PjmhQ`fP6 z6YDpz2lW4hqeOi>5)2==EzSxFZPo5b+OxPdx)~(!6Ff%#PIGGh(>`pgC&w)fkgVqqJ^W`cA-GX$g)VVyRvB51Nj!E zyHX*Z(*$98Qt*s?O=Y-wBSRMB!F=Vht}PV_#p8?W>|(H!p1o|uWW z&Vt>wn%G!Pm!O*f57~2l;u%lMKBF?GiK5GDL&BR=^vx_ZSrw{_IEfRwkm5c>iGW@R zd`S*#Hat-Ns`Ez-7VlcA&e$*U_9G_M=&sKx>-%_6afkEk%vgu*+>ESEUVXS94^?BT z6^l7mF;<lg(5phuEO?76{gUNyGj1TgX_&w2T zDPm4J58y-fn+K^>F*Y1T+!zV19GI1ni$v*k5@==28IQi2Xptx*Q>9l>oU+DzA+D0y z1aq+vRB;%KbW*i_LT#HePhR6CxxW3jbpY+ASpw`5sayz?@?!T(=3;IvT^@fv)JM6o>7rMhyr!(k}hTU*5823Bwv@g2dz#Re+O3Rsyd2|~p z_`|_9i~kY0qM}FyAegCE3#j6ogRjI2XiAj=?|Szulp|tMebDd*i4mP@T$bP?Ro94`^fE0yz!(raz^7J zmJp92w{07nQ(V@S?3km6Q7jmUS z;)X67xZb2YM#2^%<;eAWpsOa~G@x>t&d$ypB0vXo3Egy*o*mymxqaMmT%_)c zFgPYH=2<#dnlxkzV$0S3V-?DHu3GFa*OH_BZ=C;#Le9V2xQLB+q56HjxV4R6{(pN+wI52Tx34tS;l7@jf3&pX!;b)SvJdr z5nI|XD%sv{_4Qk{2w9wh5u_1fGXF~$iX`@8^q+#d$v}o2EBiX*QLeCc^1z6kDIIQ* zSfyUVpQ5?vFsI1!ZL67qt~LYV$SGx~RVvPZIR7ujWSokb2D{KmaWA9@qY)-nWKC@> z-e;M?Y1go%9F>Ko83X#-H@sHVm(8> zFB@4JfnP`98v4HEaT?OvgH}lOwMk78P8gqQ8Nk@eG-sd-=TBjTi=|BYQbf>NDj0M_ z1rkO)<1AaSK*e8zsi(Es66Wdir%NFXixm?A^bICZv4AO7#qb>~B%2XL+4)bXZ=h$1 z<}*|`&_5BM{{Vl+@V}e}Oiy66(-&+)Whtg6FxjN=@7Ji8YLS3!7ofb6VxX5uCoCq; zpTcwm<)iG0!)^f`(A*5WV3*vqhL!;>#$qf^ zb-*e}VuiNpoOen?d+hwj7F8;NUo9fk2&gwupqGrVjkO76#%=}@hGBxnbtW{ca{-n_ zpzl}-at&6%2Dm5}a;A*)dHjek{PQ0dSj!T-0DQu%d`+705fW0ll$0SvrY=>4KP~y6 zs0A<)3&Lo{Mi>WL9?J!{japHnc(6W+qJe}KDujf8Ko=4Y6|n%gqIbx1nYT@d;|g84uA+?fdf>Kpj)V2+?V;+ z0My|8DexCHV8P=oMHM1KfGnUMRAJ$;`x zCz&5fGZ*3Su?esijp4n9;k8>!9zdVcVsluSTG(U_+Ai#~yoRS&UxOYL3aRV7gw=@- zGSe{f8jKVRpX(V+xuIPa*sZHknGW(r0`8Mj@i@(Yk#EHQ4Rd~~rJtp=mA9K)DrmR2 zwyFS{)wJWp96nlS7zoB_Ih&_drZ{in8|%Hw9SToG3>KE$2rwlqvRJ0Dkv$IPV9;l( z4Ry<~?Trs`ltFm`+w$mvC}oH(%Gl(C4(g61%u1%DZH-O2T4l*5~!!Z~7JdB(O>T!hAzC4;1yR#{?G zQl(HQ|M>0Sf76KS<$x32LC3ocobdc`JoLGkdgIU!JE7n0xLx1#I=(;Z`aypOII*+% zo`4hMw%ciU`Uo|2-ARAojK{s}fD`?Z(;IYO9yoCsJ+B#@==Z$-MR39r*L~!65uE6B zUoANCAuvCL9aD?lzB~AJN1U{Y%~G6vS;6aS@o!Uy+$HndW${ey4FVitJVnSDqHAX ztA$5GJ{A(U@^?5`pmeP>RZ7)ZQJ`$Ef35Z&NFPcA+wPly>#D8dXVp-ivutOur$CX2 zXB$o5if6bHd&MZx@6dCSu9%v#NKQoqGVH^8pbiz?K$mHZ_-)((efuH3tq@UNj(__f z>(Q*nC4)JSfh}TMMFJD&0#*uhN{-|b*66I*={MNXUTMPBv7_78VNRtd>=7sfV#6>D zQ&b344in%yTK?a^|MzzRE8IFO2J2|pJ%lUD6I6P}5|v~T&5mr&4fBPF8SK=~P@Td| zU_q)@&SG0B(8^QHX>xqZR_;rwqE5 z&N)()!6)EHL_=zhXvCB_2c{C{e@3YiVL1bRL%7Rf_sM|b!WpHCzz_+G%Y{J+aM+ox zo(QT9cGiJ_oFVoSE91<-s@K z*74}b%GH{jLsigAsk!~fpF|z$?Gg4nAF2p`+!e6!521f2n7<;qKW5c=nDq{v%@s>v zH;KBWjk{_@>_%#;d$`R+ekY=9qa8Wqn!5TRYCS9}wX0|6V{IE8OVAF$2{i&cb_ z_2%v}u)aklp6hgqT6~UmJM5lXQw!J$rg$zZ;89(UmMK$MQHJpU+FFy6uY^rwLn!YX z3K3{D62P(g%*Ko$a4*N4Q#O~gIRgtQA{vTo3f=-Gkinb*QDB=2Tpam!CUNcqTP0x7 z5hiKP9PCh&)c%2x9cp`;<)(oKs0xA%Dc354cCEZ3+Ox8IFwoH%oR0^vDs`34)a1`{{0Ehgg-@AiOl(LJY-jog1= zz?s~W-FF7;H-#@YRVip`jW#5uGjRG<7W~*|`Db#iQ>BAG?4?h%S9};N~k}Mpwy}B2tM35J4<$Mqi-PndS?N9_4FI z>n-M5a08wFI+NP+3x*_5Q_P}b#IV{dhk>?&SSWpaW2Pj0dig-dD^JvenEAGS+#^+; zuMPJKT(U}m+gTU_f-aT^7TyDP*b6&xT9e%2J@${R-|e(I-uNgNK+HiRD+4(SU;{gg z4sq2E#R9pL%w@2~&cm5N_k4>jxRxk&gikWlVj|3f-0zM{EF>i-BM>dTU>8uDFLgRc z?E>`^M4E4<6e<|J`$pe@+E|U? z7Ja%Pa+n??fulghq9*@B%q#ZGcYkI+2V)S=jzevoJ3uR#?>56Uys+n!I;XIkL|k16 z$^vSk>5m0yxLzF-td2Jft?m-|)Mf-fVR70Bm}4Ic7s!-S;X3MP^@zYdLdN10EiOo) zhM-pr+rsTaGFFMK^lH7P(*QZJmnbk}#aV^I|$+F9ze6gFI@JbYk~iMAz9P#>C7f|eU_ z?Ty?FY&^>E9=KKs6MtMTs_nvo9|@mXA4y&G)?If|Ii4@cT5Xfk=1dZMgv;bd5Ma2& z0(e8L?6fMHuNq(;fW<1FsY1I0)~0s4D?iLoSPsY@6@7SV@~><6s87hDb=C-iDI31O zX1-7PqG9}-V0?8k_G;*S&@r$tNE@vE{lMBg@4`LWW@Xf1?eCxy8 z+bHX+AJVv~f~@z{Y8w+6F%U0$^SJ)VE@%L{%J5y)n2uPWn?bR|kbn|oSli%jYI;t#cw3j~ zmFCE4JUlNEPjnlv*{{J9YbQAs^b?1BIbQQMIn67@VXTJFd;Jh;gIc|2)XEp=S85Q$ z2C99J7=K-;wstZ4idU&CaQE*TboZS{cge@U=jd*;PHoWLcU4q7(OqxQy%gQ;^!cbe z>i7fR4gKL@>htk1=!AjS8%)RDT_GHI7T+6mw+fj!z?;jRzIQEj*K@qiD+=LQr{^`J zyWP&%y@>A8i}PNu2;umOUrD|z@x3pZElN-~-^Ue*Zf9J2^dR{Mh?5JzT|)-CP`aPcIJSyW-|$toLX?N3;5# z@8=qG)a>WpRZ;ES&vnM*OZRgyMX(Kzf32V&^^1D}V%uKnDu;(M~6 z8@FBf?|@#S8c#Z7*gsu!J2&WhFZE%pZREV}v%224-JGS(?&jP{&wItYIT4E|7_{Y1 zcK(w&sGg9@_9SvwO9DeBqxHtA(OjCB_uJH}%hCoyNI zJjJlG^(V55WawqE@mf}1^x(xF%X*+fP{9mYdIJw{)L+Zu&%WP`Pg7<;c(H&uAFBm6 z^$D2fxYktpyK0GY9=uo>oO@~sl6#19t4H=BsjatyIS=+d2F@Ei#(fuz!J;{VxIc@7 zwb#Em*yj9j6;5jS^gEZ$QT+W7PYo`O4?X8^e}xQV`UmnHwHQ7lq->ab z3{-*vkg0DQb@(}BbeIxJicXMRnFPN3^s_;9_ze?uOa0J+_tUm8?|aqW4;xqF>gJ* z_4|g4dwtgO3dfkH{43G8yx#|xH@QCc+~wUE_r~S@u8L}>%X`qdG#Er@G!^}R*X<8R zt{3+BaL^MW4+g#ny4{}G>GIxHe1BZto52@G*YbIHo#A-=N`pa+-bgTr?(Sd^y;tk> z4mrD_aGQ7SaN;*NOTM5t`~~AST+avo;z1~`=koqkCCL@zYGBqE^mPBqC=|OK-fycf z`b|ZlxRq&@G-25HVzzF=uwmP2Ig4N2H?HZ4b+xcyj>ve*vSMdcx5%5sh;Nfw$ z@P)n*R-0l&wDG}vK_Sy+9AIPw3@iN+EJSVSbb3CVp0kHd)N`sUpA*NLKr#v=hpQM@ zbv%kzjAt{tv=5bkOu-Or#AS>j{5TJIoQJ5cA_vpk^mUS340wUnaJ^cKTTn!fplHId zAg7oZw0(oxPvpOdaaL@zR7+ch2$47&(B4f~ZioOC8je*~L zRmgF6xE>Osx)!VSMzjcN_rW#{%TcLjty0Hu$CA!7n^-U#I(~_^y7suo!q<&Dcc-tVe+*uFsaFtqtdH9Zt#QrWZT^IL)nJ>~ak(;_pY;YMGVy)dr z6mjPh8{*{FWAsP>M>T!>_~t3nuY3&lfabIop$4ZA8QRxWwgk(mLQ>_CKI#E%Dg(A= zU2N2PX$rd-=3~9hga#Tf7~#v}wOd1Z>$4N;@`;hWi5Brj@Wf%`a(?$NeKdI2yN`E~ z*K^PDuEzK@c-Olss-1Y(uo$1TOTfE+B|>z~dmb+gt~K${uZKiE%=A(&>8-^E&3Nl) z(mEEk<_&Dbw-@5u3Pu$}knt%@o^>zp)JCw5>+zF=D?yMAVD&c2{_21gVhR?eq4>5J zLXy{_ha1dDh}0BT*^ojmb_;75LD*6~7p6rXmr-C@5UCYcYFUY)KDpr$v4s zgJacm&@ruMA{O?6N}dv4)n*p4dP2VQqSp@lM4e%LYK!8*CLN?tum}hv4)VICwgKmv zDaLCBpiy{a>_BMYpu#1;9v;1nwO+IUil_ouo(L<7s^$xpAPOluG&3A=LG0lrCc$AR zRMrM=mKdrT<84#nif?U;zghCsx@Vgmo<4$>fBEju_w9?!HL#pL#1cm>hL9A#h@Q$o zCg@E^)Cjr*8*YoYah8gx%|tTCNY?hzbScglvU$Y@YylhVIbrWG4WZJiyf_1~91|pG zo&1Q&NQ=c-?6nPQimeq-#$&v4i(25`k{v>ow(_GAW`OQMO%#GXpD_pov zZ@%q!SY?|n4rHk5L}=V?S(iUb6(Nonlqf#-33800V;R7`&t)e(eOb=P5D&qa~7 zSq-wL$o7Q*uolnjv%^ux@qTl}D$T^Yakv!}5zn@%8}_q1%2`=w1m(qf3G?S-U+XOo zFfMxtrUI0V9TIb?nj?%x9!=W>=_D2znS#9lzG8PvNm~JF8Gn@rGEXzJhUTC;#Ub4l zyD+8-NDNqrz+OBP6gM5T0)MnI^m|yhs1Cx781mdu9D`X5#$ij0@uGTyu!^4MDrzab zymM`Kv3oFS3n;jP!F(8l?ni-|E6QZDg*YOJxDe9C$mC|OGLR7%;z;G7VBTd!+GrBV zZN4e(Sqq2BnTR5auV7ZeT#?-HcLM9US`-Fkdk*Rm2f>_x?WMB|*c7cf5a66z_3_0@ z5xgwe9Bg3|@BB?6Sfj>?+#pyDg0+tbR%f*D2v%c!8U*WI71b^TYcL*+E<><-BjJw* zes?4Wq7zL0anGN+yf+whN4-(txjPW7oyGST!MfHfWnR}AcrW*rvaR&Ib_B~C>_V_Q zulSj=YXevNfLGnjxW-6m_jr2#G^#fQDg4>@(Iyy2`CkyP`Va%xQRuiUUDTrN{Mnzb z4qI(7@QHw3EVOo)P9}eb^?uqb#O$^wiNyQR%7?;$8@G`whRshLX2+S_&5f{UloWNjP}l;IK&L%z)&v2q#N9Td8!pCVf{=s_uYG_w zi3nyj1gv3q71s*F~POX`m5-r*c-bC>J{v&dg~s9JxSeEANmmLM_8K=zDB= zU*0#}gmLp4a)>+-$K0oXNiVI@UK02QyJrMlcz7d$T*B})QOxnPt`v{YIgMEid8{^Z!r0=SzcEO}Ww|8kQrylqGAPmQY zzCRj{r#=rx13nD9{ zL9jd07F{&1`F)rV>iEj3ouxVY9Kb~KMwWXf%dId=Ayv(40%aU#p5^(EctGE<&!gnSO5_Vz@2ZbAMPf4eXsi?Ee9U`?Xtd`~U#n?eH=H diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/otherresearchproduct/update_zenodocommunity/publication b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/otherresearchproduct/update_zenodocommunity/publication new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/otherresearchproduct/update_zenodocommunity/software b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/otherresearchproduct/update_zenodocommunity/software new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/publication/update_datasource/dataset b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/publication/update_datasource/dataset new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/publication/update_datasource/datasource b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/publication/update_datasource/datasource new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/publication/update_datasource/otherresearchproduct b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/publication/update_datasource/otherresearchproduct new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/publication/update_datasource/publication b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/publication/update_datasource/publication new file mode 100644 index 000000000..12b385b22 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/publication/update_datasource/publication @@ -0,0 +1,10 @@ +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055113942,"id":"50|ec_fp7health::000085c89f4b96dc2269bd37edb35306","originalId":["ec_fp7health::000085c89f4b96dc2269bd37edb35306"],"collectedfrom":[{"key":"openaire____::ec_fp7_health","value":"HEALTH FP7 Publications Database","dataInfo":null}],"pid":[{"value":"10.1016/j.fgb.2012.05.007","qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofcollection":"2016-02-11T18:10:02.893Z","dateoftransformation":"2016-03-08T16:42:14.571Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2016-02-11T18:10:02.893Z","altered":true,"baseURL":"file:///var/lib/dnet/health_crossref","identifier":"","datestamp":"","metadataNamespace":""}},"author":[{"fullname":"Lewis, Leanne E.","name":"Leanne E.","surname":"Lewis","rank":1,"pid":null,"affiliation":null},{"fullname":"Bain, Judith M.","name":"Judith M.","surname":"Bain","rank":2,"pid":null,"affiliation":null},{"fullname":"Lowes, Christina","name":"Christina","surname":"Lowes","rank":3,"pid":null,"affiliation":null},{"fullname":"Gow, Neil A.R.","name":"Neil A. R.","surname":"Gow","rank":4,"pid":null,"affiliation":null},{"fullname":"Erwig, Lars-Peter","name":"Lars-Peter","surname":"Erwig","rank":5,"pid":null,"affiliation":null}],"resulttype":{"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"eng","classname":"English","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Candida albicans infection inhibits macrophage cell division and proliferation","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[],"dateofacceptance":{"value":"2012-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":{"value":"Academic Press","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"embargoenddate":null,"source":[{"value":"Fungal Genetics and Biology; Vol 49","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by/3.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0001","classname":"peerReviewed","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"openaire____::1256f046-bf1f-4afc-8b47-d0b147148b18","value":"Unknown Repository","dataInfo":null},"url":["http://dx.doi.org/10.1016/j.fgb.2012.05.007"],"distributionlocation":"","collectedfrom":{"key":"10|doajarticles::8cec81178926caaca531afbd8eb5d64c","value":"HEALTH FP7 Publications Database","dataInfo":null},"dateofacceptance":{"value":"2012-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"journal":{"name":"Fungal Genetics and Biology","issnPrinted":"1087-1845","issnOnline":"","issnLinking":"","ep":"680","iss":"9","sp":"679","vol":"49","edition":"","conferenceplace":null,"conferencedate":null,"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055114128,"id":"50|ec_fp7health::000b9e61f83f5a4b0c35777b7bccdf38","originalId":["ec_fp7health::000b9e61f83f5a4b0c35777b7bccdf38"],"collectedfrom":[{"key":"openaire____::ec_fp7_health","value":"HEALTH FP7 Publications Database","dataInfo":null}],"pid":[{"value":"10.1096/fj.09-145573","qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofcollection":"2016-02-11T18:10:11.811Z","dateoftransformation":"2016-03-08T16:42:14.579Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2016-02-11T18:10:11.811Z","altered":true,"baseURL":"file:///var/lib/dnet/health_crossref","identifier":"","datestamp":"","metadataNamespace":""}},"author":[{"fullname":"Tamassia, N.","name":"N.","surname":"Tamassia","rank":1,"pid":null,"affiliation":null},{"fullname":"Castellucci, M.","name":"M.","surname":"Castellucci","rank":2,"pid":null,"affiliation":null},{"fullname":"Rossato, M.","name":"M.","surname":"Rossato","rank":3,"pid":null,"affiliation":null},{"fullname":"Gasperini, S.","name":"S.","surname":"Gasperini","rank":4,"pid":null,"affiliation":null},{"fullname":"Bosisio, D.","name":"D.","surname":"Bosisio","rank":5,"pid":null,"affiliation":null},{"fullname":"Giacomelli, M.","name":"M.","surname":"Giacomelli","rank":6,"pid":null,"affiliation":null},{"fullname":"Badolato, R.","name":"R.","surname":"Badolato","rank":7,"pid":null,"affiliation":null},{"fullname":"Cassatella, M. A.","name":"M. A.","surname":"Cassatella","rank":8,"pid":null,"affiliation":null},{"fullname":"Bazzoni, F.","name":"F.","surname":"Bazzoni","rank":9,"pid":null,"affiliation":null}],"resulttype":{"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"eng","classname":"English","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Uncovering an IL-10-dependent NF-kappa B recruitment to the IL-1ra promoter that is impaired in STAT3 functionally defective patients","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[],"dateofacceptance":{"value":"2010-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":{"value":"FEDERATION AMER SOC EXP BIOL","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"embargoenddate":null,"source":[{"value":"The FASEB Journal; Vol 24","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":null,"accessright":{"classid":"CLOSED","classname":"Closed Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0001","classname":"peerReviewed","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"10|doajarticles::8cec81178926caaca531afbd8eb5d64c","value":"Unknown Repository","dataInfo":null},"url":["http://dx.doi.org/10.1096/fj.09-145573"],"distributionlocation":"","collectedfrom":{"key":"openaire____::ec_fp7_health","value":"HEALTH FP7 Publications Database","dataInfo":null},"dateofacceptance":{"value":"2010-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"journal":{"name":"The FASEB Journal","issnPrinted":"1530-6860","issnOnline":"","issnLinking":"","ep":"1375","iss":"5","sp":"1365","vol":"24","edition":"","conferenceplace":null,"conferencedate":null,"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055114168,"id":"50|ec_fp7health::000c8195edd542e4e64ebb32172cbf89","originalId":["ec_fp7health::000c8195edd542e4e64ebb32172cbf89"],"collectedfrom":[{"key":"openaire____::ec_fp7_health","value":"HEALTH FP7 Publications Database","dataInfo":null}],"pid":[{"value":"10.1097/FJC.0b013e31828780eb","qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofcollection":"2016-02-11T18:09:48.436Z","dateoftransformation":"2016-03-08T16:42:14.58Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2016-02-11T18:09:48.436Z","altered":true,"baseURL":"file:///var/lib/dnet/health_crossref","identifier":"","datestamp":"","metadataNamespace":""}},"author":[{"fullname":"Ford, John","name":"John","surname":"Ford","rank":1,"pid":null,"affiliation":null},{"fullname":"Milnes, James","name":"James","surname":"Milnes","rank":2,"pid":null,"affiliation":null},{"fullname":"Wettwer, Erich","name":"Erich","surname":"Wettwer","rank":3,"pid":null,"affiliation":null},{"fullname":"Christ, Torsten","name":"Torsten","surname":"Christ","rank":4,"pid":null,"affiliation":null},{"fullname":"Rogers, Marc","name":"Marc","surname":"Rogers","rank":5,"pid":null,"affiliation":null},{"fullname":"Sutton, Kathy","name":"Kathy","surname":"Sutton","rank":6,"pid":null,"affiliation":null},{"fullname":"Madge, David","name":"David","surname":"Madge","rank":7,"pid":null,"affiliation":null},{"fullname":"Virag, Laszlo","name":"Laszlo","surname":"Virag","rank":8,"pid":null,"affiliation":null},{"fullname":"Jost, Norbert","name":"Norbert","surname":"Jost","rank":9,"pid":null,"affiliation":null},{"fullname":"Horvath, Zoltan","name":"Zoltan","surname":"Horvath","rank":10,"pid":null,"affiliation":null},{"fullname":"Matschke, Klaus","name":"Klaus","surname":"Matschke","rank":11,"pid":null,"affiliation":null},{"fullname":"Varro, Andras","name":"Andras","surname":"Varro","rank":12,"pid":null,"affiliation":null},{"fullname":"Ravens, Ursula","name":"Ursula","surname":"Ravens","rank":13,"pid":null,"affiliation":null}],"resulttype":{"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"eng","classname":"English","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Human Electrophysiological and Pharmacological Properties of XEN-D0101: A Novel Atrial-Selective Kv1.5/I-Kur Inhibitor","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[],"dateofacceptance":{"value":"2013-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":{"value":"LIPPINCOTT WILLIAMS & WILKINS","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"embargoenddate":null,"source":[{"value":"Journal of Cardiovascular Pharmacology; Vol 61","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":null,"accessright":{"classid":"CLOSED","classname":"Closed Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0001","classname":"peerReviewed","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"10|opendoar____::96da2f590cd7246bbde0051047b0d6f7","value":"Unknown Repository","dataInfo":null},"url":["http://dx.doi.org/10.1097/FJC.0b013e31828780eb"],"distributionlocation":"","collectedfrom":{"key":"10|opendoar____::1a551829d50f1400b0dab21fdd969c04","value":"HEALTH FP7 Publications Database","dataInfo":null},"dateofacceptance":{"value":"2013-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"journal":{"name":"Journal of Cardiovascular Pharmacology","issnPrinted":"0160-2446","issnOnline":"","issnLinking":"","ep":"415","iss":"5","sp":"408","vol":"61","edition":"","conferenceplace":null,"conferencedate":null,"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055114204,"id":"50|ec_fp7health::0010eb63e181e3e91b8b6dc6b3e1c798","originalId":["ec_fp7health::0010eb63e181e3e91b8b6dc6b3e1c798"],"collectedfrom":[{"key":"openaire____::ec_fp7_health","value":"HEALTH FP7 Publications Database","dataInfo":null}],"pid":[{"value":"10.1016/j.ajpath.2013.06.019","qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofcollection":"2016-02-11T18:10:16.68Z","dateoftransformation":"2016-03-08T16:42:14.582Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2016-02-11T18:10:16.68Z","altered":true,"baseURL":"file:///var/lib/dnet/health_crossref","identifier":"","datestamp":"","metadataNamespace":""}},"author":[{"fullname":"Pizzolla, Angela","name":"Angela","surname":"Pizzolla","rank":1,"pid":null,"affiliation":null},{"fullname":"Wing, Kajsa","name":"Kajsa","surname":"Wing","rank":2,"pid":null,"affiliation":null},{"fullname":"Holmdahl, Rikard","name":"Rikard","surname":"Holmdahl","rank":3,"pid":null,"affiliation":null}],"resulttype":{"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"eng","classname":"English","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"A Glucose-6-Phosphate Isomerase Peptide Induces T and B Cell-Dependent Chronic Arthritis in C57BL/10 Mice Arthritis without Reactive Oxygen Species and Complement","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[],"dateofacceptance":{"value":"2013-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":{"value":"ELSEVIER SCIENCE INC","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"embargoenddate":null,"source":[{"value":"The American Journal of Pathology; Vol 183","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://www.elsevier.com/open-access/userlicense/1.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0001","classname":"peerReviewed","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"10|doajarticles::8cec81178926caaca531afbd8eb5d64c","value":"Unknown Repository","dataInfo":null},"url":["http://dx.doi.org/10.1016/j.ajpath.2013.06.019"],"distributionlocation":"","collectedfrom":{"key":"10|opendoar____::1a551829d50f1400b0dab21fdd969c04","value":"HEALTH FP7 Publications Database","dataInfo":null},"dateofacceptance":{"value":"2013-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"journal":{"name":"The American Journal of Pathology","issnPrinted":"0002-9440","issnOnline":"","issnLinking":"","ep":"1155","iss":"4","sp":"1144","vol":"183","edition":"","conferenceplace":null,"conferencedate":null,"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055114228,"id":"50|ec_fp7health::00110d3f9d05812aa683d5117964bdcd","originalId":["ec_fp7health::00110d3f9d05812aa683d5117964bdcd"],"collectedfrom":[{"key":"openaire____::ec_fp7_health","value":"HEALTH FP7 Publications Database","dataInfo":null}],"pid":[{"value":"10.1128/MCB.00231-12","qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofcollection":"2016-02-11T18:10:21.541Z","dateoftransformation":"2016-03-08T16:42:14.583Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2016-02-11T18:10:21.541Z","altered":true,"baseURL":"file:///var/lib/dnet/health_crossref","identifier":"","datestamp":"","metadataNamespace":""}},"author":[{"fullname":"Gronroos, E.","name":"E.","surname":"Gronroos","rank":1,"pid":null,"affiliation":null},{"fullname":"Kingston, I. J.","name":"I. J.","surname":"Kingston","rank":2,"pid":null,"affiliation":null},{"fullname":"Ramachandran, A.","name":"A.","surname":"Ramachandran","rank":3,"pid":null,"affiliation":null},{"fullname":"Randall, R. A.","name":"R. A.","surname":"Randall","rank":4,"pid":null,"affiliation":null},{"fullname":"Vizan, P.","name":"P.","surname":"Vizan","rank":5,"pid":null,"affiliation":null},{"fullname":"Hill, C. S.","name":"C. S.","surname":"Hill","rank":6,"pid":null,"affiliation":null}],"resulttype":{"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"eng","classname":"English","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Transforming Growth Factor beta Inhibits Bone Morphogenetic Protein-Induced Transcription through Novel Phosphorylated Smad1/5-Smad3 Complexes","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[],"dateofacceptance":{"value":"2012-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":{"value":"AMER SOC MICROBIOLOGY","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"embargoenddate":null,"source":[{"value":"Molecular and Cellular Biology; Vol 32","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":null,"accessright":{"classid":"CLOSED","classname":"Closed Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0001","classname":"peerReviewed","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"openaire____::1256f046-bf1f-4afc-8b47-d0b147148b18","value":"Unknown Repository","dataInfo":null},"url":["http://dx.doi.org/10.1128/MCB.00231-12"],"distributionlocation":"","collectedfrom":{"key":"openaire____::ec_fp7_health","value":"HEALTH FP7 Publications Database","dataInfo":null},"dateofacceptance":{"value":"2012-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"journal":{"name":"Molecular and Cellular Biology","issnPrinted":"0270-7306","issnOnline":"","issnLinking":"","ep":"2916","iss":"14","sp":"2904","vol":"32","edition":"","conferenceplace":null,"conferencedate":null,"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055114253,"id":"50|ec_fp7health::0013c229505d753b29370b1029f196d3","originalId":["ec_fp7health::0013c229505d753b29370b1029f196d3"],"collectedfrom":[{"key":"openaire____::ec_fp7_health","value":"HEALTH FP7 Publications Database","dataInfo":null}],"pid":[{"value":"10.1111/j.1469-0691.2011.03696.x","qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofcollection":"2016-02-11T18:10:16.003Z","dateoftransformation":"2016-03-08T16:42:14.586Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2016-02-11T18:10:16.003Z","altered":true,"baseURL":"file:///var/lib/dnet/health_crossref","identifier":"","datestamp":"","metadataNamespace":""}},"author":[{"fullname":"Espinal, P.","name":"P.","surname":"Espinal","rank":1,"pid":null,"affiliation":null},{"fullname":"Seifert, H.","name":"H.","surname":"Seifert","rank":2,"pid":null,"affiliation":null},{"fullname":"Dijkshoorn, L.","name":"L.","surname":"Dijkshoorn","rank":3,"pid":null,"affiliation":null},{"fullname":"Vila, J.","name":"J.","surname":"Vila","rank":4,"pid":null,"affiliation":null},{"fullname":"Roca, I.","name":"I.","surname":"Roca","rank":5,"pid":null,"affiliation":null}],"resulttype":{"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"eng","classname":"English","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Rapid and accurate identification of genomic species from the Acinetobacter baumannii (Ab) group by MALDI-TOF MS.","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[],"dateofacceptance":{"value":"2012-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":{"value":"Blackwell Publishing","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"embargoenddate":null,"source":[{"value":"Clinical Microbiology and Infection; Vol 18","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://www.elsevier.com/open-access/userlicense/1.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0001","classname":"peerReviewed","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"openaire____::1256f046-bf1f-4afc-8b47-d0b147148b18","value":"Unknown Repository","dataInfo":null},"url":["http://dx.doi.org/10.1111/j.1469-0691.2011.03696.x"],"distributionlocation":"","collectedfrom":{"key":"openaire____::ec_fp7_health","value":"HEALTH FP7 Publications Database","dataInfo":null},"dateofacceptance":{"value":"2012-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"journal":{"name":"Clinical Microbiology and Infection","issnPrinted":"1198-743X","issnOnline":"","issnLinking":"","ep":"1103","iss":"11","sp":"1097","vol":"18","edition":"","conferenceplace":null,"conferencedate":null,"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055114274,"id":"50|ec_fp7health::00161e52ec870cc48b5a0a40c5d88e58","originalId":["ec_fp7health::00161e52ec870cc48b5a0a40c5d88e58"],"collectedfrom":[{"key":"openaire____::ec_fp7_health","value":"HEALTH FP7 Publications Database","dataInfo":null}],"pid":[{"value":"10.1021/ac200237j","qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofcollection":"2016-02-11T18:09:49.972Z","dateoftransformation":"2016-03-08T16:42:14.588Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2016-02-11T18:09:49.972Z","altered":true,"baseURL":"file:///var/lib/dnet/health_crossref","identifier":"","datestamp":"","metadataNamespace":""}},"author":[{"fullname":"Xu, Shoujiang","name":"Shoujiang","surname":"Xu","rank":1,"pid":null,"affiliation":null},{"fullname":"Liu, Yang","name":"Yang","surname":"Liu","rank":2,"pid":null,"affiliation":null},{"fullname":"Wang, Taihong","name":"Taihong","surname":"Wang","rank":3,"pid":null,"affiliation":null},{"fullname":"Li, Jinghong","name":"Jinghong","surname":"Li","rank":4,"pid":null,"affiliation":null}],"resulttype":{"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"eng","classname":"English","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Positive Potential Operation of a Cathodic Electrogenerated Chemiluminescence Immunosensor Based on Luminol and Graphene for Cancer Biomarker Detection","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[],"dateofacceptance":{"value":"2011-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":{"value":"AMER CHEMICAL SOC","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"embargoenddate":null,"source":[{"value":"Analytical Chemistry; Vol 83","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":null,"accessright":{"classid":"CLOSED","classname":"Closed Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0001","classname":"peerReviewed","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"openaire____::1256f046-bf1f-4afc-8b47-d0b147148b18","value":"Unknown Repository","dataInfo":null},"url":["http://dx.doi.org/10.1021/ac200237j"],"distributionlocation":"","collectedfrom":{"key":"openaire____::ec_fp7_health","value":"HEALTH FP7 Publications Database","dataInfo":null},"dateofacceptance":{"value":"2011-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"journal":{"name":"Analytical Chemistry","issnPrinted":"0003-2700","issnOnline":"","issnLinking":"","ep":"3823","iss":"10","sp":"3817","vol":"83","edition":"","conferenceplace":null,"conferencedate":null,"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055114298,"id":"50|ec_fp7health::001b3ce526a763539e684a5c8bec1223","originalId":["ec_fp7health::001b3ce526a763539e684a5c8bec1223"],"collectedfrom":[{"key":"openaire____::ec_fp7_health","value":"HEALTH FP7 Publications Database","dataInfo":null}],"pid":[{"value":"10.1038/nrmicro3067","qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofcollection":"2016-02-11T18:10:14.775Z","dateoftransformation":"2016-03-08T16:42:14.589Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2016-02-11T18:10:14.775Z","altered":true,"baseURL":"file:///var/lib/dnet/health_crossref","identifier":"","datestamp":"","metadataNamespace":""}},"author":[{"fullname":"Chandler, Michael","name":"Michael","surname":"Chandler","rank":1,"pid":null,"affiliation":null},{"fullname":"de la Cruz, Fernando","name":"Fernando","surname":"La Cruz","rank":2,"pid":null,"affiliation":null},{"fullname":"Dyda, Fred","name":"Fred","surname":"Dyda","rank":3,"pid":null,"affiliation":null},{"fullname":"Hickman, Alison B.","name":"Alison B.","surname":"Hickman","rank":4,"pid":null,"affiliation":null},{"fullname":"Moncalian, Gabriel","name":"Gabriel","surname":"Moncalian","rank":5,"pid":null,"affiliation":null},{"fullname":"Ton-Hoang, Bao","name":"Bao","surname":"Ton-Hoang","rank":6,"pid":null,"affiliation":null}],"resulttype":{"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"eng","classname":"English","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Breaking and joining single-stranded DNA: the HUH endonuclease superfamily","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[],"dateofacceptance":{"value":"2013-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":{"value":"NATURE PUBLISHING GROUP","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"embargoenddate":null,"source":[{"value":"Nature Reviews Microbiology; Vol 11","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":null,"accessright":{"classid":"CLOSED","classname":"Closed Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0001","classname":"peerReviewed","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"openaire____::1256f046-bf1f-4afc-8b47-d0b147148b18","value":"Unknown Repository","dataInfo":null},"url":["http://dx.doi.org/10.1038/nrmicro3067"],"distributionlocation":"","collectedfrom":{"key":"openaire____::ec_fp7_health","value":"HEALTH FP7 Publications Database","dataInfo":null},"dateofacceptance":{"value":"2013-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"journal":{"name":"Nature Reviews Microbiology","issnPrinted":"1740-1526","issnOnline":"","issnLinking":"","ep":"538","iss":"8","sp":"525","vol":"11","edition":"","conferenceplace":null,"conferencedate":null,"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055114326,"id":"50|ec_fp7health::001bd73b6a5ada26a6021d608de6c6a4","originalId":["ec_fp7health::001bd73b6a5ada26a6021d608de6c6a4"],"collectedfrom":[{"key":"openaire____::ec_fp7_health","value":"HEALTH FP7 Publications Database","dataInfo":null}],"pid":[{"value":"10.1016/S0140-6736(12)62202-8","qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofcollection":"2016-02-11T18:10:29.817Z","dateoftransformation":"2016-03-08T16:42:14.591Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2016-02-11T18:10:29.817Z","altered":true,"baseURL":"file:///var/lib/dnet/health_crossref","identifier":"","datestamp":"","metadataNamespace":""}},"author":[{"fullname":"Hansel, Trevor T","name":"Trevor T.","surname":"Hansel","rank":1,"pid":null,"affiliation":null},{"fullname":"Johnston, Sebastian L","name":"Sebastian L.","surname":"Johnston","rank":2,"pid":null,"affiliation":null},{"fullname":"Openshaw, Peter J","name":"Peter J.","surname":"Openshaw","rank":3,"pid":null,"affiliation":null}],"resulttype":{"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"eng","classname":"English","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Microbes and mucosal immune responses in asthma","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[],"dateofacceptance":{"value":"2013-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":{"value":"ELSEVIER SCIENCE INC","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"embargoenddate":null,"source":[{"value":"The Lancet; Vol 381","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":null,"accessright":{"classid":"CLOSED","classname":"Closed Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0001","classname":"peerReviewed","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"openaire____::1256f046-bf1f-4afc-8b47-d0b147148b18","value":"Unknown Repository","dataInfo":null},"url":["http://dx.doi.org/10.1016/S0140-6736(12)62202-8"],"distributionlocation":"","collectedfrom":{"key":"openaire____::ec_fp7_health","value":"HEALTH FP7 Publications Database","dataInfo":null},"dateofacceptance":{"value":"2013-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"journal":{"name":"The Lancet","issnPrinted":"0140-6736","issnOnline":"","issnLinking":"","ep":"873","iss":"9869","sp":"861","vol":"381","edition":"","conferenceplace":null,"conferencedate":null,"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055114352,"id":"50|ec_fp7health::001d16a111fbca498017c8d39b0068bf","originalId":["ec_fp7health::001d16a111fbca498017c8d39b0068bf"],"collectedfrom":[{"key":"openaire____::ec_fp7_health","value":"HEALTH FP7 Publications Database","dataInfo":null}],"pid":[{"value":"10.1016/j.expneurol.2011.08.012","qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofcollection":"2016-02-11T18:09:51.03Z","dateoftransformation":"2016-03-08T16:42:14.592Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2016-02-11T18:09:51.03Z","altered":true,"baseURL":"file:///var/lib/dnet/health_crossref","identifier":"","datestamp":"","metadataNamespace":""}},"author":[{"fullname":"Mazarati, Andréy","name":"Andréy","surname":"Mazarati","rank":1,"pid":null,"affiliation":null},{"fullname":"Maroso, Mattia","name":"Mattia","surname":"Maroso","rank":2,"pid":null,"affiliation":null},{"fullname":"Iori, Valentina","name":"Valentina","surname":"Iori","rank":3,"pid":null,"affiliation":null},{"fullname":"Vezzani, Annamaria","name":"Annamaria","surname":"Vezzani","rank":4,"pid":null,"affiliation":null},{"fullname":"Carli, Mirjana","name":"Mirjana","surname":"Carli","rank":5,"pid":null,"affiliation":null}],"resulttype":{"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"eng","classname":"English","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"High-mobility group box-1 impairs memory in mice through both toll-like receptor 4 and Receptor for Advanced Glycation End Products","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[],"dateofacceptance":{"value":"2011-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[{"value":"Experimental Neurology; Vol 232","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":null,"accessright":{"classid":"CLOSED","classname":"Closed Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0001","classname":"peerReviewed","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"openaire____::1256f046-bf1f-4afc-8b47-d0b147148b18","value":"Unknown Repository","dataInfo":null},"url":["http://dx.doi.org/10.1016/j.expneurol.2011.08.012"],"distributionlocation":"","collectedfrom":{"key":"openaire____::ec_fp7_health","value":"HEALTH FP7 Publications Database","dataInfo":null},"dateofacceptance":{"value":"2011-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"journal":{"name":"Experimental Neurology","issnPrinted":"0014-4886","issnOnline":"","issnLinking":"","ep":"148","iss":"2","sp":"143","vol":"232","edition":"","conferenceplace":null,"conferencedate":null,"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}} \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/publication/update_datasource/publication_10.json.gz b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/publication/update_datasource/publication_10.json.gz deleted file mode 100644 index 99c4015e715ae7540f024ef729623d3e0484d739..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5257 zcmV;46n5($iwFocB%odZ18{X>Y-wX*bZKvHUokK)YIARH0PS7fa@#nTzTc-n>0)c9 zNA8|+J60mwD%(kSPuEroBq0knMQA|Ej+4p7KF&VJ-tI%~0V#@<6w8tlXXp%4 zUDYOZz=NN|1K|6BgD;Jcq@*8@c;nzpgT?nOVSYp#2O|g(5L1N5e| zDKr;N*@Px3nM@l8*m5n+vM@G0Q-?}RO_ugg8hjj0oiQa*IzBki;Gb&+t~WA$&kh4! zw>>{J9B4+vGBg_s!3DcvF^T%{fiDt6ar^nsF^(i_TI%qiaBNmXX+UN*u!zT#^(b7!fSrmN*l~Y++ zP*l7GYi+DyyK~zf-S}-?!+Kk@+M4D-?tjck#72yY<<1T{Tj|j}Wv$y-_%VH)(kHrW z@$1*7GUx5gS$3c_Wj>l)PFC^;^qSq$^cKd~*ge47fu^@z&-i6a`?6B0c@nx@t`&$c6q!3IIu;M?2D*xfq&0+( zUsU?4npWz{axAo2&u9Zv7c`kgsq{(9-e6if*OgCy`Ipu2Jb#~bay;TU>6f)V! zAOU$1sDfUkqOjN}dUXQ62j%LQTVUyj)!NFg?Xi@8To$NSbwxzk!!Gt9pzil1PQz@F zmEUSu9fbc}%ne8Y)lQ4?x-cF&wQFe@{%fda-?rJCxr#z;PT^P@l7y>k&=>$T%HvUcJ&=i{C}&YiU@nYHzt*5l6EAbL|^O zR@tIZ@1bDe$HUB^T&IOy)0+Qs5l1YZH?e#ku=oxNlP6G#!q~2s{|0}{9!j!ANB()w zp%{&9+@>L`8uOwTqYSfI3?}pB>Qq5l(sq{g^*_F-w)!zFfr)i@154n0)W##%7+J*h zwZO0($MGFM2*Q!!KF<oL%h;dK3ytWL;2qsXF=Nl9A1Y7uwt%YxMaum;HZI z(Q~p|p-g6>!BgCf;$~je7`x8RypJ=04Aw%hoA!VFE0N9g>dbw{RCf)0_O zftay$B9l{&(lO1X1(EE`gv00uX(UA~K~N((zY7tI(baYL+CZaOoRxz_(IX0J#{S$> zGzBw<_a#pogQ9fmXhz>p>1YaJ`7xBv$??(gW%s&&agMrY$Cv2pqKA%uAELwl#Xz<0 zOAtKQ;Q3CvSI39wZ4Oc8w2p47=KghPo%xjRDe9p&xVSn#T4}&u#1k0w{%z=o?*Z+9 z1pBax>+C~3yc8Ct2EbQIF|7Iabz;FXw3hAK+SbGZHXLigELa7zfDPMPPz%}Ep{NB# zE!3v1o~IVaOEz)K?Tk{VaMbib3-ZOt)(prVP61q;Tdn&PokGnd?xU&s$IG|@E|FpI7< zS(#A5vNPd5IfhGqLj|UEt%w=+`?Nf`~I8@pNh29 zQYMsYr0CZC!m^fAF7BZRn&|I5f)z)pXYpLBnah^nrnY0(q~LV#pm*LyWLEh+i^pp3 z>NO2oXMs*<6B46iIW3Z>aG*Gu$pj?IcG(SKk}@H|B5epl;JLu05Fer6kI!32 z8rJXu>VlWMrxEI=f|00oMWZ>>^X?wEtxmu7ZYEGaKl8)I)4i`~pE}yj_tQRxLcM+r z?K9{PhyC;3#q~A%tv?v_yJuJEKVdHaH~ILke}1K!_$9cXIVvclQX52>bESTh}83+<89_`!tG z6KLF?CsLN{f{A6pE4|RtM%dIqNl1JhkHXNiy+AX+3tv?OF-Wy$kmR8dyKTEW6G-}tolz-@ zQ!fH($UcAO+2;Okd_$|4>*Be@TnpjGeq_>bFq%rb;x?(MDIO~v4~t8Pe>eR2lt+`0 zjH4#HWOqbV2#dvY6@AZz8{3je3PM&8a=rZEI)vOsZ=zWM3*MI98p7IeItKTE`U#)F zvOJ+^2n!qVAgCXQGnjaz>-_vS>d`1_9W72A^u~h6EI?h6js;9-Ww0pJvz)^Ltnttp zEXON9JuvVrvlJGMB!`(VK0V&h7+pJ4s#Ra=bU_zfrVNyaD)_<>76=t^8M}sR|6d1PUW+32@bJ*4&@=Zi zR<#2@>Bx0yE7#eXB~;9_cCZR3rNT-58s|Blftf zoc_R83Ei6obdRxVF0lK2ELY&Y0`F_nR?ma?n<7hLtcAwN3pLBdIw7`egpkMaY|{^e z@OkilbN2dx_gHs3XT8HVI0XZ@u&zM+`dRX}puLXUmbr6eDc5)@)V?fFyNR4l(;L^tcpxJsfq#2Q$LIvA#Rc-ggBF|G6LiR9iq5#0j`>X< zsa9^6PbrIAIrR~u>|MoJfYPzxvzzgJ=X_3`K%F9R(;>Q=kPvsQmi)gl=Q2J~6)@?Q z*^{~hqc&8=B1+O(zjt{d!z5k2`9n49OK=%y946e^cF2r~l|ew|!_sXShOXNF*WoeL zF8SwS_J8%Q4Qr^ytBRYX#=$xw>6<nRm0$uk&GIT7%*FD35F+tP45%%oRcpgUIoV`XcI>w#bHa2aq zrP&^q+%IlxhV9wyPYP16pH=U$4_?!DzRlUSUvMM*ygc=S)Q^*?{E9b1wsV<10re{? zZ%<8|=(M8YbXmh(W=|mfh~3^LW6nj~M1$&m)XPdru{}Zb4@{mIsB!{tSF!ENDS=%> zCj~@O4pqKk8J7ovtzmNoYAaB?UOsyr)V?HO!r7D^)>|_nH|!U#EjL2B8|H?_T;3U* z%s17`$wC(XzZ<}$J>@>M0~N?8GKNgd82U@s|0}u?d^Sb?BRcC2j{2?ZixYH~aYPEY zenlpuaO>LO)`zfyy?c;XqYv|+^TTSYRev~a-HTY9g}FLoP%b~mFfuO*4V_PSi%s~d z1AZOsTJ6DCd+-%GunCJ%M8PlqUA{a~u*KNA#ui}gxh=;ue&3ocz*sX%wg4Awfs7E< zD0kf}wm`83b!w~U*#c)%#2gz_OQ(VBXh8rYgGDrAYJnBHF15CZn6o*1eb@p`#~l*r zlD&3r6+KWtGv4~51+VQn`p)#g%iXlFC{MlUf!}9MbT#I)TSl@Ke}UIml1hmB_t~Bl z!GM99KbAE7vHEh5*^?gljmQ8z*MyCERYRFj3Bp4vL3nl}2L`N(-oi{^SxF_OT9vIy z0tMtNAirMTbsfkbN>wrd&XA|l_rL@j*3Dvp;S|G4Fv|7R@Fw zr%nRdP^dqd%wnFb`vw%KPw#2x%~1U*ws zcVR*Fm}ZDX<_8lP?s9Nn1=#tO zzJ@ZZE6`=b#U;9|Lb@n) zxn6DcJi6>{3hCk-0kw3SIJRLK9<^PQSb^))0PFe|d(k&%uMxUzxSd!`BspbhwxjUn z`Wf=p+t1Cm<5)Z6%ifD^KVOumUiflPhBb-gCAen{##$Pc&}o@ay`44}-xFsJDT)Z{ ziP`5SI+64DIILW}T1ly3wYmJB(DTt_NI;dKm6+`Fr^>!`5ZV)TK4rn(B#Ryci+CQR z!)g$jN>Vj8N|s}50D9%_RUX5HhDnWYh%Z>RzsiKl1+0a9W50ke*F0{W@(jN}B$ckp zA4)w7xrPe+qJZ_+IjpP$>xY68IsMO2?_17d`L|q^M6{L25hbJ{Iy&zjWIN(d-=88G zbTpm?5tTvjli8Gt5t*>)QDNb)%NA7h`PyLN=iTe~m&a)M{&3L0I_;moL2oWE-Vas7 zz62tEPSTm6D2E=CrG0wyUGK`G6)>)V@t+BduYeM-3L00VU>%|H4PJf?A;*rXwXmgc zAFahQ+yy}H7WlZOTMKv`tLsV?JYK7|dKMlxbbAv#9y*3^6N`jIw~4LkIJ7l4q;_Bv za|?KUWA+-sMzD*EPN6Dxkc6CcSm&bFb}UXIG%y!!H*4d{Lfy z0p+LAEHrAOYeDb1K-VP}TBNMtyqWx-IQd&Xj`JAkR}|`*{cY?A8X^RriYi%zH%#8OAn`yX`9TRWPoCaepR^`{d$T0lL4m z_xM?NS?2zI4ZmHO+}8N5>y#UuJ=gXM2-mgo0>aG)5QX6?47WCI^*n}a=^HTI5ZeS} zJn{o#daj0@zzq%0*EHMpN6%xpo3qyo!@X_OPt%yr1drw+p4_&kF0iblQEwYg!;*2j zb_Q|vmx`vbC{DdF+%xi-$Yb$M)Qv;&zyI@C;+ryo{@;H;F5#Pn{GKT88G(xMCORW& z%1Ehe@m$p;MfZel`_M&A^npaOt>TKJYEr37if@hBt_0wwpFfj0Yd=UKqFSqJQY9jH zo};nfcIO@u5tCcf#Vx5;tW2nEO_#EbZMQiqBw8WS^>Xv;Nc1Tz87s$*o?aSg(hVSTx_p&G< zdJ{d)qkbQQ%uw)f7NqK`$ya7XDu&0u7~Rgto2`M_<4;p6V6hUXBtqxelzg$(R#&$p zDv(xzv_Bi9eQI_5s$g2>#p?*ue%srxL$(?=Tc+!7581-Euz+m!0Gst, and not Rst<>Gst).","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2012-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[{"id":"http://zenodo.org/communities/euromixproject"}],"externalReference":[],"instance":[{"license":{"value":"https://creativecommons.org/licenses/by-sa/3.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0029","classname":"Software","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"10|opendoar____::fd4c2dc64ccb8496e6f1f94c85f30d06","value":"ProdInra","dataInfo":null},"url":["http://prodinra.inra.fr/record/255707"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::1582","value":"ProdInra","dataInfo":null},"dateofacceptance":{"value":"2012-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"documentationUrl":[],"license":[],"codeRepositoryUrl":null,"programmingLanguage":null} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055721330,"id":"50|od______1582::5aec1186054301b66c0c5dc35972a589","originalId":["od______1582::5aec1186054301b66c0c5dc35972a589"],"collectedfrom":[{"key":"opendoar____::1582","value":"ProdInra","dataInfo":null}],"pid":[],"dateofcollection":"2019-01-24T16:45:07Z","dateoftransformation":"","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2019-01-23T18:54:28.567Z","altered":true,"baseURL":"http://oai.prodinra.inra.fr/ft","identifier":"oai:prodinra.inra.fr:402973","datestamp":"2018-03-19T00:00:00Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Muratorio, Sylvie","name":"Sylvie","surname":"Muratorio","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"software","classname":"software","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"eng","classname":"English","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"modèle de simulation","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"modèle physiologique","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"approche génétique","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"castanea","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"fagus sylvatica","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"PDG Documentation, version 2","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[],"dateofacceptance":{"value":"2017-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"https://creativecommons.org/licenses/by-sa/3.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0029","classname":"Software","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::1582","value":"ProdInra","dataInfo":null},"url":["http://prodinra.inra.fr/record/402973"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::1582","value":"ProdInra","dataInfo":null},"dateofacceptance":{"value":"2017-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"documentationUrl":[],"license":[],"codeRepositoryUrl":null,"programmingLanguage":null} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055728345,"id":"50|od______1582::626bf0af9988f811e6290d694587edf5","originalId":["od______1582::626bf0af9988f811e6290d694587edf5"],"collectedfrom":[{"key":"opendoar____::1582","value":"ProdInra","dataInfo":null}],"pid":[],"dateofcollection":"2019-01-24T16:45:07Z","dateoftransformation":"","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2019-01-23T18:57:53.59Z","altered":true,"baseURL":"http://oai.prodinra.inra.fr/ft","identifier":"oai:prodinra.inra.fr:396375","datestamp":"2018-12-20T00:00:00Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Chades, Iadine","name":"Iadine","surname":"Chades","rank":1,"pid":null,"affiliation":null},{"fullname":"Chapron, Guillaume","name":"Guillaume","surname":"Chapron","rank":2,"pid":null,"affiliation":null},{"fullname":"Cros, Marie-Josee","name":"Marie-Josee","surname":"Cros","rank":3,"pid":null,"affiliation":null},{"fullname":"Garcia, Frederick","name":"Frederick","surname":"Garcia","rank":4,"pid":null,"affiliation":null},{"fullname":"Sabbadin, Regis","name":"Regis","surname":"Sabbadin","rank":5,"pid":null,"affiliation":null}],"resulttype":{"classid":"software","classname":"software","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"eng","classname":"English","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[],"title":[{"value":"Package MDPtoolbox R","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"The Markov Decision Processes (MDP) toolbox proposes functions related to the resolution of discrete-time Markov Decision Processes: finite horizon, value iteration, policy iteration, linear programming algorithms with some variants and also proposes some functions related to Reinforcement Learning.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2017-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":null,"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0029","classname":"Software","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::1582","value":"ProdInra","dataInfo":null},"url":["http://prodinra.inra.fr/record/396375"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::1582","value":"ProdInra","dataInfo":null},"dateofacceptance":{"value":"2017-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"documentationUrl":[],"license":[],"codeRepositoryUrl":null,"programmingLanguage":null} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055729488,"id":"50|od______1582::639909adfad9d708308f2aedb733e4a0","originalId":["od______1582::639909adfad9d708308f2aedb733e4a0"],"collectedfrom":[{"key":"opendoar____::1582","value":"ProdInra","dataInfo":null}],"pid":[],"dateofcollection":"2019-01-24T16:45:07Z","dateoftransformation":"","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2019-01-23T18:50:56.823Z","altered":true,"baseURL":"http://oai.prodinra.inra.fr/ft","identifier":"oai:prodinra.inra.fr:408837","datestamp":"2018-03-19T00:00:00Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Becheler, Ronan","name":"Ronan","surname":"Becheler","rank":1,"pid":null,"affiliation":null},{"fullname":"Masson, Jean-Pierre","name":"Jean-Pierre","surname":"Masson","rank":2,"pid":null,"affiliation":null},{"fullname":"Arnaud-Haond, Sophie","name":"Sophie","surname":"Arnaud-Haond","rank":3,"pid":null,"affiliation":null},{"fullname":"Halkett, Fabien","name":"Fabien","surname":"Halkett","rank":4,"pid":null,"affiliation":null},{"fullname":"Mariette, Stéphanie","name":"Stéphanie","surname":"Mariette","rank":5,"pid":null,"affiliation":null},{"fullname":"Guillemin, Marie-Laure","name":"Marie-Laure","surname":"Guillemin","rank":6,"pid":null,"affiliation":null},{"fullname":"Valero, Myriam","name":"Myriam","surname":"Valero","rank":7,"pid":null,"affiliation":null},{"fullname":"Destombe, Christophe","name":"Christophe","surname":"Destombe","rank":8,"pid":null,"affiliation":null},{"fullname":"Stoeckel, Solenn","name":"Solenn","surname":"Stoeckel","rank":9,"pid":null,"affiliation":null}],"resulttype":{"classid":"software","classname":"software","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"eng","classname":"English","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"clonalite","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"eucaryote","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"reproduction clonale","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"approche bayésienne","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"écologie végétale","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"logiciel","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"endogamie","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"autoincompatibilité","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"ClonEstiMate 1.01","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"ClonEstiMate, a Bayesian method for quantifying rates of clonality of populations genotyped at two-time steps","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2016-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"https://creativecommons.org/licenses/by-sa/3.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0029","classname":"Software","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::1582","value":"ProdInra","dataInfo":null},"url":["http://prodinra.inra.fr/record/408837"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::1582","value":"ProdInra","dataInfo":null},"dateofacceptance":{"value":"2016-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"documentationUrl":[],"license":[],"codeRepositoryUrl":null,"programmingLanguage":null} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055738736,"id":"50|od______1582::6e7a9b21a2feef45673890432af34244","originalId":["od______1582::6e7a9b21a2feef45673890432af34244"],"collectedfrom":[{"key":"opendoar____::1582","value":"ProdInra","dataInfo":null}],"pid":[],"dateofcollection":"2019-01-24T16:45:07Z","dateoftransformation":"","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2019-01-23T18:53:57.871Z","altered":true,"baseURL":"http://oai.prodinra.inra.fr/ft","identifier":"oai:prodinra.inra.fr:403174","datestamp":"2018-03-19T00:00:00Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Delenne, Jean-Yves","name":"Jean-Yves","surname":"Delenne","rank":1,"pid":null,"affiliation":null},{"fullname":"Richefeu, Vincent","name":"Vincent","surname":"Richefeu","rank":2,"pid":null,"affiliation":null},{"fullname":"Frank, Xavier","name":"Xavier","surname":"Frank","rank":3,"pid":null,"affiliation":null},{"fullname":"Radjaï, Farhang","name":"Farhang","surname":"Radjaï","rank":4,"pid":null,"affiliation":null}],"resulttype":{"classid":"software","classname":"software","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"eng","classname":"English","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Modélisation et simulation","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Modeling and Simulation","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"structure granulaire","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"algorithme","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"simulation numérique","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"flux de gaz","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"flux de liquide","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Flowbox","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"The code FLOWbox is dedicated to the computation of flow through porous and granular materials.\nFLOWbox is based on an optimized 3D Lattice Boltzmann algorithm for the computation of liquid or gas flows directly at the scale of heterogeneities. FLOWbox intends to be a powerful and versatile software able to operate on highly detailed microstructures in a systematic fashion. These microstructures can be generated either from numerical simulation or from tomography.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2015-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"https://creativecommons.org/licenses/by-sa/3.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0029","classname":"Software","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::1582","value":"ProdInra","dataInfo":null},"url":["http://prodinra.inra.fr/record/403174"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::1582","value":"ProdInra","dataInfo":null},"dateofacceptance":{"value":"2015-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"documentationUrl":[],"license":[],"codeRepositoryUrl":null,"programmingLanguage":null} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055741711,"id":"50|od______1582::71dd00de0e70764d2800a766c0b165db","originalId":["od______1582::71dd00de0e70764d2800a766c0b165db"],"collectedfrom":[{"key":"opendoar____::1582","value":"ProdInra","dataInfo":null}],"pid":[],"dateofcollection":"2019-01-24T16:45:07Z","dateoftransformation":"","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2019-01-23T18:35:31.489Z","altered":true,"baseURL":"http://oai.prodinra.inra.fr/ft","identifier":"oai:prodinra.inra.fr:442564","datestamp":"2018-11-12T00:00:00Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Bitteur, Sylvaine","name":"Sylvaine","surname":"Bitteur","rank":1,"pid":null,"affiliation":null},{"fullname":"Lortal, Sylvie","name":"Sylvie","surname":"Lortal","rank":2,"pid":null,"affiliation":null}],"resulttype":{"classid":"software","classname":"software","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"eng","classname":"English","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[],"title":[{"value":"Charte graphique et site web MILK International Symposium","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[],"dateofacceptance":{"value":"2014-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"https://creativecommons.org/licenses/by-nd/3.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0029","classname":"Software","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::1582","value":"ProdInra","dataInfo":null},"url":["http://prodinra.inra.fr/record/442564"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::1582","value":"ProdInra","dataInfo":null},"dateofacceptance":{"value":"2014-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"documentationUrl":[],"license":[],"codeRepositoryUrl":null,"programmingLanguage":null} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055745351,"id":"50|od______1582::764d9ba73f2f3e794e00a80b075330ef","originalId":["od______1582::764d9ba73f2f3e794e00a80b075330ef"],"collectedfrom":[{"key":"opendoar____::1582","value":"ProdInra","dataInfo":null}],"pid":[],"dateofcollection":"2019-01-24T16:45:07Z","dateoftransformation":"","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2019-01-23T18:41:37.345Z","altered":true,"baseURL":"http://oai.prodinra.inra.fr/ft","identifier":"oai:prodinra.inra.fr:407117","datestamp":"2018-05-28T00:00:00Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Favre, Colette","name":"Colette","surname":"Favre","rank":1,"pid":null,"affiliation":null},{"fullname":"Grel, Audrey","name":"Audrey","surname":"Grel","rank":2,"pid":null,"affiliation":null},{"fullname":"Granier, Evelyne","name":"Evelyne","surname":"Granier","rank":3,"pid":null,"affiliation":null},{"fullname":"Cosserat-Mangeot, Régine","name":"Régine","surname":"Cosserat-Mangeot","rank":4,"pid":null,"affiliation":null},{"fullname":"Bachacou, Jean","name":"Jean","surname":"Bachacou","rank":5,"pid":null,"affiliation":null},{"fullname":"LEROY, Nathalie","name":"Nathalie","surname":"Leroy","rank":6,"pid":null,"affiliation":null},{"fullname":"Dupouey, Jean-Luc","name":"Jean-Luc","surname":"Dupouey","rank":7,"pid":null,"affiliation":null}],"resulttype":{"classid":"software","classname":"software","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"fra/fre","classname":"French","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"digitalisation","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"carte des sols","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"géoréférencement","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"vectorisation","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"France","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"usage du sol","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Digitalisation des cartes anciennes : Manuel pour la vectorisation de l'usage des sols et le géo-référencement des minutes 1:40 000 de la carte d' Etat-Major","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[],"dateofacceptance":{"value":"2017-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"https://creativecommons.org/licenses/by-sa/3.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0029","classname":"Software","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::1582","value":"ProdInra","dataInfo":null},"url":["http://prodinra.inra.fr/record/407117"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::1582","value":"ProdInra","dataInfo":null},"dateofacceptance":{"value":"2017-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"documentationUrl":[],"license":[],"codeRepositoryUrl":null,"programmingLanguage":null} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055748883,"id":"50|od______1582::7a6ace3d7ad42ac813e4955f66a44435","originalId":["od______1582::7a6ace3d7ad42ac813e4955f66a44435"],"collectedfrom":[{"key":"opendoar____::1582","value":"ProdInra","dataInfo":null}],"pid":[],"dateofcollection":"2019-01-24T16:45:07Z","dateoftransformation":"","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2019-01-23T18:52:22.609Z","altered":true,"baseURL":"http://oai.prodinra.inra.fr/ft","identifier":"oai:prodinra.inra.fr:407676","datestamp":"2018-03-19T00:00:00Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Desjeux, Yann","name":"Yann","surname":"Desjeux","rank":1,"pid":null,"affiliation":null},{"fullname":"Latruffe, Laure","name":"Laure","surname":"Latruffe","rank":2,"pid":null,"affiliation":null}],"resulttype":{"classid":"software","classname":"software","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"eng","classname":"English","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Stochastic Frontier Analysis (SFA);R; R package","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"sfadv: Advanced Methods for Stochastic Frontier Analysis. R package version 1.0.1","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Stochastic frontier analysis with advanced methods.\nIn particular, it applies the approach proposed by Latruffe et al. (2017) to estimate a stochastic frontier with technical inefficiency effects when one input is endogenous.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2017-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"https://creativecommons.org/licenses/by-sa/3.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0029","classname":"Software","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::1582","value":"ProdInra","dataInfo":null},"url":["http://prodinra.inra.fr/record/407676"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::1582","value":"ProdInra","dataInfo":null},"dateofacceptance":{"value":"2017-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"documentationUrl":[],"license":[],"codeRepositoryUrl":null,"programmingLanguage":null} diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/software/software_10.json.gz b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/sample/software/software_10.json.gz deleted file mode 100644 index 3dcadf41d70ba3ac0cd5661d34778d279d35e352..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6727 zcmV-N8o1>jiwFo3Y@l8M19NX?ba!ELWnVEcE^2dcZUF6FUvt|yl7GKXfm7Aq*j2Jj ziljto_V&uLoy2o?lCrbAH@7pDf=Ez8LJ?XLw4==CVc+hfte^Hn_;o+jbp!k-S&nQ^ z5-Vd&Wo#1!&_H*ifo}W={L%`kq(@04TEkyjJUQnXAH}RSjA)#(Hl#!>O<7nage+!~ zg`>H8!;+xRE7LratzpaR3|d7=d_POYIZLR@*@74KGYupc2@(!snq?fQw6b}|r!$et zVUUU}yP)yAVajGA<5Hw^D6X<5bjog%ISVFi%2dT+!sM`~y{BkrRi2{88*+X8*lqz` zna_Y-O!6tqB%RJ$L)-H`%kz2z-|_pcPKW6uHm{k@7K5XnX_}{H1lr;y(jdL>#jK1QVrV3RV2lddKF_7BOQRR?-$Fg_2?rXa@ zbiJX~|KC=bQ>HY@B9TsMksIEy_mHaV$0`NoO|AH)=>7rAf|SonrI^t49Qb?ywKi!w zuWf(mxI@S5xZVKi(pWO3qo9GHha;M?Kb{^#nTeFM;qESw>j1rwLti`UN0jbH5|uP$ ziR2MyDe{W);c~X2>vp|iW7|7+_qAmW)jw>goDOsK_=7%&8#ye2cLY^pGpbFc@Fh)%{i9WOP3 zCF3RPhsikR*<=OrvWC?}>$e|qgyx9^qof8vmXF@TXi{P>Mk8sRv2##kLW5i~0u7$= zl%gFY8L%9NH6wvYMMBRx7+Qh@`L8*R3+*M1E|At@2h?-j^u}ZLo_uWD0J1O@kZW@R z`NU*E7H0$Usp){Mo(~rLa+ew57P*^zxYe9+W44bUQIjQ?$fR13t$i?DAi;4;XA}O{ zoDn7okZ0tAjmS{~wklDhfyOYLr`WW7N)D&|m=iE8OLA>$<`l*`(exG-bm7Iaj-pui zf&81r3{6E{%CevfW>xTsRe?5TF*~P;4AnfPX3XY1R!^ZcfNslj$}6=6MUL~DWw#95 zZ_9RQcs@d#j6+0EGi*Ad=~%#WM+-uy!o-vYg;;Qglduvgp419O{~ZW0&3u$gHS_A} zNfp!+l&&g(cBE>yz#Jo741hqh43;hvS!YTL62IuhU@`xHV2KEYn)L%Qo#qLbFbVC3 z$gvh`5fnnZ5TCM2l1PRpS)nmdVax-TWDDAZ)n`zDkTS3-=L{b{Gfz0t=Y!4pGfqXaqcGvl#a?LxP%iA>ci*I04@z8xr_U z!knpwL6vp0iVoFn*E9wENXrsjW>6whY9aNipP-AVa;6Mbn7(Lo{~Kncy01}?-?0bZQ}O2J8mETxR^k?HPq8p^g-s<;g6U1bi@tf~N$7>H2XiW2je(TSMd{I-DK(UT$M^aN`uIMLf+U ziFtxmg&5HUEyov0J{>{-fOo3@=*(qgOO=Ce0xjF$ZZG9*aCsrC3puefN=q67RWjUy zJlRC#aYzy&Q7SRUu#g#4b2^)%azHa1p01^a5~xxn z;#gd07NA{KVp$&4TgWeDW2q`N3slyMW2jr#3Wp;3PNM3;h>6uE1dJgDK1dRP>$*K8 zz*^!$yNONo|Cah~lTN3z1@8h8=hGzHR;gX}2L5v32V_v0ef85Jy&?@W42rZqJb~Ab z9&=WX15lMj84DcnxvW%bt5h3ER9yZZN9+j422Yf^5-I-x)(6-+2NsfNEMX~FyTZhQ zMuKF<(8WY++EDQfC^Z<;u#C?$)Tw2A`CJzI^9&?cRoE_MrR^xpA{_LlCR~lxWy4Yk zdMA7yz6ONINd1G8P=YWjtLk;^-;eNQlrRy1$m0a#g$;3tD4ddh@#d#&s zr4p^@JOn<1`G6XgeJ{-_=1KA(W*`;J0Ih*(KgX~!c@BJ>@BEqZ1cuF+fb#R0k5Zmb zJLI{zKqXT#ZK&n~g(cMpN6}}9`2h7mALYNuflxL>(s!C!&~OmAT1hBn!Di~G{bsLK zzvimG6!{5FgE-!Q@LZ8EyfLBvoK6j2pzAtnZ+tmn`YC6C42z~#pDpW^ei z6^8EX3U*{lF~U56p|Q|nHM(%cfv6m9l!08yCR~nSEm_Cx|J-Ha`sqD2t7rDq9w{Os zz}4g%d+-Jp!Zeu6H*^%dnNjFJZDB}3RiQHN*n_WuoXDs_P8j6GXF^UaZaDgV%6ieL z=h#s{@M*8xXFlutecuZQgF!d^tjLLCt8Rmk6UNFL)KtX zOTC^w0^>1q2VQq%`y+p3F|W4{IkB$z<3&#Nov(tN=o{q3qeD)B%bcUfU#=B0<&rY6 zkf}(}5}PeCgQYM6F<6R?lm_nKTwP-4iVd9c)n#&ZDa|-}Xvr9q%R@_m&SqsKfIv6( zO4W>u2e()Td$fsiQJZa7{|wFW*boTVF*r;CXK_Yev2>bC{A@Xxot>TDEFc34+eFdc z0TlN9ki0s5`RsJ>guFUDJ^ABnviIVEG?VvH1IUxZ{g)?)${;K?9gE)@@WMowkMvTtnw!`)e;CLx>Oz1fyKEcR13~AJoDSgkU`E*D?8(go~_Z6N@@ueuklOWE+ z0$Zk#Cy4i`R(B#9mfG;NMxK>vFDe*SFqcKYUaUlyHFDOiLkJO^k6W6(t08=i>eXPS72X1v zKBXZmMN@)&3YZ?*`st@b{imK%e)>t_`;g0u$=6S1;b$5uB7A%5pP1Wn0M zy2vG3z_Ut@lPwJ|+eXY$`LYRLOT`8B{V4{?0OO#>n?szN|#8OW)CZ5PNqGbSqPqI^*L6mT@p4Lly|Sc9rx?Wj5nP&Vdx$29_+DZ_-!LZ<$w>Vj5}LP?5wTQ44lUv0u{PI7@l ztJbyx7F@u(dNxwfup8&i>`P6Yjdp%wek|lwPL3d6HDl^k!Qw=Ty4A?HV&43%&zMTd zDzM&tr!aDMZ;+^Uv};S#T?5?xu!4^*y;)F^xK}mPS;Lg%0>HC zGuY{8p21GP{tTwO01m0Po?}sbZv(2mgQm6crnY#8VZM^pQ0xziiAWmT2MNE&lx>3t zzEjM?oUvo@z$@{uQB5W@cZaRx(4vuwH?R05%5b{VB3Dr@?6KVN4;KP1zs39 z-k{&5o{!x6B;a>l@ka;v^%3yf^*i2|KPt_I*{FZhQEB$zD*=8_a`n|E7y2l9zI2d0 zUpz*>%H1-U-(PiFnh3A{5#vX*_+fVu!Z5gphlHP$BAHEqzET{6)OvXQ9IR$0%wS44 zMIr%3--=Nqg@=?vM8|nXGQ8(Sg5zm~u^D)|R|n6?LDQLP6SX5KlI6Qu3}kN8rTgpe z@aUV+#&0h|ZGz34s5U0pyeimy@0jf^FW%7Mi%p6q8!QiNbMwU;y>4$5 zSu`39d_VGSoAtT_E9?zi&+o$+_SQs`*A{<_(d7Qnb2?xCR#IosbNb$@V6xr)dZ$P1 zPiTnnup=5STuE9z*O6qM_10nJ_A(vd0QgtWavsMN3{Fj}nY2I-OSEo#Bk2;o6pv6m zp($rOKZuMq$So!|2ttvHnsX;=&w%@!w#ida7MAkhT}`c-RMXN0Dr)YXsGZT#2*rcv zeT;e5Y8?2XA8R^g){2s6g8#P;{;v)h(BK`EBPR#1q!95)yeH=Cx^HNRfhMgoA4-*$z(n zH&Ypsh$ndUY;mx0yTpP@mKHH97&ttbH&YPeOH(ypMXifP6_d1ym3;kJNwGRNv664x zB$!yqZ%U-=VkHCD_iv1qID>&TpkYMALD;u^$MT~t1t#_#hq=^R7b{s;{4vH#tfANI z_}wpl)TV3szH`%2n_mri_k@9x#w-QLTL9Ati{h3*+J*@Ig0aOXJ{N!TW5Vm4U-7M9O7wPt~PhE1M>rC?IF$x}MwtijLf zxh7ZSy=TZo#eAe>&@*{;Ih)XAp?8w~>+e-!lcBoQt)nMPQ6Y?Lu_*>>bj<0P=IOOS zRGHWa4wdC9y1hHm{SyVn6F_th&fT;@uY9ga>8ur<{+;N6p^#!a0;cXyQdlJkeY9g0 zziXt`6w5*tP5*AOFDY2?j>V{=F-wxBiWbj}=vR@sqBStluX{jWgBZ+P%%wr{Jt&5Z zPWnM6NCaJ)4p9tIP4#r8HyY z!H*W!Cqo)3Jm?gjgknsm+zf_?oiK7Kc!I~xz~X-dJ^=WHMk1R5$=`<+)nO+236+f4 z9To33o9o|@NN>Nn-V}A4Q1XP%S;lEXrc6$RzOEHSkDxcaeKWh$=1U3lrNnP__rKU~+N9loV{*;z|6>>F zn%#fL?>oI4cmG+R4n|#@b|c0jJonb|2bSw}Y2>(F*KMuY{a;u7G4B35L$BZQ`}UXL z{derX+j`*L{{w~_jI7v{|M8qPkoV1`=3Zx9#G>zm0XW5L3?r7e$)7lgNZl$~ZrfG~ zHL)^#>wf$HPq9#&{0}|n3bjxqSI;#em9urf9sg4rzNJ_H#2x>1Le+j|wdG$XG`fVj z`M>_Jb=ysUvwyp4|Mou9pU90j_P}}# ztP16+3D-YtxW4>y#mM1dM@D6#@DhZp%ha6i_0S@T;`}`xNjRn-j372a5O~`-Hy2`V z%5LJ*SX_+6`G8`y<5##mgJBqA6uJKHR^M1f%MWj(Slum> zihPXs(ZFy0{cEw_OUV>(H{rnR&QD3B-s+M{yb1=IGUXrO**PG`R7xH&@B7i1O$_uq%)qqDK(754&A`A3wTVje4 z(%EEgz9hcWU|F+q;F*mBvsY{OYRz8lZ+5R%Z4PYGUac{?X0P_Ki*(Iit!wvf`=-6x zz8!{^6*7zUt$xo9yS`=7KHhaQvU^@QYJGaIc5U%TxL50VL&xs8{ue*i&UL$9Z{=RC zZSUCKul1!imy+eFz9EG!oNU)fX?(7&^SAD|Q+o^}ipICPjzwo}-FA=GY-p}Ij%^>5 zUaGB{*#yJUdV3R6F4%~i93B5RIZD(z8JP->=h+))Que0AGi!WS>GC2 zea~?$7OiuH*A;(+j<9PFoqor0y)O@1?%Ve2Bg?#Fb0hA(gvzKcMnH zSOwg{+&KZe$AoN?)2qwzLLlLk{p;`JYlq}7m22Jays{@0%scqO8f@NLz?m-wE{cq; z`yEkse0ciu$2NIEQQ=WpHi zXtK?Yltd}rjjpxeDNb`=8vPxDft0j_f8QMTSZdp1tCrhgf){sK(lsyR#9&pZPFIgJ zBa4TW#rWz{q*s^G)urm-RDVxn1oB`4IS0!H%Q<7G9$pmqiC17itPd)KJi}{d!W^dx zBZdJJn zvos^AJX4Y$`pOngTUrSdo4iu7hZF`HVOsA1;LVza4b(4F6i+L%qP z2SInLDC#C{Y8jJjHnkqRNY`y@`M&SmxT!^ZG+<8Hr=i=Wfp0s^9e7^U>rvNroppB> zuPy!vH?_R(u-onQtS^u5wEDe%ul3+3@nmmV{=Q9qq|F;Zil>FUMlfINo~TX=42&Yy zCQJ9WT)(lU%-S-z&R?@taV7;!02zK}_%wypFeoC~12X`2EhAfJPxrRJKmDGZl9|4P z*F;4&BPxnE)L0PPD+D} zp*0$J$QG{Kx5@7hULFl?&@gM@?9#WC?NWMS_4|LokD_5d Date: Thu, 12 Oct 2023 09:13:42 +0200 Subject: [PATCH 057/148] [graph cleaning] avoid NPEs --- .../oaf/utils/GraphCleaningFunctions.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java index 3c3e8052e..324e3dd58 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java @@ -509,12 +509,19 @@ public class GraphCleaningFunctions extends CleaningFunctions { // from the script from Dimitris if ("0000".equals(i.getRefereed().getClassid())) { - final boolean isFromCrossref = ModelConstants.CROSSREF_ID - .equals(i.getCollectedfrom().getKey()); - final boolean hasDoi = i - .getPid() - .stream() - .anyMatch(pid -> PidType.doi.toString().equals(pid.getQualifier().getClassid())); + final boolean isFromCrossref = Optional + .ofNullable(i.getCollectedfrom()) + .map(KeyValue::getKey) + .map(id -> id.equals(ModelConstants.CROSSREF_ID)) + .orElse(false); + final boolean hasDoi = Optional + .ofNullable(i.getPid()) + .map( + pid -> pid + .stream() + .anyMatch( + p -> PidType.doi.toString().equals(p.getQualifier().getClassid()))) + .orElse(false); final boolean isPeerReviewedType = PEER_REVIEWED_TYPES .contains(i.getInstancetype().getClassname()); final boolean noOtherLitType = r From dda602fff7c65341d0db9dcb9b0b5db2cf5be7ee Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Thu, 12 Oct 2023 10:05:46 +0200 Subject: [PATCH 058/148] [AMF] docs --- dhp-workflows/dhp-actionmanager/README.md | 72 +++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 dhp-workflows/dhp-actionmanager/README.md diff --git a/dhp-workflows/dhp-actionmanager/README.md b/dhp-workflows/dhp-actionmanager/README.md new file mode 100644 index 000000000..9899c4a98 --- /dev/null +++ b/dhp-workflows/dhp-actionmanager/README.md @@ -0,0 +1,72 @@ +# Action Management Framework + +This module implements the oozie workflow for the integration of pre-built contents into the OpenAIRE Graph. + +Such contents can be + +* brand new, non-existing records to be introduced as nodes of the graph +* updates (or enrichment) for records that does exist in the graph (e.g. a new subject term for a publication) +* relations among existing nodes + +The actionset contents are organised into logical containers, each of them can contain multiple versions contents and is characterised by + +* a name +* an identifier +* the paths on HDFS where each version of the contents is stored + +Each version is then characterised by + +* the creation date +* the last update date +* the indication where it is the latest one or it is an expired version, candidate for garbage collection + +## ActionSet serialization + +Each actionset version contains records compliant to the graph internal data model, i.e. subclasses of `eu.dnetlib.dhp.schema.oaf.Oaf`, +defined in the external schemas module + +``` + + eu.dnetlib.dhp + ${dhp-schemas.artifact} + ${dhp-schemas.version} + +``` + +When the actionset contains a relationship, the model class to use is `eu.dnetlib.dhp.schema.oaf.Relation`, otherwise +when the actionset contains an entity, it is a `eu.dnetlib.dhp.schema.oaf.OafEntity` or one of its subclasses +`Datasource`, `Organization`, `Project`, `Result` (or one of its subclasses `Publication`, `Dataset`, etc...). + +Then, each OpenAIRE Graph model class instance must be wrapped using the class `eu.dnetlib.dhp.schema.action.AtomicAction`, a generic +container that defines two attributes + +* `T payload` the OpenAIRE Graph class instance containing the data; +* `Class clazz` must contain the class whose instance is contained in the payload. + +Each AtomicAction can be then serialised in JSON format using `com.fasterxml.jackson.databind.ObjectMapper` from + +``` + + com.fasterxml.jackson.core + jackson-databind + ${dhp.jackson.version} + +``` + +Then, the JSON serialization must be stored as a GZip compressed sequence file (`org.apache.hadoop.mapred.SequenceFileOutputFormat`). +As such, it contains a set of tuples, a key and a value defined as `org.apache.hadoop.io.Text` where + +* the `key` must be set to the class canonical name contained in the `AtomicAction`; +* the `value` must be set to the AtomicAction JSON serialization. + +The following snippet provides an example of how create an actionset version of Relation records: + +``` + rels // JavaRDD + .map(relation -> new AtomicAction(Relation.class, relation)) + .mapToPair( + aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()), + new Text(OBJECT_MAPPER.writeValueAsString(aa)))) + .saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, GzipCodec.class); +``` + From 76447958bb538c75872d6b5f0fef184e97b42d55 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Thu, 12 Oct 2023 12:23:20 +0200 Subject: [PATCH 059/148] cleanup & docs --- README.md | 128 +++++++++++++++++- dhp-workflows/dhp-distcp/pom.xml | 13 -- .../dhp/distcp/oozie_app/config-default.xml | 18 --- .../dnetlib/dhp/distcp/oozie_app/workflow.xml | 46 ------- dhp-workflows/docs/oozie-installer.markdown | 111 --------------- dhp-workflows/pom.xml | 1 - 6 files changed, 127 insertions(+), 190 deletions(-) delete mode 100644 dhp-workflows/dhp-distcp/pom.xml delete mode 100644 dhp-workflows/dhp-distcp/src/main/resources/eu/dnetlib/dhp/distcp/oozie_app/config-default.xml delete mode 100644 dhp-workflows/dhp-distcp/src/main/resources/eu/dnetlib/dhp/distcp/oozie_app/workflow.xml delete mode 100644 dhp-workflows/docs/oozie-installer.markdown diff --git a/README.md b/README.md index 0a0bd82ab..2c1440f44 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,128 @@ # dnet-hadoop -Dnet-hadoop is the project that defined all the OOZIE workflows for the OpenAIRE Graph construction, processing, provisioning. \ No newline at end of file + +Dnet-hadoop is the project that defined all the [OOZIE workflows](https://oozie.apache.org/) for the OpenAIRE Graph construction, processing, provisioning. + +How to build, package and run oozie workflows +==================== + +Oozie-installer is a utility allowing building, uploading and running oozie workflows. In practice, it creates a `*.tar.gz` +package that contains resources that define a workflow and some helper scripts. + +This module is automatically executed when running: + +`mvn package -Poozie-package -Dworkflow.source.dir=classpath/to/parent/directory/of/oozie_app` + +on module having set: + +``` + + eu.dnetlib.dhp + dhp-workflows + +``` + +in `pom.xml` file. `oozie-package` profile initializes oozie workflow packaging, `workflow.source.dir` property points to +a workflow (notice: this is not a relative path but a classpath to directory usually holding `oozie_app` subdirectory). + +The outcome of this packaging is `oozie-package.tar.gz` file containing inside all the resources required to run Oozie workflow: + +- jar packages +- workflow definitions +- job properties +- maintenance scripts + +Required properties +==================== + +In order to include proper workflow within package, `workflow.source.dir` property has to be set. It could be provided +by setting `-Dworkflow.source.dir=some/job/dir` maven parameter. + +In oder to define full set of cluster environment properties one should create `~/.dhp/application.properties` file with +the following properties: + +- `dhp.hadoop.frontend.user.name` - your user name on hadoop cluster and frontend machine +- `dhp.hadoop.frontend.host.name` - frontend host name +- `dhp.hadoop.frontend.temp.dir` - frontend directory for temporary files +- `dhp.hadoop.frontend.port.ssh` - frontend machine ssh port +- `oozieServiceLoc` - oozie service location required by run_workflow.sh script executing oozie job +- `nameNode` - name node address +- `jobTracker` - job tracker address +- `oozie.execution.log.file.location` - location of file that will be created when executing oozie job, it contains output +produced by `run_workflow.sh` script (needed to obtain oozie job id) +- `maven.executable` - mvn command location, requires parameterization due to a different setup of CI cluster +- `sparkDriverMemory` - amount of memory assigned to spark jobs driver +- `sparkExecutorMemory` - amount of memory assigned to spark jobs executors +- `sparkExecutorCores` - number of cores assigned to spark jobs executors + +All values will be overriden with the ones from `job.properties` and eventually `job-override.properties` stored in module's +main folder. + +When overriding properties from `job.properties`, `job-override.properties` file can be created in main module directory +(the one containing `pom.xml` file) and define all new properties which will override existing properties. +One can provide those properties one by one as command line `-D` arguments. + +Properties overriding order is the following: + +1. `pom.xml` defined properties (located in the project root dir) +2. `~/.dhp/application.properties` defined properties +3. `${workflow.source.dir}/job.properties` +4. `job-override.properties` (located in the project root dir) +5. `maven -Dparam=value` + +where the maven `-Dparam` property is overriding all the other ones. + +Workflow definition requirements +==================== + +`workflow.source.dir` property should point to the following directory structure: + + [${workflow.source.dir}] + | + |-job.properties (optional) + | + \-[oozie_app] + | + \-workflow.xml + +This property can be set using maven `-D` switch. + +`[oozie_app]` is the default directory name however it can be set to any value as soon as `oozieAppDir` property is +provided with directory name as value. + +Sub-workflows are supported as well and sub-workflow directories should be nested within `[oozie_app]` directory. + +Creating oozie installer step-by-step +===================================== + +Automated oozie-installer steps are the following: + +1. creating jar packages: `*.jar` and `*tests.jar` along with copying all dependencies in `target/dependencies` +2. reading properties from maven, `~/.dhp/application.properties`, `job.properties`, `job-override.properties` +3. invoking priming mechanism linking resources from import.txt file (currently resolving subworkflow resources) +4. assembling shell scripts for preparing Hadoop filesystem, uploading Oozie application and starting workflow +5. copying whole `${workflow.source.dir}` content to `target/${oozie.package.file.name}` +6. generating updated `job.properties` file in `target/${oozie.package.file.name}` based on maven, +`~/.dhp/application.properties`, `job.properties` and `job-override.properties` +7. creating `lib` directory (or multiple directories for sub-workflows for each nested directory) and copying jar packages +created at step (1) to each one of them +8. bundling whole `${oozie.package.file.name}` directory into single tar.gz package + +Uploading oozie package and running workflow on cluster +======================================================= + +In order to simplify deployment and execution process two dedicated profiles were introduced: + +- `deploy` +- `run` + +to be used along with `oozie-package` profile e.g. by providing `-Poozie-package,deploy,run` maven parameters. + +The `deploy` profile supplements packaging process with: +1) uploading oozie-package via scp to `/home/${user.name}/oozie-packages` directory on `${dhp.hadoop.frontend.host.name}` machine +2) extracting uploaded package +3) uploading oozie content to hadoop cluster HDFS location defined in `oozie.wf.application.path` property (generated dynamically by maven build process, based on `${dhp.hadoop.frontend.user.name}` and `workflow.source.dir` properties) + +The `run` profile introduces: +1) executing oozie application uploaded to HDFS cluster using `deploy` command. Triggers `run_workflow.sh` script providing runtime properties defined in `job.properties` file. + +Notice: ssh access to frontend machine has to be configured on system level and it is preferable to set key-based authentication in order to simplify remote operations. \ No newline at end of file diff --git a/dhp-workflows/dhp-distcp/pom.xml b/dhp-workflows/dhp-distcp/pom.xml deleted file mode 100644 index c3d3a7375..000000000 --- a/dhp-workflows/dhp-distcp/pom.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - dhp-workflows - eu.dnetlib.dhp - 1.2.5-SNAPSHOT - - 4.0.0 - - dhp-distcp - - - \ No newline at end of file diff --git a/dhp-workflows/dhp-distcp/src/main/resources/eu/dnetlib/dhp/distcp/oozie_app/config-default.xml b/dhp-workflows/dhp-distcp/src/main/resources/eu/dnetlib/dhp/distcp/oozie_app/config-default.xml deleted file mode 100644 index 905fb9984..000000000 --- a/dhp-workflows/dhp-distcp/src/main/resources/eu/dnetlib/dhp/distcp/oozie_app/config-default.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - jobTracker - yarnRM - - - nameNode - hdfs://nameservice1 - - - sourceNN - webhdfs://namenode2.hadoop.dm.openaire.eu:50071 - - - oozie.use.system.libpath - true - - \ No newline at end of file diff --git a/dhp-workflows/dhp-distcp/src/main/resources/eu/dnetlib/dhp/distcp/oozie_app/workflow.xml b/dhp-workflows/dhp-distcp/src/main/resources/eu/dnetlib/dhp/distcp/oozie_app/workflow.xml deleted file mode 100644 index 91b97332b..000000000 --- a/dhp-workflows/dhp-distcp/src/main/resources/eu/dnetlib/dhp/distcp/oozie_app/workflow.xml +++ /dev/null @@ -1,46 +0,0 @@ - - - - sourceNN - the source name node - - - sourcePath - the source path - - - targetPath - the target path - - - hbase_dump_distcp_memory_mb - 6144 - memory for distcp action copying InfoSpace dump from remote cluster - - - hbase_dump_distcp_num_maps - 1 - maximum number of simultaneous copies of InfoSpace dump from remote location - - - - - - - Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] - - - - - -Dmapreduce.map.memory.mb=${hbase_dump_distcp_memory_mb} - -pb - -m ${hbase_dump_distcp_num_maps} - ${sourceNN}/${sourcePath} - ${nameNode}/${targetPath} - - - - - - - \ No newline at end of file diff --git a/dhp-workflows/docs/oozie-installer.markdown b/dhp-workflows/docs/oozie-installer.markdown deleted file mode 100644 index d2de80dcc..000000000 --- a/dhp-workflows/docs/oozie-installer.markdown +++ /dev/null @@ -1,111 +0,0 @@ -General notes -==================== - -Oozie-installer is a utility allowing building, uploading and running oozie workflows. In practice, it creates a `*.tar.gz` package that contains resouces that define a workflow and some helper scripts. - -This module is automatically executed when running: - -`mvn package -Poozie-package -Dworkflow.source.dir=classpath/to/parent/directory/of/oozie_app` - -on module having set: - - - eu.dnetlib.dhp - dhp-workflows - - -in `pom.xml` file. `oozie-package` profile initializes oozie workflow packaging, `workflow.source.dir` property points to a workflow (notice: this is not a relative path but a classpath to directory usually holding `oozie_app` subdirectory). - -The outcome of this packaging is `oozie-package.tar.gz` file containing inside all the resources required to run Oozie workflow: - -- jar packages -- workflow definitions -- job properties -- maintenance scripts - -Required properties -==================== - -In order to include proper workflow within package, `workflow.source.dir` property has to be set. It could be provided by setting `-Dworkflow.source.dir=some/job/dir` maven parameter. - -In oder to define full set of cluster environment properties one should create `~/.dhp/application.properties` file with the following properties: - -- `dhp.hadoop.frontend.user.name` - your user name on hadoop cluster and frontend machine -- `dhp.hadoop.frontend.host.name` - frontend host name -- `dhp.hadoop.frontend.temp.dir` - frontend directory for temporary files -- `dhp.hadoop.frontend.port.ssh` - frontend machine ssh port -- `oozieServiceLoc` - oozie service location required by run_workflow.sh script executing oozie job -- `nameNode` - name node address -- `jobTracker` - job tracker address -- `oozie.execution.log.file.location` - location of file that will be created when executing oozie job, it contains output produced by `run_workflow.sh` script (needed to obtain oozie job id) -- `maven.executable` - mvn command location, requires parameterization due to a different setup of CI cluster -- `sparkDriverMemory` - amount of memory assigned to spark jobs driver -- `sparkExecutorMemory` - amount of memory assigned to spark jobs executors -- `sparkExecutorCores` - number of cores assigned to spark jobs executors - -All values will be overriden with the ones from `job.properties` and eventually `job-override.properties` stored in module's main folder. - -When overriding properties from `job.properties`, `job-override.properties` file can be created in main module directory (the one containing `pom.xml` file) and define all new properties which will override existing properties. One can provide those properties one by one as command line -D arguments. - -Properties overriding order is the following: - -1. `pom.xml` defined properties (located in the project root dir) -2. `~/.dhp/application.properties` defined properties -3. `${workflow.source.dir}/job.properties` -4. `job-override.properties` (located in the project root dir) -5. `maven -Dparam=value` - -where the maven `-Dparam` property is overriding all the other ones. - -Workflow definition requirements -==================== - -`workflow.source.dir` property should point to the following directory structure: - - [${workflow.source.dir}] - | - |-job.properties (optional) - | - \-[oozie_app] - | - \-workflow.xml - -This property can be set using maven `-D` switch. - -`[oozie_app]` is the default directory name however it can be set to any value as soon as `oozieAppDir` property is provided with directory name as value. - -Subworkflows are supported as well and subworkflow directories should be nested within `[oozie_app]` directory. - -Creating oozie installer step-by-step -===================================== - -Automated oozie-installer steps are the following: - -1. creating jar packages: `*.jar` and `*tests.jar` along with copying all dependancies in `target/dependencies` -2. reading properties from maven, `~/.dhp/application.properties`, `job.properties`, `job-override.properties` -3. invoking priming mechanism linking resources from import.txt file (currently resolving subworkflow resources) -4. assembling shell scripts for preparing Hadoop filesystem, uploading Oozie application and starting workflow -5. copying whole `${workflow.source.dir}` content to `target/${oozie.package.file.name}` -6. generating updated `job.properties` file in `target/${oozie.package.file.name}` based on maven, `~/.dhp/application.properties`, `job.properties` and `job-override.properties` -7. creating `lib` directory (or multiple directories for subworkflows for each nested directory) and copying jar packages created at step (1) to each one of them -8. bundling whole `${oozie.package.file.name}` directory into single tar.gz package - -Uploading oozie package and running workflow on cluster -======================================================= - -In order to simplify deployment and execution process two dedicated profiles were introduced: - -- `deploy` -- `run` - -to be used along with `oozie-package` profile e.g. by providing `-Poozie-package,deploy,run` maven parameters. - -`deploy` profile supplements packaging process with: -1) uploading oozie-package via scp to `/home/${user.name}/oozie-packages` directory on `${dhp.hadoop.frontend.host.name}` machine -2) extracting uploaded package -3) uploading oozie content to hadoop cluster HDFS location defined in `oozie.wf.application.path` property (generated dynamically by maven build process, based on `${dhp.hadoop.frontend.user.name}` and `workflow.source.dir` properties) - -`run` profile introduces: -1) executing oozie application uploaded to HDFS cluster using `deploy` command. Triggers `run_workflow.sh` script providing runtime properties defined in `job.properties` file. - -Notice: ssh access to frontend machine has to be configured on system level and it is preferable to set key-based authentication in order to simplify remote operations. \ No newline at end of file diff --git a/dhp-workflows/pom.xml b/dhp-workflows/pom.xml index 64f5f2d26..369c71b5b 100644 --- a/dhp-workflows/pom.xml +++ b/dhp-workflows/pom.xml @@ -25,7 +25,6 @@ dhp-workflow-profiles dhp-aggregation - dhp-distcp dhp-actionmanager dhp-graph-mapper dhp-dedup-openaire From 6cf64d5d8b3b9826bce76d94f0548ee96eff2736 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 13 Oct 2023 10:09:26 +0200 Subject: [PATCH 060/148] [SWH] renamed 'Software Heritage Identifier' to 'Software Hash Identifier' --- .../src/main/java/eu/dnetlib/dhp/swh/utils/SWHConstants.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConstants.java b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConstants.java index eae839cfd..2a0403044 100644 --- a/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConstants.java +++ b/dhp-workflows/dhp-swh/src/main/java/eu/dnetlib/dhp/swh/utils/SWHConstants.java @@ -12,7 +12,7 @@ public class SWHConstants { public static final String SWHID = "swhid"; - public static final String SWHID_CLASSNAME = "Software Heritage Identifier"; + public static final String SWHID_CLASSNAME = "Software Hash Identifier"; public static final String SWH_ID = "10|openaire____::dbfd07503aaa1ed31beed7dec942f3f4"; From 54fbf09ac63d1076a51ab791f093b33d9c4520bb Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Mon, 16 Oct 2023 08:57:47 +0200 Subject: [PATCH 061/148] [raw graph] WIP: mapping original resource types --- .../raw/AbstractMdRecordToOafMapper.java | 64 +- .../dhp/oa/graph/raw/OafToOafMapper.java | 28 +- .../dhp/oa/graph/raw/OdfToOafMapper.java | 22 +- .../dnetlib/dhp/oa/graph/raw/MappersTest.java | 60 +- .../dnetlib/dhp/oa/graph/clean/synonyms.txt | 855 +++++++++++++++++- .../eu/dnetlib/dhp/oa/graph/clean/terms.txt | 292 +++++- 6 files changed, 1244 insertions(+), 77 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java index 2fed7d627..5f0acbab7 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java @@ -42,10 +42,6 @@ public abstract class AbstractMdRecordToOafMapper { protected static final String DATACITE_SCHEMA_KERNEL_3 = "http://datacite.org/schema/kernel-3"; protected static final String DATACITE_SCHEMA_KERNEL_3_SLASH = "http://datacite.org/schema/kernel-3/"; - protected static final String OPENAIRE_COAR_RESOURCE_TYPES_3_1 = "openaire::coar_resource_types_3_1"; - - public static final String OPENAIRE_USER_RESOURCE_TYPES = "openaire::user_resource_types"; - protected static final Qualifier ORCID_PID_TYPE = qualifier( ModelConstants.ORCID_PENDING, ModelConstants.ORCID_CLASSNAME, @@ -129,7 +125,9 @@ public abstract class AbstractMdRecordToOafMapper { final String type = getResultType(doc, instances); - return createOafs(doc, type, instances, collectedFrom, entityInfo, lastUpdateTimestamp); + final Qualifier metaResourceType = getMetaResourceType(instances); + + return createOafs(doc, type, metaResourceType, instances, collectedFrom, entityInfo, lastUpdateTimestamp); } catch (DocumentException e) { log.error("Error with record:\n" + xml); return Lists.newArrayList(); @@ -155,6 +153,30 @@ public abstract class AbstractMdRecordToOafMapper { return type; } + protected Qualifier getMetaResourceType(final List instances) { + + if (vocs.vocabularyExists(OPENAIRE_META_RESOURCE_TYPE)) { + Optional instanceTypeMapping = instances + .stream() + .flatMap(i -> i.getInstanceTypeMapping().stream()) + .filter(t -> OPENAIRE_COAR_RESOURCE_TYPES_3_1.equals(t.getVocabularyName())) + .findFirst(); + + if (!instanceTypeMapping.isPresent()) { + throw new IllegalStateException("unable to find an instance from " + OPENAIRE_COAR_RESOURCE_TYPES_3_1); + } else { + final String typeCode = instanceTypeMapping.get().getTypeCode(); + return Optional + .ofNullable(vocs.lookupTermBySynonym(OPENAIRE_META_RESOURCE_TYPE, typeCode)) + .orElseThrow(() -> + new IllegalStateException("unable to find a synonym for '" + typeCode + "' in " + + OPENAIRE_META_RESOURCE_TYPE)); + } + } else { + throw new IllegalStateException("vocabulary '" + OPENAIRE_META_RESOURCE_TYPE + "' not available"); + } + } + private KeyValue getProvenanceDatasource(final Document doc, final String xpathId, final String xpathName) { final String dsId = doc.valueOf(xpathId); final String dsName = doc.valueOf(xpathName); @@ -169,12 +191,13 @@ public abstract class AbstractMdRecordToOafMapper { protected List createOafs( final Document doc, final String type, + final Qualifier metaResourceType, final List instances, final KeyValue collectedFrom, final DataInfo info, final long lastUpdateTimestamp) { - final OafEntity entity = createEntity(doc, type, instances, collectedFrom, info, lastUpdateTimestamp); + final OafEntity entity = createEntity(doc, type, metaResourceType, instances, collectedFrom, info, lastUpdateTimestamp); final Set originalId = Sets.newHashSet(entity.getOriginalId()); originalId.add(entity.getId()); @@ -207,6 +230,7 @@ public abstract class AbstractMdRecordToOafMapper { private OafEntity createEntity(final Document doc, final String type, + final Qualifier metaResourceType, final List instances, final KeyValue collectedFrom, final DataInfo info, @@ -214,12 +238,12 @@ public abstract class AbstractMdRecordToOafMapper { switch (type.toLowerCase()) { case "publication": final Publication p = new Publication(); - populateResultFields(p, doc, instances, collectedFrom, info, lastUpdateTimestamp); + populateResultFields(p, metaResourceType, doc, instances, collectedFrom, info, lastUpdateTimestamp); p.setJournal(prepareJournal(doc, info)); return p; case "dataset": final Dataset d = new Dataset(); - populateResultFields(d, doc, instances, collectedFrom, info, lastUpdateTimestamp); + populateResultFields(d, metaResourceType, doc, instances, collectedFrom, info, lastUpdateTimestamp); d.setStoragedate(prepareDatasetStorageDate(doc, info)); d.setDevice(prepareDatasetDevice(doc, info)); d.setSize(prepareDatasetSize(doc, info)); @@ -230,7 +254,7 @@ public abstract class AbstractMdRecordToOafMapper { return d; case "software": final Software s = new Software(); - populateResultFields(s, doc, instances, collectedFrom, info, lastUpdateTimestamp); + populateResultFields(s, metaResourceType, doc, instances, collectedFrom, info, lastUpdateTimestamp); s.setDocumentationUrl(prepareSoftwareDocumentationUrls(doc, info)); s.setLicense(prepareSoftwareLicenses(doc, info)); s.setCodeRepositoryUrl(prepareSoftwareCodeRepositoryUrl(doc, info)); @@ -240,7 +264,7 @@ public abstract class AbstractMdRecordToOafMapper { case "otherresearchproducts": default: final OtherResearchProduct o = new OtherResearchProduct(); - populateResultFields(o, doc, instances, collectedFrom, info, lastUpdateTimestamp); + populateResultFields(o, metaResourceType, doc, instances, collectedFrom, info, lastUpdateTimestamp); o.setContactperson(prepareOtherResearchProductContactPersons(doc, info)); o.setContactgroup(prepareOtherResearchProductContactGroups(doc, info)); o.setTool(prepareOtherResearchProductTools(doc, info)); @@ -377,11 +401,13 @@ public abstract class AbstractMdRecordToOafMapper { private void populateResultFields( final Result r, + final Qualifier metaResourceType, final Document doc, final List instances, final KeyValue collectedFrom, final DataInfo info, final long lastUpdateTimestamp) { + r.setMetaResourceType(metaResourceType); r.setDataInfo(info); r.setLastupdatetimestamp(lastUpdateTimestamp); r.setId(createOpenaireId(50, doc.valueOf("//dri:objIdentifier"), false)); @@ -531,15 +557,17 @@ public abstract class AbstractMdRecordToOafMapper { if (vocs.vocabularyExists(OPENAIRE_COAR_RESOURCE_TYPES_3_1)) { // TODO verify what the vocabs return when a synonym is not defined - Qualifier coarTerm = vocs.lookupTermBySynonym(OPENAIRE_COAR_RESOURCE_TYPES_3_1, originalType); - mappings.add(OafMapperUtils.instanceTypeMapping(originalType, coarTerm)); + Optional.ofNullable(vocs.lookupTermBySynonym(OPENAIRE_COAR_RESOURCE_TYPES_3_1, originalType)) + .ifPresent(coarTerm -> { + mappings.add(OafMapperUtils.instanceTypeMapping(originalType, coarTerm)); + if (vocs.vocabularyExists(OPENAIRE_USER_RESOURCE_TYPES)) { - if (vocs.vocabularyExists(OPENAIRE_USER_RESOURCE_TYPES)) { - - // TODO verify what the vocabs return when a synonym is not defined - Qualifier userTerm = vocs.lookupTermBySynonym(OPENAIRE_USER_RESOURCE_TYPES, coarTerm.getClassid()); - mappings.add(OafMapperUtils.instanceTypeMapping(originalType, userTerm)); - } + // TODO verify what the vocabs return when a synonym is not defined + Optional + .ofNullable(vocs.lookupTermBySynonym(OPENAIRE_USER_RESOURCE_TYPES, coarTerm.getClassid())) + .ifPresent(type -> mappings.add(OafMapperUtils.instanceTypeMapping(originalType, type))); + } + }); } return mappings; diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java index 5cdb434f1..6bcc8ec44 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java @@ -1,29 +1,27 @@ package eu.dnetlib.dhp.oa.graph.raw; -import static eu.dnetlib.dhp.schema.common.ModelConstants.*; -import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.*; - -import java.net.URLDecoder; -import java.util.*; -import java.util.stream.Collectors; - -import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils; -import org.apache.commons.lang3.StringUtils; -import org.dom4j.Document; -import org.dom4j.Element; -import org.dom4j.Node; - import com.google.common.collect.Lists; - import eu.dnetlib.dhp.common.PacePerson; import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup; import eu.dnetlib.dhp.schema.oaf.*; import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions; import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; import eu.dnetlib.dhp.schema.oaf.utils.ModelHardLimits; +import org.apache.commons.lang3.StringUtils; +import org.dom4j.Document; +import org.dom4j.Element; +import org.dom4j.Node; -import static org.apache.commons.lang3.StringUtils.contains; +import java.net.URLDecoder; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static eu.dnetlib.dhp.schema.common.ModelConstants.*; +import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.*; public class OafToOafMapper extends AbstractMdRecordToOafMapper { diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java index c01775327..c383e9300 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java @@ -220,11 +220,6 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper { } } - @Override - protected List prepareInstanceTypeMapping(Document doc) { - return null; - } - /** * The Datacite element * @@ -235,13 +230,20 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper { */ @Override protected String findOriginalType(Document doc) { - final Element resourceType = (Element) doc.selectSingleNode( - "//metadata/*[local-name() = 'resource']/*[local-name() = 'resourceType']"); + String resourceType = Optional.ofNullable((Element) doc.selectSingleNode( + "//*[local-name()='metadata']/*[local-name() = 'resource']/*[local-name() = 'resourceType']")) + .map(element -> { + final String resourceTypeURI = element.attributeValue("anyURI"); + final String resourceTypeTxt = element.getText(); - final String resourceTypeURI = resourceType.attributeValue("anyURI"); - final String resourceTypeTxt = resourceType.getText(); + return ObjectUtils.firstNonNull(resourceTypeURI, resourceTypeTxt); + }) + .orElse(doc.valueOf( + "//*[local-name()='metadata']/*[local-name() = 'resource']/*[local-name() = 'CobjCategory']/text()") + ); + + return resourceType; - return ObjectUtils.firstNonNull(resourceTypeURI, resourceTypeTxt); } @Override diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java index 5df2b7a3b..4d30a3fa8 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java @@ -74,7 +74,7 @@ class MappersTest { assertTrue(StringUtils.isNotBlank(p.getDateofcollection())); assertTrue(StringUtils.isNotBlank(p.getDateoftransformation())); - assertTrue(p.getAuthor().size() > 0); + assertFalse(p.getAuthor().isEmpty()); final Optional author = p .getAuthor() .stream() @@ -97,14 +97,14 @@ class MappersTest { assertEquals("Votsi", author.get().getSurname()); assertEquals("Nefta", author.get().getName()); - assertTrue(p.getSubject().size() > 0); + assertFalse(p.getSubject().isEmpty()); assertTrue(StringUtils.isNotBlank(p.getJournal().getIssnOnline())); assertTrue(StringUtils.isNotBlank(p.getJournal().getName())); assertTrue(p.getPid().isEmpty()); assertNotNull(p.getInstance()); - assertTrue(p.getInstance().size() > 0); + assertFalse(p.getInstance().isEmpty()); p .getInstance() .forEach(i -> { @@ -121,7 +121,7 @@ class MappersTest { Optional coarType = instance.getInstanceTypeMapping() .stream() - .filter(itm -> AbstractMdRecordToOafMapper.OPENAIRE_COAR_RESOURCE_TYPES_3_1.equals(itm.getVocabularyName())) + .filter(itm -> ModelConstants.OPENAIRE_COAR_RESOURCE_TYPES_3_1.equals(itm.getVocabularyName())) .findFirst(); assertTrue(coarType.isPresent()); @@ -130,7 +130,7 @@ class MappersTest { Optional userType = instance.getInstanceTypeMapping() .stream() - .filter(itm -> AbstractMdRecordToOafMapper.OPENAIRE_USER_RESOURCE_TYPES.equals(itm.getVocabularyName())) + .filter(itm -> ModelConstants.OPENAIRE_USER_RESOURCE_TYPES.equals(itm.getVocabularyName())) .findFirst(); assertTrue(userType.isPresent()); @@ -228,7 +228,7 @@ class MappersTest { assertTrue(StringUtils.isNotBlank(p.getDateofcollection())); assertTrue(StringUtils.isNotBlank(p.getDateoftransformation())); - assertTrue(p.getAuthor().size() > 0); + assertFalse(p.getAuthor().isEmpty()); final Optional author = p .getAuthor() .stream() @@ -251,13 +251,13 @@ class MappersTest { assertEquals("Votsi", author.get().getSurname()); assertEquals("Nefta", author.get().getName()); - assertTrue(p.getSubject().size() > 0); - assertTrue(p.getPid().size() > 0); + assertFalse(p.getSubject().isEmpty()); + assertFalse(p.getPid().isEmpty()); assertEquals("PMC1517292", p.getPid().get(0).getValue()); assertEquals("pmc", p.getPid().get(0).getQualifier().getClassid()); assertNotNull(p.getInstance()); - assertTrue(p.getInstance().size() > 0); + assertFalse(p.getInstance().isEmpty()); p .getInstance() .forEach(i -> { @@ -287,7 +287,7 @@ class MappersTest { final List list = new OafToOafMapper(vocs, true, true).processMdRecord(xml); - assertTrue(list.size() > 0); + assertFalse(list.isEmpty()); assertTrue(list.get(0) instanceof Publication); final Publication p = (Publication) list.get(0); @@ -343,7 +343,7 @@ class MappersTest { assertTrue(d.getOriginalId().stream().anyMatch(oid -> oid.equals("oai:zenodo.org:3234526"))); assertValidId(d.getCollectedfrom().get(0).getKey()); assertTrue(StringUtils.isNotBlank(d.getTitle().get(0).getValue())); - assertTrue(d.getAuthor().size() > 0); + assertFalse(d.getAuthor().isEmpty()); final Optional author = d .getAuthor() @@ -377,13 +377,13 @@ class MappersTest { final Field affiliation = opAff.get(); assertEquals("ISTI-CNR", affiliation.getValue()); - assertTrue(d.getSubject().size() > 0); - assertTrue(d.getInstance().size() > 0); - assertTrue(d.getContext().size() > 0); - assertTrue(d.getContext().get(0).getId().length() > 0); + assertFalse(d.getSubject().isEmpty()); + assertFalse(d.getInstance().isEmpty()); + assertFalse(d.getContext().isEmpty()); + assertFalse(d.getContext().get(0).getId().isEmpty()); assertNotNull(d.getInstance()); - assertTrue(d.getInstance().size() > 0); + assertFalse(d.getInstance().isEmpty()); d .getInstance() .forEach(i -> { @@ -457,7 +457,7 @@ class MappersTest { // assertEquals("oai:pub.uni-bielefeld.de:2949739", p.getOriginalId().get(0)); assertValidId(p.getCollectedfrom().get(0).getKey()); - assertTrue(p.getAuthor().size() > 0); + assertFalse(p.getAuthor().isEmpty()); final Optional author = p .getAuthor() @@ -469,14 +469,14 @@ class MappersTest { assertEquals("Potwarka", author.get().getSurname()); assertEquals("Luke R.", author.get().getName()); - assertTrue(p.getSubject().size() > 0); - assertTrue(p.getInstance().size() > 0); + assertFalse(p.getSubject().isEmpty()); + assertFalse(p.getInstance().isEmpty()); assertNotNull(p.getTitle()); assertFalse(p.getTitle().isEmpty()); assertNotNull(p.getInstance()); - assertTrue(p.getInstance().size() > 0); + assertFalse(p.getInstance().isEmpty()); p .getInstance() .forEach(i -> { @@ -624,9 +624,9 @@ class MappersTest { assertValidId(s.getId()); assertValidId(s.getCollectedfrom().get(0).getKey()); assertTrue(StringUtils.isNotBlank(s.getTitle().get(0).getValue())); - assertTrue(s.getAuthor().size() > 0); - assertTrue(s.getSubject().size() > 0); - assertTrue(s.getInstance().size() > 0); + assertFalse(s.getAuthor().isEmpty()); + assertFalse(s.getSubject().isEmpty()); + assertFalse(s.getInstance().isEmpty()); final Relation r1 = (Relation) list.get(1); final Relation r2 = (Relation) list.get(2); @@ -882,7 +882,7 @@ class MappersTest { assertEquals(2, p.getOriginalId().size()); assertTrue(p.getOriginalId().stream().anyMatch(oid -> oid.equals("df76e73f-0483-49a4-a9bb-63f2f985574a"))); assertValidId(p.getCollectedfrom().get(0).getKey()); - assertTrue(p.getAuthor().size() > 0); + assertFalse(p.getAuthor().isEmpty()); final Optional author = p .getAuthor() @@ -892,14 +892,14 @@ class MappersTest { assertEquals("Museum Sønderjylland", author.get().getFullname()); - assertTrue(p.getSubject().size() > 0); - assertTrue(p.getInstance().size() > 0); + assertFalse(p.getSubject().isEmpty()); + assertFalse(p.getInstance().isEmpty()); assertNotNull(p.getTitle()); assertFalse(p.getTitle().isEmpty()); assertNotNull(p.getInstance()); - assertTrue(p.getInstance().size() > 0); + assertFalse(p.getInstance().isEmpty()); p .getInstance() .forEach(i -> { @@ -939,10 +939,10 @@ class MappersTest { System.out.println("***************"); final Dataset p = (Dataset) list.get(0); - assertTrue(p.getInstance().size() > 0); + assertFalse(p.getInstance().isEmpty()); for (String url : p.getInstance().get(0).getUrl()) { System.out.println(url); - assertTrue(!url.contains("&")); + assertFalse(url.contains("&")); } } @@ -959,7 +959,7 @@ class MappersTest { assertTrue(o.isPresent()); Publication p = (Publication) o.get(); - assertTrue(p.getInstance().size() > 0); + assertFalse(p.getInstance().isEmpty()); assertEquals("https://doi.org/10.1155/2015/439379", p.getInstance().get(0).getUrl().get(0)); diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/synonyms.txt b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/synonyms.txt index b3fa94b20..a17a3949a 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/synonyms.txt +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/synonyms.txt @@ -1246,8 +1246,857 @@ dnet:relation_subRelType @=@ relationship @=@ publicationDataset dnet:provenanceActions @=@ iis @=@ erroneous label to be cleaned FOS @=@ 0101 mathematics @=@ FOS: Mathematics FOS @=@ 0102 computer and information sciences @=@ FOS: Computer and information sciences +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ddb1 @=@ Chemical Structures +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ddb1 @=@ Data Cube +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ddb1 @=@ Dataset/Dataset +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ddb1 @=@ Ensemble de données / Dataset +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ddb1 @=@ Research Data +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ddb1 @=@ dataset +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ddb1 @=@ http://purl.org/coar/resource_type/c_ddb1 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/ACF7-8YT9 @=@ collection +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/ACF7-8YT9 @=@ Collection of Datasets +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/ACF7-8YT9 @=@ RO-crate +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/ACF7-8YT9 @=@ Supplementary Collection of Datasets +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/ACF7-8YT9 @=@ Supplementary Dataset +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_12cc @=@ http://purl.org/coar/resource_type/c_12cc +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_12cc @=@ Planimetría +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_cb28 @=@ clinicalTrial +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_cb28 @=@ http://purl.org/coar/resource_type/c_cb28 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_12cd @=@ Carte géographique / Map +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_12cd @=@ http://purl.org/coar/resource_type/c_12cd +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_12cd @=@ map +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_12cd @=@ Mapa +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18cc @=@ Audio +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18cc @=@ http://purl.org/coar/resource_type/c_18cc +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18cc @=@ sound +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_86bc @=@ Book Prospectus +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_86bc @=@ Dictionary Entry +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_86bc @=@ Disclosure +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_86bc @=@ Literature review +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_86bc @=@ Reseña bibliográfica +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_86bc @=@ bibliography +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_86bc @=@ http://purl.org/coar/resource_type/c_86bc +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8a7e @=@ AUDIOVISUAL_DOCUMENT +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8a7e @=@ Audiovisual/Audiovisual +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8a7e @=@ http://purl.org/coar/resource_type/c_8a7e +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ecc8 @=@ Diagram +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ecc8 @=@ Drawing +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ecc8 @=@ Figure +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ecc8 @=@ Image/Image +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ecc8 @=@ Imagen - Image +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ecc8 @=@ Imagen 3-D +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ecc8 @=@ Imagen +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ecc8 @=@ Photo +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ecc8 @=@ Plot +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ecc8 @=@ StillImage|PRESERVED_SPECIMEN +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ecc8 @=@ fotó +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ecc8 @=@ grafika +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ecc8 @=@ http://purl.org/coar/resource_type/c_ecc8 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ecc8 @=@ image-diagram +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ecc8 @=@ image-drawing +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ecc8 @=@ image-figure +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ecc8 @=@ image-other +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ecc8 @=@ image-photo +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ecc8 @=@ image-plot +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ecc8 @=@ image +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_12ce @=@ film +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_12ce @=@ Film, vidéo / Motion picture, video +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_12ce @=@ http://purl.org/coar/resource_type/c_12ce +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_12ce @=@ Video +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_12ce @=@ vídeo +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/C53B-JCY5 @=@ H1 Myönnetty patentti +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/C53B-JCY5 @=@ http://purl.org/coar/resource_type/C53B-JCY5 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_15cd @=@ brevet +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_15cd @=@ http://purl.org/coar/resource_type/c_15cd +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_15cd @=@ Patent +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_15cd @=@ Patente +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_15cd @=@ Registered Copyright +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_15cd @=@ Traducción de patente +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/SB3Y-W4EH @=@ Solicitud de patente +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_e9a0 @=@ http://purl.org/coar/resource_type/c_e9a0 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_e9a0 @=@ interactiveResource +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_e059 @=@ http://purl.org/coar/resource_type/c_e059 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_e059 @=@ Learning Object +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_e059 @=@ learningObject +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_e059 @=@ PEDAGOGICAL_DOCUMENT +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ Autre / Other +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ EGI Virtual Appliance +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ Event/Event +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ Exhibition +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ Kita / Other +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ Model/Model +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ Otro - Other +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ Physical Object +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ Research Tool +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ Service +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ Trabajo de divulgación +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ UNKNOWN +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ University Academic Unit +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ application +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ artefact +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ carte +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ composition +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ corpus +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ event +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ http://purl.org/coar/resource_type/c_1843 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ http://purl.org/coar/resource_type/c_26e4 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ info:eu-repo/semantics/other +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ misc +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ model +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ other research product +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ other +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ otro +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ physicalObject +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ revue +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ sonstige Veröffentlichung +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ text/tg.edition+tg.aggregation+xml +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/9DKX-KSAF @=@ Modelo de utilidad +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7ad9 @=@ http://purl.org/coar/resource_type/c_7ad9 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7ad9 @=@ Online Resource +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7ad9 @=@ Sitio web +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7ad9 @=@ Web publication/site +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7877 @=@ Clinical Study +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7877 @=@ http://purl.org/coar/resource_type/c_7877 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ Livre / Book +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ Book (monograph) +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ Book (non peer-reviewed) +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ Book (peer-reviewed) +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ Book - monograph - editorial book +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ Book as author +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ Books +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ Buch +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ Edited Book +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ International Book/Monograph +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ Knyga / Book +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ Książka +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ Lehr- oder Fachbuch +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ Libro - Book +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ Libro +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ Monografia +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ Monograph +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ National Book/Monograph +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ atlas +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ book-series +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ book-set +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ book-track +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ book +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ book_series +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ book_title +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ doc-type:book +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ eBook +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ edited-book +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ http://purl.org/coar/resource_type/c_2f33 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ http://purl.org/eprint/type/Book +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ info:eu-repo/semantics/book +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ könyv +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ ouvrage +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ publication-book +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ reference-book +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ scientific book +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ Книга +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ Учебник +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ Монография +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ A3 Kirjan tai muun kokoomateoksen osa +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ Beitrag in einem Lehr- oder Fachbuch +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ Book Part (author) +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ Book Section / Chapter +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ Book Section +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ Book chapter or Essay in book +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ Book editorial +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ Book section +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ BookChapter +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ Book_Chapter +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ Buchbeitrag +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ Capítulo de Libro - Book Section +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ Capítulo de libro +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ Chapitre de livre / Book chapter +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ Contribution to International Book/Monograph ISI/JCR +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ Contribution to International Book/Monograph +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ Contribution to National Book/Monograph +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ Contribution to book (non peer-reviewed) +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ Contribution to book (peer-reviewed) +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ Knygos dalis / Book chapter +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ Part of book - chapter +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ Rozdział z książki +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ Scientific publication - Book Chapter +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ book chapter +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ book part +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ book-chapter +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ book-part +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ book-section +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ bookPart +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ book_content +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ chapitre_ouvrage +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ chapter +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ doc-type:bookPart +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ http://purl.org/coar/resource_type/c_3248 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ http://purl.org/eprint/type/BookItem +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ info:eu-repo/semantics/Chapter +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ info:eu-repo/semantics/bookPart +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ könyvfejezet +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ publication-section +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ reference-entry +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ reference_entry +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ scientific book chapter +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ Глава монографии +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ba08 @=@ Book Review +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ba08 @=@ book-review +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ba08 @=@ http://purl.org/coar/resource_type/c_ba08 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ba08 @=@ info:eu-repo/semantics/bookreview +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ba08 @=@ reseña de libro +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ A4 Artikkeli konferenssijulkaisussa +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ Article in monograph or in proceedings +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ Article in proceedings +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ Conference Paper +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ Conference article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ Conference papers +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ ConferencePaper +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ PROCEEDING_PAPER +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ Papers in Conference Proceedings openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ Proceedings paper -openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ Conference article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ Scientific publication - Conference Paper +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ Scientific publication - Conference Short Paper +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ conference_paper +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ http://purl.org/coar/resource_type/c_5794 openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ http://purl.org/eprint/type/ConferencePaper -openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ Conference article -openaire::user_resource_types @=@ Article @=@ http://purl.org/coar/resource_type/c_5794 \ No newline at end of file +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ http://purl.org/escidoc/metadata/ves/publication-types/conference-report +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ info:eu-repo/semantics/Conference Paper +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ info:eu-repo/semantics/conferencePaper +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ proceeding with peer review +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ proceedings-article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ publication-conferencepaper +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ Επιστημονική δημοσίευση - Ανακοίνωση Συνεδρίου (Short Paper) +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18cp @=@ Conference preprint +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18cp @=@ http://purl.org/coar/resource_type/c_18cp +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18co @=@ http://purl.org/coar/resource_type/c_18co +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7acd @=@ correction +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7acd @=@ corrigenda +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7acd @=@ http://purl.org/coar/resource_type/c_7acd +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7acd @=@ partial-retraction +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7acd @=@ reply +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7acd @=@ retraction +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ab20 @=@ Data Management Plan (NSF Generic) +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ab20 @=@ Data Management Plan +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ab20 @=@ OutputManagementPlan +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ab20 @=@ http://purl.org/coar/resource_type/c_ab20 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ab20 @=@ http://purl.org/spar/fabio/DataMangementPlan +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ab20 @=@ plan de gestión de datos +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ab20 @=@ publication-datamanagementplan +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_beb9 @=@ Data Descriptor +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_beb9 @=@ data-article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_beb9 @=@ DataPaper +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_beb9 @=@ http://purl.org/coar/resource_type/c_beb9 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/H9BQ-739P @=@ Peer review +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/H9BQ-739P @=@ peer-review +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_816b @=@ http://purl.org/coar/resource_type/c_816b +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_816b @=@ info:eu-repo/semantics/preprint +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_816b @=@ Pre Print +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_816b @=@ Pre-print +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_816b @=@ publication-preprint +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_816b @=@ Препринт +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_93fc @=@ ACTIVITY_REPORT +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_93fc @=@ Case Report +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_93fc @=@ Commissioned report +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_93fc @=@ Internal note +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_93fc @=@ Rapport / Report +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_93fc @=@ brief-report +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_93fc @=@ case-report +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_93fc @=@ chapitre_rapport +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_93fc @=@ compte rendu +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_93fc @=@ doc-type:report +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_93fc @=@ document_institutionnel +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_93fc @=@ http://purl.org/coar/resource_type/c_18hj +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_93fc @=@ http://purl.org/coar/resource_type/c_18wq +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_93fc @=@ http://purl.org/coar/resource_type/c_18ww +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_93fc @=@ http://purl.org/coar/resource_type/c_93fc +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_93fc @=@ http://purl.org/coar/resource_type/c_ba1f +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_93fc @=@ http://purl.org/eprint/type/Report +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_93fc @=@ info:eu-repo/semantics/report +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_93fc @=@ publication-report +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_93fc @=@ rapport_expertise +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_93fc @=@ rapport_mission +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_93fc @=@ report-paper +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_93fc @=@ report-paper_title +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_93fc @=@ report-series +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_93fc @=@ report +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_93fc @=@ support_cours +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ A1 Alkuperäisartikkeli tieteellisessä aikakauslehdessä +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ Article (author) +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ Article from Conference in a Journal +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ Article in journal +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ Article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ ArticleArtikel +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ Articles in Journals +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ Articolo +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ Artículo - Article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ Artículo +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ Articulo +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ Aufsatz +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ Journal article (on-line or printed) +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ Journal article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ Journal articles +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ Journal paper +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ JournalArticle +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ Makale +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ Original article (non peer-reviewed) +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ Original article (peer-reviewed) +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ Peer-reviewed Article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ Publication - Article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ Published Journal Article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ Research Article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ Scientific publication - Journal Article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ Straipsnis / Article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ Taxonomic Paper +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ Zeitschriftenbeitrag +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ art +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ article in non peer-reviewed journal +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ article in peer-reviewed journal +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ article_site_web +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ artykuł +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ doc-type:Journal Article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ doc-type:article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ foly +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ folyóiratcikk +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ http://purl.org/coar/resource_type/c_6501 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ http://purl.org/escidoc/metadata/ves/publication-types/article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ http://purl.org/ontology/bibo/AcademicArticle +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ info:eu-repo/semantics/Journal Article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ info:eu-repo/semantics/article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ journal-article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ journal_article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ non peer-reviewed article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ publication-article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ rapid-communication +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ research-article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ text (article) +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ Статья +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ Рецензована стаття +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_dcae04bc @=@ A2 Katsausartikkeli tieteellisessä aikakauslehdessä +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_dcae04bc @=@ Journal Article/Review +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_dcae04bc @=@ Review article (non peer-reviewed) +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_dcae04bc @=@ Review article (peer-reviewed) +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_dcae04bc @=@ article-commentary +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_dcae04bc @=@ artículo de revisión +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_dcae04bc @=@ Review Article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_dcae04bc @=@ review-article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_dcae04bc @=@ Revisión +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_dcae04bc @=@ RezensionReview +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_dcae04bc @=@ http://purl.org/coar/resource_type/c_dcae04bc +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7bab @=@ http://purl.org/coar/resource_type/c_7bab +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_71bd @=@ Documento tecnico +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_71bd @=@ http://purl.org/coar/resource_type/c_71bd +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_71bd @=@ publication-softwaredocumentation +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_71bd @=@ Software documentation +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/H6QP-SC1X @=@ Trademark +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_393c @=@ http://purl.org/coar/resource_type/c_393c +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_393c @=@ Workflow +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_393c @=@ Workflow/Workflow +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7a1f @=@ Bachelor's Degree +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7a1f @=@ Bachelor's +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7a1f @=@ Bachelors Thesis +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7a1f @=@ Graduate Thesis +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7a1f @=@ Proyecto fin de carrera +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7a1f @=@ Undergraduate Thesis +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7a1f @=@ Undergraduate diploma +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7a1f @=@ bachelor thesis +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7a1f @=@ http://purl.org/coar/resource_type/c_7a1f +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7a1f @=@ info:eu-repo/semantics/bachelorThesis +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7a1f @=@ выпускная бакалаврская работа +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ Daktaro disertacija / Doctoral dissertation +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ Diss +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ Dissertation +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ Doctoral Dissertation +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ Doctoral +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ DoctoralThesis +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ HabilitationThesis +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ PhD Theses +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ PhD thesis +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ Tenure-Promotion +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ Tesi di dottorato +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ Tesis +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ Text.Thesis.Doctoral +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ Theses +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ Thesis or Dissertation +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ Thesis.Doctoral +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ Thesis +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ doc-type:doctoralThesis +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ doctoral thesis +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ dok +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ doktori dolgozat +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ http://purl.org/coar/resource_type/c_db06 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ http://purl.org/eprint/type/Thesis +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ info:eu-repo/semantics/doctoralThesis +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ publication-thesis +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ tesis doctoral +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ these exercice +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ these +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_bdcc @=@ Diploma Project +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_bdcc @=@ MSc Thesis +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_bdcc @=@ Magistro darbas / Master thesis +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_bdcc @=@ Master Degree +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_bdcc @=@ Master's Degree +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_bdcc @=@ Master's +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_bdcc @=@ Masterarbeit u.a. +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_bdcc @=@ Masters (Taught) +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_bdcc @=@ Masters thesis +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_bdcc @=@ Masters-Thesis.Magister +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_bdcc @=@ Tesina +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_bdcc @=@ Thesis.Master +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_bdcc @=@ Trabajo fin de Máster +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_bdcc @=@ doc-type:masterThesis +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_bdcc @=@ hdr +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_bdcc @=@ http://purl.org/coar/resource_type/c_bdcc +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_bdcc @=@ info:eu-repo/semantics/masterThesis +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_bdcc @=@ master thesis +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_bdcc @=@ masterThesis +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_bdcc @=@ memoire +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_bdcc @=@ tesis de maestría +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18wz @=@ http://purl.org/coar/resource_type/c_18wz +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_186u @=@ http://purl.org/coar/resource_type/c_186u +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_186u @=@ http://purl.org/spar/fabio/DataManagementPolicy +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_186u @=@ http://purl.org/spar/fabio/DataManagementPolicyDocument +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_186u @=@ JRC Reference Reports +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18op @=@ Deliverable +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18op @=@ http://purl.org/coar/resource_type/c_18op +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18op @=@ Project deliverable +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18op @=@ Project Report +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18op @=@ publication-deliverable +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18ws @=@ D4 Julkaistu kehittämis- tai tutkimusraportti tai -selvitys +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18ws @=@ EUR - Scientific and Technical Research Reports +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18ws @=@ Project milestone +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18ws @=@ RESEARCH_REPORT +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18ws @=@ http://purl.org/coar/resource_type/c_18ws +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18ws @=@ research report +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18ws @=@ научный доклад +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18gh @=@ Departmental Technical Report +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18gh @=@ Informe Técnico +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18gh @=@ Tech-Report +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18gh @=@ Technical Report +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18gh @=@ document_technique +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18gh @=@ http://purl.org/coar/resource_type/c_18gh +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18gh @=@ informe a organismo financiador +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18gh @=@ publication-technicalnote +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_46ec @=@ Graduate diploma +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_46ec @=@ Hochschulschrift +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_46ec @=@ Tesis/trabajos de grado – Thesis +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_46ec @=@ Thèse ou mémoire / Thesis or Dissertation +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8042 @=@ Arbeitspapier +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8042 @=@ Departmental Bulletin Paper +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8042 @=@ Documento de trabajo +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8042 @=@ Paper +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8042 @=@ Project description +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8042 @=@ Research-Paper +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8042 @=@ ResearchPaper +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8042 @=@ Working / discussion paper +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8042 @=@ Working Paper / Technical Report +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8042 @=@ Working Paper +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8042 @=@ doc-type:workingPaper +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8042 @=@ http://purl.org/coar/resource_type/c_8042 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8042 @=@ http://purl.org/escidoc/metadata/ves/publication-types/paper +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8042 @=@ info:eu-repo/semantics/paper +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8042 @=@ info:eu-repo/semantics/workingPaper +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8042 @=@ publication-workingpaper +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8042 @=@ workingPaper +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_6947 @=@ Blog +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_6947 @=@ Entrada de blog +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_6947 @=@ http://purl.org/coar/resource_type/c_6947 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ Conference Abstract +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ Conference Paper/Proceeding/Abstract +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ Conference Program +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ Conference contribution +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ Conference or Workshop Item +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ Conference paper, poster, etc. +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ Conference paper/abstract +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ Conference report +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ Contribution à un congrès / Conference object +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ Contributions to Conferences +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ International Conference Abstract/Poster +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ International Conference ISI/JCR +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ International Conference communication/abstract/poster +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ Konferenzbeitrag +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ Resumen comunicación Congreso +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ Resúmen comunicación Congreso +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ communication_invitee +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ communication_sans_actes +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ conference item +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ conference object +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ conference +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ conferenceObject +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ doc-type:conferenceObject +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ http://purl.org/coar/resource_type/c_c94f +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ info:eu-repo/semantics/conferenceItem +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ info:eu-repo/semantics/conferenceObject +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ info:eu-repo/semantics/conferenceitemnotinproceedings +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_6670 @=@ Comunicación Congreso +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_6670 @=@ Comunicación de congreso +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_6670 @=@ Conference Poster +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_6670 @=@ National Conference Abstract/Poster +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_6670 @=@ National Conference communication/abstract/poster +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_6670 @=@ Póster de congreso +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_6670 @=@ Póster +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_6670 @=@ Scientific publication - Conference Poster +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_6670 @=@ http://purl.org/coar/resource_type/c_6670 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_6670 @=@ http://purl.org/eprint/type/ConferencePoster +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_6670 @=@ plakat +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_6670 @=@ poster +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/R60J-J5BD @=@ Conference lecture +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/R60J-J5BD @=@ Ponencia - Conference or Workshop Item +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/R60J-J5BD @=@ Presentación +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/R60J-J5BD @=@ Presentation +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/R60J-J5BD @=@ invited conference talk +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/R60J-J5BD @=@ ponencia +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/R60J-J5BD @=@ prezentacja +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_f744 @=@ Actes de congrès / Conference proceedings +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_f744 @=@ Conference Proceedings +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_f744 @=@ GL4 Conference Proceedings +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_f744 @=@ International Conference +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_f744 @=@ National Conference +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_f744 @=@ PREFACE_PROCEEDINGS +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_f744 @=@ Proceedings (peer-reviewed) +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_f744 @=@ Proceedings of a Conference +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_f744 @=@ Tagungsband +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_f744 @=@ actas de congreso +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_f744 @=@ actes_congres +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_f744 @=@ communication_avec_actes +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_f744 @=@ communication_par_affiche +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_f744 @=@ conference proceeding +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_f744 @=@ http://purl.org/coar/resource_type/c_f744 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_f744 @=@ http://purl.org/escidoc/metadata/ves/publication-types/proceedings +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_f744 @=@ info:eu-repo/semantics/conferenceproceedings +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_f744 @=@ proceeding, seminar, workshop without peer review +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_f744 @=@ proceedings +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_b239 @=@ Editorial +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_b239 @=@ Editorial ISI/JCR +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_b239 @=@ Editors +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_b239 @=@ Editors (non peer-reviewed) +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_b239 @=@ Editors (peer-reviewed) +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_b239 @=@ http://purl.org/coar/resource_type/c_b239 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_0640 @=@ http://purl.org/coar/resource_type/c_0640 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_0640 @=@ International Journal +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_0640 @=@ International Journal ISI/JCR +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_0640 @=@ Journal (full / special issue) +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_0640 @=@ National Journal +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_0857 @=@ habilitation à diriger des recherches +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_0857 @=@ http://purl.org/coar/resource_type/c_0857 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_545b @=@ Article - letter to the editor +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_545b @=@ Article / Letter to editor +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_545b @=@ Article / Letter to the editor +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_545b @=@ Article-letter to the editor +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_545b @=@ Article/Letter to editor +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_545b @=@ http://purl.org/coar/resource_type/c_545b +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_545b @=@ letter +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2cd9 @=@ Magazine Article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2cd9 @=@ Revista +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2cd9 @=@ Sammelband +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2cd9 @=@ revista divulgativa +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_0040 @=@ foreword +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_0040 @=@ Abstract +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_0040 @=@ Beitrag im Sammelband +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_0040 @=@ Manuscript +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_0040 @=@ Manuscrito +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_0040 @=@ afterword +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_0040 @=@ avantpropos +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_0040 @=@ http://purl.org/coar/resource_type/c_0040 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_0040 @=@ http://purl.org/coar/resource_type/c_3e5a +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_0040 @=@ postface +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2fe3 @=@ revuedepresse +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_998f @=@ Article / Newspaper +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_998f @=@ Newspaper Article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_998f @=@ Newspaper or magazine article +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_998f @=@ http://purl.org/coar/resource_type/c_998f +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_998f @=@ in-brief +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/QX5C-AR31 @=@ Berichtsreihe +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/QX5C-AR31 @=@ Encyclopedia Entry +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/QX5C-AR31 @=@ Funding Submission +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/QX5C-AR31 @=@ Índice +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/QX5C-AR31 @=@ License +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/QX5C-AR31 @=@ Manual +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/QX5C-AR31 @=@ Other publication (non peer-review) +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/QX5C-AR31 @=@ Other publication (peer-review) +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/QX5C-AR31 @=@ Supervised Student Publication +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/QX5C-AR31 @=@ Text/Text +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/QX5C-AR31 @=@ Text +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/QX5C-AR31 @=@ Translation +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/QX5C-AR31 @=@ Travail étudiant / Student work +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/QX5C-AR31 @=@ chronique +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/QX5C-AR31 @=@ contributionToPeriodical +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/QX5C-AR31 @=@ historicalDocument +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/QX5C-AR31 @=@ info:eu-repo/semantics/contributionToPeriodical +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/QX5C-AR31 @=@ literature +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/QX5C-AR31 @=@ other publication +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/QX5C-AR31 @=@ publication-other +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/QX5C-AR31 @=@ sa_component +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/QX5C-AR31 @=@ standard-series +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/QX5C-AR31 @=@ standard +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_baaf @=@ Proposal +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_baaf @=@ http://purl.org/coar/resource_type/c_baaf +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_baaf @=@ research-proposal +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_efa0 @=@ Book/Film/Article review +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_efa0 @=@ http://purl.org/coar/resource_type/c_efa0 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_efa0 @=@ info:eu-repo/semantics/review +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18cf @=@ http://purl.org/coar/resource_type/c_18cf +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1162 @=@ Comentario +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1162 @=@ Comment/debate +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1162 @=@ annotation +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1162 @=@ http://purl.org/coar/resource_type/c_1162 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1162 @=@ info:eu-repo/semantics/annotation +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8544 @=@ Inaugural lecture +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8544 @=@ Material didáctico +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8544 @=@ Matériel didactique / Educational material +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8544 @=@ Public-Lecture +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8544 @=@ Teaching Resource +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8544 @=@ http://purl.org/coar/resource_type/c_8544 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8544 @=@ info:eu-repo/semantics/lecture +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8544 @=@ lesson +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8544 @=@ note de lecture +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8544 @=@ notedelecture +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8544 @=@ Учебный материал +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18cd @=@ Partitura +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18cd @=@ document_audiovisuel +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18cd @=@ http://purl.org/coar/resource_type/c_18cd +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18cw @=@ Estudio y edición crítica de música +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18cw @=@ http://purl.org/coar/resource_type/c_18cw +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c950 @=@ Software/Software +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c950 @=@ Software +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c950 @=@ http://purl.org/coar/resource_type/c_c950 +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5ce6 @=@ Jupyter Notebook +openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5ce6 @=@ http://purl.org/coar/resource_type/c_5ce6 +openaire::user_resource_types @=@ Article @=@ text:conference_output:conference_proceedings:conference_paper +openaire::user_resource_types @=@ Article @=@ http://purl.org/coar/resource_type/c_5794 +openaire::user_resource_types @=@ Article @=@ text:conference_output:conference_paper_not_in_proceedings +openaire::user_resource_types @=@ Article @=@ http://purl.org/coar/resource_type/c_18cp +openaire::user_resource_types @=@ Article @=@ text:conference_output:conference_poster_not_in_proceedings +openaire::user_resource_types @=@ Article @=@ http://purl.org/coar/resource_type/c_18co +openaire::user_resource_types @=@ Article @=@ text:journal:journal_article:corrigendum +openaire::user_resource_types @=@ Article @=@ http://purl.org/coar/resource_type/c_7acd +openaire::user_resource_types @=@ Article @=@ text:journal:journal_article:data_paper +openaire::user_resource_types @=@ Article @=@ http://purl.org/coar/resource_type/c_beb9 +openaire::user_resource_types @=@ Article @=@ text:journal:journal_article +openaire::user_resource_types @=@ Article @=@ http://purl.org/coar/resource_type/c_6501 +openaire::user_resource_types @=@ Article @=@ text:preprint +openaire::user_resource_types @=@ Article @=@ http://purl.org/coar/resource_type/c_816b +openaire::user_resource_types @=@ Article @=@ text:journal:journal_article:research_article +openaire::user_resource_types @=@ Article @=@ http://purl.org/coar/resource_type/c_2df8fbb1 +openaire::user_resource_types @=@ Article @=@ text:journal:journal_article:review_article +openaire::user_resource_types @=@ Article @=@ http://purl.org/coar/resource_type/c_dcae04bc +openaire::user_resource_types @=@ Article @=@ text:journal:journal_article:software_paper +openaire::user_resource_types @=@ Article @=@ http://purl.org/coar/resource_type/c_7bab +openaire::user_resource_types @=@ Article @=@ text:working_paper +openaire::user_resource_types @=@ Article @=@ http://purl.org/coar/resource_type/c_8042 +openaire::user_resource_types @=@ Thesis @=@ text:thesis:bachelor_thesis +openaire::user_resource_types @=@ Thesis @=@ http://purl.org/coar/resource_type/c_7a1f +openaire::user_resource_types @=@ Thesis @=@ text:thesis:doctoral_thesis +openaire::user_resource_types @=@ Thesis @=@ http://purl.org/coar/resource_type/c_db06 +openaire::user_resource_types @=@ Thesis @=@ text:thesis:master_thesis +openaire::user_resource_types @=@ Thesis @=@ http://purl.org/coar/resource_type/c_bdcc +openaire::user_resource_types @=@ Thesis @=@ text:thesis +openaire::user_resource_types @=@ Thesis @=@ http://purl.org/coar/resource_type/c_46ec +openaire::user_resource_types @=@ Report @=@ text:report +openaire::user_resource_types @=@ Report @=@ http://purl.org/coar/resource_type/c_93fc +openaire::user_resource_types @=@ Report @=@ text:report:memorandum +openaire::user_resource_types @=@ Report @=@ http://purl.org/coar/resource_type/c_18wz +openaire::user_resource_types @=@ Report @=@ text:report:policy_report +openaire::user_resource_types @=@ Report @=@ http://purl.org/coar/resource_type/c_186u +openaire::user_resource_types @=@ Report @=@ text:report:research_protocol +openaire::user_resource_types @=@ Report @=@ http://purl.org/coar/resource_type/YZ1N-ZFT9 +openaire::user_resource_types @=@ Report @=@ text:report:research_report +openaire::user_resource_types @=@ Report @=@ http://purl.org/coar/resource_type/c_18ws +openaire::user_resource_types @=@ Report @=@ text:report:technical_report +openaire::user_resource_types @=@ Report @=@ http://purl.org/coar/resource_type/c_18gh +openaire::user_resource_types @=@ Project result @=@ text:report:data_management_plan +openaire::user_resource_types @=@ Project result @=@ http://purl.org/coar/resource_type/c_ab20 +openaire::user_resource_types @=@ Project result @=@ text:report:project_deliverable +openaire::user_resource_types @=@ Project result @=@ http://purl.org/coar/resource_type/c_18op +openaire::user_resource_types @=@ Book/Chapter @=@ text:book +openaire::user_resource_types @=@ Book/Chapter @=@ http://purl.org/coar/resource_type/c_2f33 +openaire::user_resource_types @=@ Book/Chapter @=@ text:book:book_part +openaire::user_resource_types @=@ Book/Chapter @=@ http://purl.org/coar/resource_type/c_3248 +openaire::user_resource_types @=@ Clinical Study @=@ text:report:clinical_study +openaire::user_resource_types @=@ Clinical Study @=@ http://purl.org/coar/resource_type/c_7877 +openaire::meta_resource_types @=@ Research Literature @=@ text:bibliography +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_86bc +openaire::meta_resource_types @=@ Research Literature @=@ patent:design_patent +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/C53B-JCY5 +openaire::meta_resource_types @=@ Research Literature @=@ patent +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_15cd +openaire::meta_resource_types @=@ Research Literature @=@ patent:PCT_application +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/SB3Y-W4EH +openaire::meta_resource_types @=@ Research Literature @=@ patent:plant_patent +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/Z907-YMBB +openaire::meta_resource_types @=@ Research Literature @=@ patent:plant_variety_protection +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/GPQ7-G5VE +openaire::meta_resource_types @=@ Research Literature @=@ patent:software_patent +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/MW8G-3CR8 +openaire::meta_resource_types @=@ Research Literature @=@ patent:utility_model +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/9DKX-KSAF +openaire::meta_resource_types @=@ Research Literature @=@ text:report:clinical_study +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_7877 +openaire::meta_resource_types @=@ Research Literature @=@ text:book +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_2f33 +openaire::meta_resource_types @=@ Research Literature @=@ text:book:book_part +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_3248 +openaire::meta_resource_types @=@ Research Literature @=@ text:review:book_review +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_ba08 +openaire::meta_resource_types @=@ Research Literature @=@ text:review:commentary +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/D97F-VB57 +openaire::meta_resource_types @=@ Research Literature @=@ text:conference_output:conference_proceedings:conference_paper +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_5794 +openaire::meta_resource_types @=@ Research Literature @=@ text:conference_output:conference_paper_not_in_proceedings +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_18cp +openaire::meta_resource_types @=@ Research Literature @=@ text:conference_output:conference_poster_not_in_proceedings +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_18co +openaire::meta_resource_types @=@ Research Literature @=@ text:journal:journal_article:corrigendum +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_7acd +openaire::meta_resource_types @=@ Research Literature @=@ text:report:data_management_plan +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_ab20 +openaire::meta_resource_types @=@ Research Literature @=@ text:journal:journal_article:data_paper +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_beb9 +openaire::meta_resource_types @=@ Research Literature @=@ text:journal:journal_article +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_6501 +openaire::meta_resource_types @=@ Research Literature @=@ text:review:peer_review +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/H9BQ-739P +openaire::meta_resource_types @=@ Research Literature @=@ text:preprint +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_816b +openaire::meta_resource_types @=@ Research Literature @=@ text:report +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_93fc +openaire::meta_resource_types @=@ Research Literature @=@ text:journal:journal_article:research_article +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_2df8fbb1 +openaire::meta_resource_types @=@ Research Literature @=@ text:journal:journal_article:review_article +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_dcae04bc +openaire::meta_resource_types @=@ Research Literature @=@ text:journal:journal_article:software_paper +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_7bab +openaire::meta_resource_types @=@ Research Literature @=@ text:technical_documentation +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_71bd +openaire::meta_resource_types @=@ Research Literature @=@ text:transcription +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/6NC7-GK9S +openaire::meta_resource_types @=@ Research Literature @=@ text:thesis:bachelor_thesis +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_7a1f +openaire::meta_resource_types @=@ Research Literature @=@ text:thesis:doctoral_thesis +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_db06 +openaire::meta_resource_types @=@ Research Literature @=@ text:thesis:master_thesis +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_bdcc +openaire::meta_resource_types @=@ Research Literature @=@ text:report:memorandum +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_18wz +openaire::meta_resource_types @=@ Research Literature @=@ text:report:policy_report +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_186u +openaire::meta_resource_types @=@ Research Literature @=@ text:report:project_deliverable +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_18op +openaire::meta_resource_types @=@ Research Literature @=@ text:report:research_protocol +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/YZ1N-ZFT9 +openaire::meta_resource_types @=@ Research Literature @=@ text:report:research_report +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_18ws +openaire::meta_resource_types @=@ Research Literature @=@ text:report:technical_report +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_18gh +openaire::meta_resource_types @=@ Research Literature @=@ text:thesis +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_46ec +openaire::meta_resource_types @=@ Research Literature @=@ text:working_paper +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_8042 +openaire::meta_resource_types @=@ Research Literature @=@ text:blog_post +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_6947 +openaire::meta_resource_types @=@ Research Literature @=@ text:conference_output +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_c94f +openaire::meta_resource_types @=@ Research Literature @=@ text:conference_output:conference_proceedings:conference_poster +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_6670 +openaire::meta_resource_types @=@ Research Literature @=@ text:conference_output:conference_presentation +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/R60J-J5BD +openaire::meta_resource_types @=@ Research Literature @=@ text:conference_output:conference_proceedings +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_f744 +openaire::meta_resource_types @=@ Research Literature @=@ text:journal:editorial +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_b239 +openaire::meta_resource_types @=@ Research Literature @=@ text:journal +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_0640 +openaire::meta_resource_types @=@ Research Literature @=@ text:letter +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_0857 +openaire::meta_resource_types @=@ Research Literature @=@ text:journal:letter_to_the_editor +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_545b +openaire::meta_resource_types @=@ Research Literature @=@ text:magazine +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_2cd9 +openaire::meta_resource_types @=@ Research Literature @=@ text:manuscript +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_0040 +openaire::meta_resource_types @=@ Research Literature @=@ text:newspaper +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_2fe3 +openaire::meta_resource_types @=@ Research Literature @=@ text:newspaper:newspaper_article +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_998f +openaire::meta_resource_types @=@ Research Literature @=@ text:other_periodical +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/QX5C-AR31 +openaire::meta_resource_types @=@ Research Literature @=@ text:research_proposal +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_baaf +openaire::meta_resource_types @=@ Research Literature @=@ text:review +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_efa0 +openaire::meta_resource_types @=@ Research Literature @=@ text +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_18cf +openaire::meta_resource_types @=@ Research Literature @=@ text:annotation +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_1162 +openaire::meta_resource_types @=@ Research Literature @=@ text:lecture +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_8544 +openaire::meta_resource_types @=@ Research Literature @=@ text:musical_notation +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_18cw +openaire::meta_resource_types @=@ Research Data @=@ dataset +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/c_ddb1 +openaire::meta_resource_types @=@ Research Data @=@ dataset:aggregated_data +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/ACF7-8YT9 +openaire::meta_resource_types @=@ Research Data @=@ cartographic_material +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/c_12cc +openaire::meta_resource_types @=@ Research Data @=@ dataset:clinical_trial_data +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/c_cb28 +openaire::meta_resource_types @=@ Research Data @=@ dataset:compiled_data +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/FXF3-D3G7 +openaire::meta_resource_types @=@ Research Data @=@ dataset:encoded_data +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/AM6W-6QAW +openaire::meta_resource_types @=@ Research Data @=@ dataset:experimental_data +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/63NG-B465 +openaire::meta_resource_types @=@ Research Data @=@ dataset:genomic_data +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/A8F1-NPV9 +openaire::meta_resource_types @=@ Research Data @=@ dataset:geospatial_data +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/2H0M-X761 +openaire::meta_resource_types @=@ Research Data @=@ dataset:laboratory_notebook +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/H41Y-FW7B +openaire::meta_resource_types @=@ Research Data @=@ cartographic_material:map +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/c_12cd +openaire::meta_resource_types @=@ Research Data @=@ dataset:measurement_and_test_data +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/DD58-GFSX +openaire::meta_resource_types @=@ Research Data @=@ dataset:observational_data +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/FF4C-28RK +openaire::meta_resource_types @=@ Research Data @=@ dataset:recorded_data +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/CQMR-7K63 +openaire::meta_resource_types @=@ Research Data @=@ dataset:simulation_data +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/W2XT-7017 +openaire::meta_resource_types @=@ Research Data @=@ sound +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/c_18cc +openaire::meta_resource_types @=@ Research Data @=@ dataset:survey_data +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/NHD0-W6SY +openaire::meta_resource_types @=@ Research Data @=@ image +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/c_c513 +openaire::meta_resource_types @=@ Research Data @=@ image:moving_image +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/c_8a7e +openaire::meta_resource_types @=@ Research Data @=@ image:still_image +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/c_ecc8 +openaire::meta_resource_types @=@ Research Data @=@ image:moving_image:video +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/c_12ce +openaire::meta_resource_types @=@ Research Data @=@ design +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/542X-3S04 +openaire::meta_resource_types @=@ Research Data @=@ design:industrial_design +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/JBNF-DYAD +openaire::meta_resource_types @=@ Research Data @=@ interactive_resource +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/c_e9a0 +openaire::meta_resource_types @=@ Research Data @=@ design:layout_design +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/BW7T-YM2G +openaire::meta_resource_types @=@ Research Data @=@ interactive_resource:website +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/c_7ad9 +openaire::meta_resource_types @=@ Research Data @=@ sound:musical_composition +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/c_18cd +openaire::meta_resource_types @=@ Research Software @=@ software:research_software +openaire::meta_resource_types @=@ Research Software @=@ http://purl.org/coar/resource_type/c_c950 +openaire::meta_resource_types @=@ Research Software @=@ software +openaire::meta_resource_types @=@ Research Software @=@ http://purl.org/coar/resource_type/c_5ce6 +openaire::meta_resource_types @=@ Research Software @=@ software:source_code +openaire::meta_resource_types @=@ Research Software @=@ http://purl.org/coar/resource_type/QH80-2R4E +openaire::meta_resource_types @=@ Other Research Products @=@ learning_object +openaire::meta_resource_types @=@ Other Research Products @=@ http://purl.org/coar/resource_type/c_e059 +openaire::meta_resource_types @=@ Other Research Products @=@ other +openaire::meta_resource_types @=@ Other Research Products @=@ http://purl.org/coar/resource_type/c_1843 +openaire::meta_resource_types @=@ Other Research Products @=@ trademark +openaire::meta_resource_types @=@ Other Research Products @=@ http://purl.org/coar/resource_type/H6QP-SC1X +openaire::meta_resource_types @=@ Other Research Products @=@ workflow +openaire::meta_resource_types @=@ Other Research Products @=@ http://purl.org/coar/resource_type/c_393c \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/terms.txt b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/terms.txt index a38c0e987..68828b3d8 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/terms.txt +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/terms.txt @@ -1122,5 +1122,295 @@ dnet:relation_subRelType @=@ dnet:relation_subRelType @=@ version @=@ version FOS @=@ Fields of Science and Technology classification @=@ 0101 mathematics @=@ 0101 mathematics FOS @=@ Fields of Science and Technology classification @=@ 0102 computer and information sciences @=@ 0102 computer and information sciences FOS @=@ Fields of Science and Technology classification @=@ 0103 physical sciences @=@ 0103 physical sciences +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ddb1 @=@ dataset +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/ACF7-8YT9 @=@ aggregated data +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_12cc @=@ cartographic material +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_cb28 @=@ clinical trial data +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/FXF3-D3G7 @=@ compiled data +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/AM6W-6QAW @=@ encoded data +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/63NG-B465 @=@ experimental data +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/A8F1-NPV9 @=@ genomic data +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/2H0M-X761 @=@ geospatial data +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/H41Y-FW7B @=@ laboratory notebook +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_12cd @=@ map +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/DD58-GFSX @=@ measurement and test data +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/FF4C-28RK @=@ observational data +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/CQMR-7K63 @=@ recorded data +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/W2XT-7017 @=@ simulation data +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18cc @=@ sound +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/NHD0-W6SY @=@ survey data +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_86bc @=@ bibliography +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c513 @=@ image +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8a7e @=@ moving image +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ecc8 @=@ still image +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_12ce @=@ video +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/C53B-JCY5 @=@ design patent +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_15cd @=@ patent +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/SB3Y-W4EH @=@ PCT application +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/Z907-YMBB @=@ plant patent +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/GPQ7-G5VE @=@ plant variety protection +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/MW8G-3CR8 @=@ software patent +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/542X-3S04 @=@ design +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/JBNF-DYAD @=@ industrial design +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_e9a0 @=@ interactive resource +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/BW7T-YM2G @=@ layout design +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_e059 @=@ learning object +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1843 @=@ other +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/9DKX-KSAF @=@ utility model +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7ad9 @=@ website +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7877 @=@ clinical study +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2f33 @=@ book +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_3248 @=@ book part +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ba08 @=@ book review +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/D97F-VB57 @=@ commentary openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5794 @=@ conference paper -openaire::user_resource_types @=@ openaire::user_resource_types @=@ Article @=@ Article \ No newline at end of file +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18cp @=@ conference paper not in proceedings +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18co @=@ conference poster not in proceedings +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7acd @=@ corrigendum +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_ab20 @=@ data management plan +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_beb9 @=@ data paper +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_6501 @=@ journal article +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/H9BQ-739P @=@ peer review +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_816b @=@ preprint +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_93fc @=@ report +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2df8fbb1 @=@ research article +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_dcae04bc @=@ review article +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7bab @=@ software paper +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_71bd @=@ technical documentation +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/6NC7-GK9S @=@ transcription +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/H6QP-SC1X @=@ trademark +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_393c @=@ workflow +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_7a1f @=@ bachelor thesis +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_db06 @=@ doctoral thesis +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_bdcc @=@ master thesis +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18wz @=@ memorandum +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_186u @=@ policy report +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18op @=@ project deliverable +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/YZ1N-ZFT9 @=@ research protocol +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18ws @=@ research report +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18gh @=@ technical report +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_46ec @=@ thesis +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8042 @=@ working paper +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_6947 @=@ blog post +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c94f @=@ conference output +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_6670 @=@ conference poster +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/R60J-J5BD @=@ conference presentation +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_f744 @=@ conference proceedings +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_b239 @=@ editorial +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_0640 @=@ journal +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_0857 @=@ letter +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_545b @=@ letter to the editor +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2cd9 @=@ magazine +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_0040 @=@ manuscript +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_2fe3 @=@ newspaper +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_998f @=@ newspaper article +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/QX5C-AR31 @=@ other periodical +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_baaf @=@ research proposal +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_efa0 @=@ review +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18cf @=@ text +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_1162 @=@ annotation +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_8544 @=@ lecture +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18cd @=@ musical composition +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_18cw @=@ musical notation +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_c950 @=@ research software +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/c_5ce6 @=@ software +openaire::coar_resource_types_3_1 @=@ openaire::coar_resource_types_3_1 @=@ http://purl.org/coar/resource_type/QH80-2R4E @=@ source code +openaire::user_resource_types @=@ openaire::user_resource_types @=@ Article @=@ Article +openaire::user_resource_types @=@ openaire::user_resource_types @=@ Thesis @=@ Thesis +openaire::user_resource_types @=@ openaire::user_resource_types @=@ Report @=@ Report +openaire::user_resource_types @=@ openaire::user_resource_types @=@ Project result @=@ Project result +openaire::user_resource_types @=@ openaire::user_resource_types @=@ Research Data @=@ Research Data +openaire::user_resource_types @=@ openaire::user_resource_types @=@ Presentation @=@ Presentation +openaire::user_resource_types @=@ openaire::user_resource_types @=@ Book/Chapter @=@ Book/Chapter +openaire::user_resource_types @=@ openaire::user_resource_types @=@ Multimedia @=@ Multimedia +openaire::user_resource_types @=@ openaire::user_resource_types @=@ Clinical Study @=@ Clinical Study +openaire::meta_resource_types @=@ openaire::meta_resource_types @=@ Research Literature @=@ Research Literature +openaire::meta_resource_types @=@ openaire::meta_resource_types @=@ Research Data @=@ Research Data +openaire::meta_resource_types @=@ openaire::meta_resource_types @=@ Research Software @=@ Research Software +openaire::meta_resource_types @=@ openaire::meta_resource_types @=@ Other Research Products @=@ Other Research Products +openaire::meta_resource_types @=@ Research Literature @=@ text:bibliography +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_86bc +openaire::meta_resource_types @=@ Research Literature @=@ patent:design_patent +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/C53B-JCY5 +openaire::meta_resource_types @=@ Research Literature @=@ patent +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_15cd +openaire::meta_resource_types @=@ Research Literature @=@ patent:PCT_application +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/SB3Y-W4EH +openaire::meta_resource_types @=@ Research Literature @=@ patent:plant_patent +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/Z907-YMBB +openaire::meta_resource_types @=@ Research Literature @=@ patent:plant_variety_protection +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/GPQ7-G5VE +openaire::meta_resource_types @=@ Research Literature @=@ patent:software_patent +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/MW8G-3CR8 +openaire::meta_resource_types @=@ Research Literature @=@ patent:utility_model +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/9DKX-KSAF +openaire::meta_resource_types @=@ Research Literature @=@ text:report:clinical_study +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_7877 +openaire::meta_resource_types @=@ Research Literature @=@ text:book +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_2f33 +openaire::meta_resource_types @=@ Research Literature @=@ text:book:book_part +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_3248 +openaire::meta_resource_types @=@ Research Literature @=@ text:review:book_review +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_ba08 +openaire::meta_resource_types @=@ Research Literature @=@ text:review:commentary +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/D97F-VB57 +openaire::meta_resource_types @=@ Research Literature @=@ text:conference_output:conference_proceedings:conference_paper +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_5794 +openaire::meta_resource_types @=@ Research Literature @=@ text:conference_output:conference_paper_not_in_proceedings +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_18cp +openaire::meta_resource_types @=@ Research Literature @=@ text:conference_output:conference_poster_not_in_proceedings +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_18co +openaire::meta_resource_types @=@ Research Literature @=@ text:journal:journal_article:corrigendum +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_7acd +openaire::meta_resource_types @=@ Research Literature @=@ text:report:data_management_plan +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_ab20 +openaire::meta_resource_types @=@ Research Literature @=@ text:journal:journal_article:data_paper +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_beb9 +openaire::meta_resource_types @=@ Research Literature @=@ text:journal:journal_article +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_6501 +openaire::meta_resource_types @=@ Research Literature @=@ text:review:peer_review +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/H9BQ-739P +openaire::meta_resource_types @=@ Research Literature @=@ text:preprint +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_816b +openaire::meta_resource_types @=@ Research Literature @=@ text:report +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_93fc +openaire::meta_resource_types @=@ Research Literature @=@ text:journal:journal_article:research_article +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_2df8fbb1 +openaire::meta_resource_types @=@ Research Literature @=@ text:journal:journal_article:review_article +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_dcae04bc +openaire::meta_resource_types @=@ Research Literature @=@ text:journal:journal_article:software_paper +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_7bab +openaire::meta_resource_types @=@ Research Literature @=@ text:technical_documentation +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_71bd +openaire::meta_resource_types @=@ Research Literature @=@ text:transcription +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/6NC7-GK9S +openaire::meta_resource_types @=@ Research Literature @=@ text:thesis:bachelor_thesis +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_7a1f +openaire::meta_resource_types @=@ Research Literature @=@ text:thesis:doctoral_thesis +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_db06 +openaire::meta_resource_types @=@ Research Literature @=@ text:thesis:master_thesis +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_bdcc +openaire::meta_resource_types @=@ Research Literature @=@ text:report:memorandum +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_18wz +openaire::meta_resource_types @=@ Research Literature @=@ text:report:policy_report +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_186u +openaire::meta_resource_types @=@ Research Literature @=@ text:report:project_deliverable +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_18op +openaire::meta_resource_types @=@ Research Literature @=@ text:report:research_protocol +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/YZ1N-ZFT9 +openaire::meta_resource_types @=@ Research Literature @=@ text:report:research_report +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_18ws +openaire::meta_resource_types @=@ Research Literature @=@ text:report:technical_report +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_18gh +openaire::meta_resource_types @=@ Research Literature @=@ text:thesis +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_46ec +openaire::meta_resource_types @=@ Research Literature @=@ text:working_paper +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_8042 +openaire::meta_resource_types @=@ Research Literature @=@ text:blog_post +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_6947 +openaire::meta_resource_types @=@ Research Literature @=@ text:conference_output +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_c94f +openaire::meta_resource_types @=@ Research Literature @=@ text:conference_output:conference_proceedings:conference_poster +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_6670 +openaire::meta_resource_types @=@ Research Literature @=@ text:conference_output:conference_presentation +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/R60J-J5BD +openaire::meta_resource_types @=@ Research Literature @=@ text:conference_output:conference_proceedings +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_f744 +openaire::meta_resource_types @=@ Research Literature @=@ text:journal:editorial +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_b239 +openaire::meta_resource_types @=@ Research Literature @=@ text:journal +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_0640 +openaire::meta_resource_types @=@ Research Literature @=@ text:letter +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_0857 +openaire::meta_resource_types @=@ Research Literature @=@ text:journal:letter_to_the_editor +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_545b +openaire::meta_resource_types @=@ Research Literature @=@ text:magazine +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_2cd9 +openaire::meta_resource_types @=@ Research Literature @=@ text:manuscript +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_0040 +openaire::meta_resource_types @=@ Research Literature @=@ text:newspaper +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_2fe3 +openaire::meta_resource_types @=@ Research Literature @=@ text:newspaper:newspaper_article +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_998f +openaire::meta_resource_types @=@ Research Literature @=@ text:other_periodical +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/QX5C-AR31 +openaire::meta_resource_types @=@ Research Literature @=@ text:research_proposal +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_baaf +openaire::meta_resource_types @=@ Research Literature @=@ text:review +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_efa0 +openaire::meta_resource_types @=@ Research Literature @=@ text +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_18cf +openaire::meta_resource_types @=@ Research Literature @=@ text:annotation +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_1162 +openaire::meta_resource_types @=@ Research Literature @=@ text:lecture +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_8544 +openaire::meta_resource_types @=@ Research Literature @=@ text:musical_notation +openaire::meta_resource_types @=@ Research Literature @=@ http://purl.org/coar/resource_type/c_18cw +openaire::meta_resource_types @=@ Research Data @=@ dataset +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/c_ddb1 +openaire::meta_resource_types @=@ Research Data @=@ dataset:aggregated_data +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/ACF7-8YT9 +openaire::meta_resource_types @=@ Research Data @=@ cartographic_material +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/c_12cc +openaire::meta_resource_types @=@ Research Data @=@ dataset:clinical_trial_data +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/c_cb28 +openaire::meta_resource_types @=@ Research Data @=@ dataset:compiled_data +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/FXF3-D3G7 +openaire::meta_resource_types @=@ Research Data @=@ dataset:encoded_data +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/AM6W-6QAW +openaire::meta_resource_types @=@ Research Data @=@ dataset:experimental_data +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/63NG-B465 +openaire::meta_resource_types @=@ Research Data @=@ dataset:genomic_data +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/A8F1-NPV9 +openaire::meta_resource_types @=@ Research Data @=@ dataset:geospatial_data +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/2H0M-X761 +openaire::meta_resource_types @=@ Research Data @=@ dataset:laboratory_notebook +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/H41Y-FW7B +openaire::meta_resource_types @=@ Research Data @=@ cartographic_material:map +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/c_12cd +openaire::meta_resource_types @=@ Research Data @=@ dataset:measurement_and_test_data +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/DD58-GFSX +openaire::meta_resource_types @=@ Research Data @=@ dataset:observational_data +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/FF4C-28RK +openaire::meta_resource_types @=@ Research Data @=@ dataset:recorded_data +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/CQMR-7K63 +openaire::meta_resource_types @=@ Research Data @=@ dataset:simulation_data +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/W2XT-7017 +openaire::meta_resource_types @=@ Research Data @=@ sound +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/c_18cc +openaire::meta_resource_types @=@ Research Data @=@ dataset:survey_data +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/NHD0-W6SY +openaire::meta_resource_types @=@ Research Data @=@ image +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/c_c513 +openaire::meta_resource_types @=@ Research Data @=@ image:moving_image +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/c_8a7e +openaire::meta_resource_types @=@ Research Data @=@ image:still_image +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/c_ecc8 +openaire::meta_resource_types @=@ Research Data @=@ image:moving_image:video +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/c_12ce +openaire::meta_resource_types @=@ Research Data @=@ design +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/542X-3S04 +openaire::meta_resource_types @=@ Research Data @=@ design:industrial_design +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/JBNF-DYAD +openaire::meta_resource_types @=@ Research Data @=@ interactive_resource +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/c_e9a0 +openaire::meta_resource_types @=@ Research Data @=@ design:layout_design +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/BW7T-YM2G +openaire::meta_resource_types @=@ Research Data @=@ interactive_resource:website +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/c_7ad9 +openaire::meta_resource_types @=@ Research Data @=@ sound:musical_composition +openaire::meta_resource_types @=@ Research Data @=@ http://purl.org/coar/resource_type/c_18cd +openaire::meta_resource_types @=@ Research Software @=@ software:research_software +openaire::meta_resource_types @=@ Research Software @=@ http://purl.org/coar/resource_type/c_c950 +openaire::meta_resource_types @=@ Research Software @=@ software +openaire::meta_resource_types @=@ Research Software @=@ http://purl.org/coar/resource_type/c_5ce6 +openaire::meta_resource_types @=@ Research Software @=@ software:source_code +openaire::meta_resource_types @=@ Research Software @=@ http://purl.org/coar/resource_type/QH80-2R4E +openaire::meta_resource_types @=@ Other Research Products @=@ learning_object +openaire::meta_resource_types @=@ Other Research Products @=@ http://purl.org/coar/resource_type/c_e059 +openaire::meta_resource_types @=@ Other Research Products @=@ other +openaire::meta_resource_types @=@ Other Research Products @=@ http://purl.org/coar/resource_type/c_1843 +openaire::meta_resource_types @=@ Other Research Products @=@ trademark +openaire::meta_resource_types @=@ Other Research Products @=@ http://purl.org/coar/resource_type/H6QP-SC1X +openaire::meta_resource_types @=@ Other Research Products @=@ workflow +openaire::meta_resource_types @=@ Other Research Products @=@ http://purl.org/coar/resource_type/c_393c \ No newline at end of file From 03670bb9ce8609a17277e2d6ab6e53190cc8fe7e Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Mon, 16 Oct 2023 10:55:47 +0200 Subject: [PATCH 062/148] [dedup] use common saveParquet and save methods to ensure outputs are compressed --- .../dhp/oa/dedup/SparkCopyOpenorgsMergeRels.java | 10 +++++----- .../dhp/oa/dedup/SparkCopyRelationsNoOpenorgs.java | 7 +------ .../eu/dnetlib/dhp/oa/dedup/SparkCreateMergeRels.java | 2 +- .../dhp/oa/dedup/SparkCreateOrgsDedupRecord.java | 6 +----- .../eu/dnetlib/dhp/oa/dedup/SparkCreateSimRels.java | 2 -- .../eu/dnetlib/dhp/oa/dedup/SparkWhitelistSimRels.java | 2 -- 6 files changed, 8 insertions(+), 21 deletions(-) diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCopyOpenorgsMergeRels.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCopyOpenorgsMergeRels.java index 9d0f61007..eca2193af 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCopyOpenorgsMergeRels.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCopyOpenorgsMergeRels.java @@ -7,6 +7,7 @@ import java.util.Optional; import org.apache.commons.io.IOUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoders; import org.apache.spark.sql.SaveMode; import org.apache.spark.sql.SparkSession; @@ -77,13 +78,12 @@ public class SparkCopyOpenorgsMergeRels extends AbstractSparkAction { log.info("Number of Openorgs Merge Relations collected: {}", mergeRelsRDD.count()); - spark + final Dataset relations = spark .createDataset( mergeRelsRDD.rdd(), - Encoders.bean(Relation.class)) - .write() - .mode(SaveMode.Append) - .parquet(outputPath); + Encoders.bean(Relation.class)); + + saveParquet(relations, outputPath, SaveMode.Append); } private boolean isMergeRel(Relation rel) { diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCopyRelationsNoOpenorgs.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCopyRelationsNoOpenorgs.java index 62cbb5bff..e10f41c82 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCopyRelationsNoOpenorgs.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCopyRelationsNoOpenorgs.java @@ -67,12 +67,7 @@ public class SparkCopyRelationsNoOpenorgs extends AbstractSparkAction { log.debug("Number of non-Openorgs relations collected: {}", simRels.count()); } - spark - .createDataset(simRels.rdd(), Encoders.bean(Relation.class)) - .write() - .mode(SaveMode.Overwrite) - .json(outputPath); - + save(spark.createDataset(simRels.rdd(), Encoders.bean(Relation.class)), outputPath, SaveMode.Overwrite); } } diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateMergeRels.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateMergeRels.java index 2f551b244..babbaaabd 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateMergeRels.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateMergeRels.java @@ -155,7 +155,7 @@ public class SparkCreateMergeRels extends AbstractSparkAction { (FlatMapFunction) cc -> ccToMergeRel(cc, dedupConf), Encoders.bean(Relation.class)); - mergeRels.write().mode(SaveMode.Overwrite).parquet(mergeRelPath); + saveParquet(mergeRels, mergeRelPath, SaveMode.Overwrite); } } diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateOrgsDedupRecord.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateOrgsDedupRecord.java index 8e5e9fd69..25e394f25 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateOrgsDedupRecord.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateOrgsDedupRecord.java @@ -72,11 +72,7 @@ public class SparkCreateOrgsDedupRecord extends AbstractSparkAction { final String mergeRelsPath = DedupUtility.createMergeRelPath(workingPath, actionSetId, "organization"); - rootOrganization(spark, entityPath, mergeRelsPath) - .write() - .mode(SaveMode.Overwrite) - .option("compression", "gzip") - .json(outputPath); + save(rootOrganization(spark, entityPath, mergeRelsPath), outputPath, SaveMode.Overwrite); } diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateSimRels.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateSimRels.java index 5b3cc3111..5f54c34df 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateSimRels.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkCreateSimRels.java @@ -82,8 +82,6 @@ public class SparkCreateSimRels extends AbstractSparkAction { final String outputPath = DedupUtility.createSimRelPath(workingPath, actionSetId, subEntity); removeOutputDir(spark, outputPath); - JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); - SparkDeduper deduper = new SparkDeduper(dedupConf); Dataset simRels = spark diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkWhitelistSimRels.java b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkWhitelistSimRels.java index 94a09ed05..65ad0c327 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkWhitelistSimRels.java +++ b/dhp-workflows/dhp-dedup-openaire/src/main/java/eu/dnetlib/dhp/oa/dedup/SparkWhitelistSimRels.java @@ -67,8 +67,6 @@ public class SparkWhitelistSimRels extends AbstractSparkAction { log.info("workingPath: '{}'", workingPath); log.info("whiteListPath: '{}'", whiteListPath); - JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); - // file format: source####target Dataset whiteListRels = spark .read() From 159388f9c220e33810d50c7a4552ac640c78eb92 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Mon, 16 Oct 2023 11:26:07 +0200 Subject: [PATCH 063/148] testing and fix some issues --- .../main/java/eu/dnetlib/dhp/api/Utils.java | 7 + .../dnetlib/dhp/bulktag/SparkBulkTagJob.java | 11 +- .../community/QueryInformationSystem.java | 34 -- ...kResultToCommunityFromOrganizationJob.java | 50 +-- .../PrepareResultCommunitySet.java | 15 +- .../SparkResultToCommunityFromProject.java | 13 +- .../input_communitytoresult_parameters.json | 13 +- ...t_preparecommunitytoresult_parameters.json | 12 +- .../oozie_app/workflow.xml | 115 +------ .../input_communitytoresult_parameters.json | 28 ++ ...t_preparecommunitytoresult_parameters.json | 33 ++ .../QueryCommunityAPITest.java | 16 +- .../PrepareAssocTest.java | 95 ++++++ .../PrepareAssocTest.java | 88 +++++ .../ResultToCommunityJobTest.java | 323 ++++++++++++++++++ .../preparedInfo/resultCommunityList.json | 36 ++ .../preparedInfo/resultCommunityList.json.gz | Bin 939 -> 0 bytes .../relation/relation | 20 ++ .../preparedInfo/resultcommunitylist | 4 + .../relation/relation | 20 ++ .../sample/dataset | 10 + .../sample/otherresearchproduct | 0 .../sample/publication | 0 .../sample/software | 0 24 files changed, 736 insertions(+), 207 deletions(-) delete mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/QueryInformationSystem.java create mode 100644 dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/input_communitytoresult_parameters.json create mode 100644 dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/input_preparecommunitytoresult_parameters.json rename dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/{bulktag => api}/QueryCommunityAPITest.java (90%) create mode 100644 dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareAssocTest.java create mode 100644 dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareAssocTest.java create mode 100644 dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultToCommunityJobTest.java create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/preparedInfo/resultCommunityList.json delete mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/preparedInfo/resultCommunityList.json.gz create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/relation/relation create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromproject/preparedInfo/resultcommunitylist create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromproject/relation/relation create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromproject/sample/dataset create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromproject/sample/otherresearchproduct create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromproject/sample/publication create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromproject/sample/software diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java index a0eacb774..43d5e7e98 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java @@ -11,6 +11,9 @@ import java.util.stream.Collectors; import javax.management.Query; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.amazonaws.util.StringUtils; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Maps; @@ -21,6 +24,7 @@ import eu.dnetlib.dhp.bulktag.community.CommunityConfiguration; import eu.dnetlib.dhp.bulktag.community.Provider; import eu.dnetlib.dhp.bulktag.criteria.VerbResolver; import eu.dnetlib.dhp.bulktag.criteria.VerbResolverFactory; +import eu.dnetlib.dhp.resulttocommunityfromorganization.SparkResultToCommunityFromOrganizationJob; /** * @author miriam.baglioni @@ -30,6 +34,8 @@ public class Utils implements Serializable { private static final ObjectMapper MAPPER = new ObjectMapper(); private static final VerbResolver resolver = VerbResolverFactory.newInstance(); + private static final Logger log = LoggerFactory.getLogger(Utils.class); + public static CommunityConfiguration getCommunityConfiguration(boolean production) throws IOException { final Map communities = Maps.newHashMap(); List validCommunities = new ArrayList<>(); @@ -126,6 +132,7 @@ public class Utils implements Serializable { throw new RuntimeException(e); } }); + return organizationMap; } diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java index b24ee129a..68c740dd5 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java @@ -95,10 +95,7 @@ public class SparkBulkTagJob { Dataset datasources = readPath( spark, inputPath - .substring( - 0, - inputPath.lastIndexOf("/")) - + "/datasource", + + "datasource", Datasource.class) .filter((FilterFunction) ds -> isOKDatasource(ds)) .map((MapFunction) ds -> ds.getId(), Encoders.STRING()); @@ -106,10 +103,10 @@ public class SparkBulkTagJob { Map>> dsm = cc.getEoscDatasourceMap(); for (String ds : datasources.collectAsList()) { - final String dsId = ds.substring(3); - if (!dsm.containsKey(dsId)) { + // final String dsId = ds.substring(3); + if (!dsm.containsKey(ds)) { ArrayList> eoscList = new ArrayList<>(); - dsm.put(dsId, eoscList); + dsm.put(ds, eoscList); } } diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/QueryInformationSystem.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/QueryInformationSystem.java deleted file mode 100644 index 5fe3cf81f..000000000 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/QueryInformationSystem.java +++ /dev/null @@ -1,34 +0,0 @@ - -package eu.dnetlib.dhp.bulktag.community; - -import java.io.IOException; -import java.util.List; - -import org.apache.commons.io.IOUtils; -import org.dom4j.DocumentException; -import org.xml.sax.SAXException; - -import com.google.common.base.Joiner; - -import eu.dnetlib.dhp.utils.ISLookupClientFactory; -import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; -import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; - -public class QueryInformationSystem { - - public static CommunityConfiguration getCommunityConfiguration(final String isLookupUrl) - throws ISLookUpException, DocumentException, SAXException, IOException { - ISLookUpService isLookUp = ISLookupClientFactory.getLookUpService(isLookupUrl); - final List res = isLookUp - .quickSearchProfile( - IOUtils - .toString( - QueryInformationSystem.class - .getResourceAsStream( - "/eu/dnetlib/dhp/bulktag/query.xq"))); - - final String xmlConf = "" + Joiner.on(" ").join(res) + ""; - - return CommunityConfigurationFactory.newInstance(xmlConf); - } -} diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/SparkResultToCommunityFromOrganizationJob.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/SparkResultToCommunityFromOrganizationJob.java index 43f425b68..fe79f1be1 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/SparkResultToCommunityFromOrganizationJob.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/SparkResultToCommunityFromOrganizationJob.java @@ -2,7 +2,7 @@ package eu.dnetlib.dhp.resulttocommunityfromorganization; import static eu.dnetlib.dhp.PropagationConstant.*; -import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession; +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; import java.util.ArrayList; import java.util.Arrays; @@ -22,6 +22,7 @@ import org.slf4j.LoggerFactory; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.schema.common.ModelConstants; +import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.oaf.Context; import eu.dnetlib.dhp.schema.oaf.Result; import scala.Tuple2; @@ -53,22 +54,15 @@ public class SparkResultToCommunityFromOrganizationJob { final String possibleupdatespath = parser.get("preparedInfoPath"); log.info("preparedInfoPath: {}", possibleupdatespath); - final String resultClassName = parser.get("resultTableName"); - log.info("resultTableName: {}", resultClassName); - - @SuppressWarnings("unchecked") - Class resultClazz = (Class) Class.forName(resultClassName); - SparkConf conf = new SparkConf(); - conf.set("hive.metastore.uris", parser.get("hive_metastore_uris")); - runWithSparkHiveSession( + runWithSparkSession( conf, isSparkSessionManaged, spark -> { - removeOutputDir(spark, outputPath); + // removeOutputDir(spark, outputPath); - execPropagation(spark, inputPath, outputPath, resultClazz, possibleupdatespath); + execPropagation(spark, inputPath, outputPath, possibleupdatespath); }); } @@ -77,22 +71,32 @@ public class SparkResultToCommunityFromOrganizationJob { SparkSession spark, String inputPath, String outputPath, - Class resultClazz, String possibleUpdatesPath) { Dataset possibleUpdates = readPath(spark, possibleUpdatesPath, ResultCommunityList.class); - Dataset result = readPath(spark, inputPath, resultClazz); - result - .joinWith( - possibleUpdates, - result.col("id").equalTo(possibleUpdates.col("resultId")), - "left_outer") - .map(resultCommunityFn(), Encoders.bean(resultClazz)) - .write() - .mode(SaveMode.Overwrite) - .option("compression", "gzip") - .json(outputPath); + ModelSupport.entityTypes + .keySet() + .parallelStream() + .forEach(e -> { + if (ModelSupport.isResult(e)) { + Class resultClazz = ModelSupport.entityTypes.get(e); + removeOutputDir(spark, outputPath + e.name()); + Dataset result = readPath(spark, inputPath + e.name(), resultClazz); + + result + .joinWith( + possibleUpdates, + result.col("id").equalTo(possibleUpdates.col("resultId")), + "left_outer") + .map(resultCommunityFn(), Encoders.bean(resultClazz)) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(outputPath + e.name()); + } + }); + } private static MapFunction, R> resultCommunityFn() { diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java index b0fbf8056..883f5ca86 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java @@ -3,6 +3,7 @@ package eu.dnetlib.dhp.resulttocommunityfromproject; import static eu.dnetlib.dhp.PropagationConstant.*; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession; +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; import java.util.*; @@ -51,16 +52,15 @@ public class PrepareResultCommunitySet { final String outputPath = parser.get("outputPath"); log.info("outputPath: {}", outputPath); - final boolean production = Boolean.valueOf(parser.get("outputPath")); + final boolean production = Boolean.valueOf(parser.get("production")); log.info("production: {}", production); final CommunityEntityMap projectsMap = Utils.getCommunityProjects(production); log.info("projectsMap: {}", new Gson().toJson(projectsMap)); SparkConf conf = new SparkConf(); - conf.set("hive.metastore.uris", parser.get("hive_metastore_uris")); - runWithSparkHiveSession( + runWithSparkSession( conf, isSparkSessionManaged, spark -> { @@ -94,24 +94,27 @@ public class PrepareResultCommunitySet { .select( new Column("source").as("resultId"), new Column("target").as("projectId")) - .groupByKey((MapFunction) r -> (String) r.getAs("source"), Encoders.STRING()) + .groupByKey((MapFunction) r -> (String) r.getAs("resultId"), Encoders.STRING()) .mapGroups((MapGroupsFunction) (k, v) -> { ResultProjectList rpl = new ResultProjectList(); rpl.setResultId(k); ArrayList cl = new ArrayList<>(); - cl.addAll(projectMap.get(v.next().getAs("target"))); + cl.addAll(projectMap.get(v.next().getAs("projectId"))); v.forEachRemaining(r -> { projectMap - .get(r.getAs("target")) + .get(r.getAs("projectId")) .forEach(c -> { if (!cl.contains(c)) cl.add(c); }); }); + if(cl.size() == 0) + return null; rpl.setCommunityList(cl); return rpl; }, Encoders.bean(ResultProjectList.class)) + .filter(Objects::nonNull) .write() .mode(SaveMode.Overwrite) .option("compression", "gzip") diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java index 6d4779ea1..daef6a317 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java @@ -4,6 +4,7 @@ package eu.dnetlib.dhp.resulttocommunityfromproject; import static eu.dnetlib.dhp.PropagationConstant.*; import static eu.dnetlib.dhp.PropagationConstant.PROPAGATION_RESULT_COMMUNITY_ORGANIZATION_CLASS_NAME; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession; +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; import java.io.Serializable; import java.util.ArrayList; @@ -61,20 +62,14 @@ public class SparkResultToCommunityFromProject implements Serializable { final String possibleupdatespath = parser.get("preparedInfoPath"); log.info("preparedInfoPath: {}", possibleupdatespath); - final String resultClassName = parser.get("resultTableName"); - log.info("resultTableName: {}", resultClassName); - - @SuppressWarnings("unchecked") - Class resultClazz = (Class) Class.forName(resultClassName); SparkConf conf = new SparkConf(); - conf.set("hive.metastore.uris", parser.get("hive_metastore_uris")); - runWithSparkHiveSession( + + runWithSparkSession( conf, isSparkSessionManaged, spark -> { -// removeOutputDir(spark, outputPath); execPropagation(spark, inputPath, outputPath, possibleupdatespath); @@ -108,7 +103,7 @@ public class SparkResultToCommunityFromProject implements Serializable { .write() .mode(SaveMode.Overwrite) .option("compression", "gzip") - .json(outputPath); + .json(outputPath + e.name()); } }); diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/input_communitytoresult_parameters.json b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/input_communitytoresult_parameters.json index b6eb309a5..0db8085d1 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/input_communitytoresult_parameters.json +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/input_communitytoresult_parameters.json @@ -5,12 +5,7 @@ "paramDescription": "the path of the sequencial file to read", "paramRequired": true }, - { - "paramName":"h", - "paramLongName":"hive_metastore_uris", - "paramDescription": "the hive metastore uris", - "paramRequired": true - }, + { "paramName": "out", "paramLongName": "outputPath", @@ -23,12 +18,6 @@ "paramDescription": "true if the spark session is managed, false otherwise", "paramRequired": false }, - { - "paramName":"tn", - "paramLongName":"resultTableName", - "paramDescription": "the name of the result table we are currently working on", - "paramRequired": true - }, { "paramName": "p", "paramLongName": "preparedInfoPath", diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/input_preparecommunitytoresult_parameters.json b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/input_preparecommunitytoresult_parameters.json index 8df509abf..8b6291e5d 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/input_preparecommunitytoresult_parameters.json +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/input_preparecommunitytoresult_parameters.json @@ -5,12 +5,6 @@ "paramDescription": "the path of the sequencial file to read", "paramRequired": true }, - { - "paramName":"ocm", - "paramLongName":"organizationtoresultcommunitymap", - "paramDescription": "the map for the association organization communities", - "paramRequired": true - }, { "paramName":"h", "paramLongName":"hive_metastore_uris", @@ -28,6 +22,12 @@ "paramLongName": "outputPath", "paramDescription": "the path used to store temporary output files", "paramRequired": true + }, + { + "paramName": "p", + "paramLongName": "production", + "paramDescription": "the path used to store temporary output files", + "paramRequired": true } ] \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml index b25822ad0..55490e25c 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml @@ -22,7 +22,7 @@ - + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] @@ -90,8 +90,8 @@ eu.dnetlib.dhp.resulttocommunityfromorganization.PrepareResultCommunitySet dhp-enrichment-${projectVersion}.jar - --executor-cores=${sparkExecutorCores} - --executor-memory=${sparkExecutorMemory} + --executor-cores=4 + --executor-memory=10G --driver-memory=${sparkDriverMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} @@ -103,20 +103,13 @@ --sourcePath${sourcePath}/relation --outputPath${workingDir}/preparedInfo/resultCommunityList --hive_metastore_uris${hive_metastore_uris} - + --production${production} - + - - - - - - - - + yarn cluster @@ -135,104 +128,14 @@ --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors} --preparedInfoPath${workingDir}/preparedInfo/resultCommunityList - --sourcePath${sourcePath}/publication - --outputPath${outputPath}/publication - --hive_metastore_uris${hive_metastore_uris} - --resultTableNameeu.dnetlib.dhp.schema.oaf.Publication - --saveGraph${saveGraph} + --sourcePath${sourcePath}/ + --outputPath${outputPath}/ - + - - - yarn - cluster - community2resultfromorganization-Dataset - eu.dnetlib.dhp.resulttocommunityfromorganization.SparkResultToCommunityFromOrganizationJob - dhp-enrichment-${projectVersion}.jar - - --executor-cores=${sparkExecutorCores} - --executor-memory=${sparkExecutorMemory} - --driver-memory=${sparkDriverMemory} - --conf spark.extraListeners=${spark2ExtraListeners} - --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} - --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} - --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.dynamicAllocation.enabled=true - --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors} - - --preparedInfoPath${workingDir}/preparedInfo/resultCommunityList - --sourcePath${sourcePath}/dataset - --outputPath${outputPath}/dataset - --hive_metastore_uris${hive_metastore_uris} - --resultTableNameeu.dnetlib.dhp.schema.oaf.Dataset - --saveGraph${saveGraph} - - - - - - - yarn - cluster - community2resultfromorganization-ORP - eu.dnetlib.dhp.resulttocommunityfromorganization.SparkResultToCommunityFromOrganizationJob - dhp-enrichment-${projectVersion}.jar - - --executor-cores=${sparkExecutorCores} - --executor-memory=${sparkExecutorMemory} - --driver-memory=${sparkDriverMemory} - --conf spark.extraListeners=${spark2ExtraListeners} - --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} - --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} - --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.dynamicAllocation.enabled=true - --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors} - - --preparedInfoPath${workingDir}/preparedInfo/resultCommunityList - --sourcePath${sourcePath}/otherresearchproduct - --outputPath${outputPath}/otherresearchproduct - --hive_metastore_uris${hive_metastore_uris} - --resultTableNameeu.dnetlib.dhp.schema.oaf.OtherResearchProduct - --saveGraph${saveGraph} - - - - - - - - yarn - cluster - community2resultfromorganization-Software - eu.dnetlib.dhp.resulttocommunityfromorganization.SparkResultToCommunityFromOrganizationJob - dhp-enrichment-${projectVersion}.jar - - --executor-cores=${sparkExecutorCores} - --executor-memory=${sparkExecutorMemory} - --driver-memory=${sparkDriverMemory} - --conf spark.extraListeners=${spark2ExtraListeners} - --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} - --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} - --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.dynamicAllocation.enabled=true - --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors} - - --preparedInfoPath${workingDir}/preparedInfo/resultCommunityList - --sourcePath${sourcePath}/software - --outputPath${outputPath}/software - --hive_metastore_uris${hive_metastore_uris} - --resultTableNameeu.dnetlib.dhp.schema.oaf.Software - --saveGraph${saveGraph} - - - - - - diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/input_communitytoresult_parameters.json b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/input_communitytoresult_parameters.json new file mode 100644 index 000000000..0db8085d1 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/input_communitytoresult_parameters.json @@ -0,0 +1,28 @@ +[ + { + "paramName":"s", + "paramLongName":"sourcePath", + "paramDescription": "the path of the sequencial file to read", + "paramRequired": true + }, + + { + "paramName": "out", + "paramLongName": "outputPath", + "paramDescription": "the path used to store temporary output files", + "paramRequired": true + }, + { + "paramName": "ssm", + "paramLongName": "isSparkSessionManaged", + "paramDescription": "true if the spark session is managed, false otherwise", + "paramRequired": false + }, + { + "paramName": "p", + "paramLongName": "preparedInfoPath", + "paramDescription": "the path where prepared info have been stored", + "paramRequired": true + } + +] \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/input_preparecommunitytoresult_parameters.json b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/input_preparecommunitytoresult_parameters.json new file mode 100644 index 000000000..8b6291e5d --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/input_preparecommunitytoresult_parameters.json @@ -0,0 +1,33 @@ +[ + { + "paramName":"s", + "paramLongName":"sourcePath", + "paramDescription": "the path of the sequencial file to read", + "paramRequired": true + }, + { + "paramName":"h", + "paramLongName":"hive_metastore_uris", + "paramDescription": "the hive metastore uris", + "paramRequired": true + }, + { + "paramName": "ssm", + "paramLongName": "isSparkSessionManaged", + "paramDescription": "true if the spark session is managed, false otherwise", + "paramRequired": false + }, + { + "paramName": "out", + "paramLongName": "outputPath", + "paramDescription": "the path used to store temporary output files", + "paramRequired": true + }, + { + "paramName": "p", + "paramLongName": "production", + "paramDescription": "the path used to store temporary output files", + "paramRequired": true + } + +] \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/QueryCommunityAPITest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/api/QueryCommunityAPITest.java similarity index 90% rename from dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/QueryCommunityAPITest.java rename to dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/api/QueryCommunityAPITest.java index a0083dab8..6ee01a6f0 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/QueryCommunityAPITest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/api/QueryCommunityAPITest.java @@ -1,5 +1,5 @@ -package eu.dnetlib.dhp.bulktag; +package eu.dnetlib.dhp.api; import java.util.List; @@ -98,14 +98,22 @@ public class QueryCommunityAPITest { @Test void getCommunityProjects() throws Exception { CommunityEntityMap projectMap = Utils.getCommunityProjects(true); - Assertions.assertFalse(projectMap.containsKey("mes")); - Assertions.assertEquals(33, projectMap.size()); + Assertions .assertTrue( projectMap .keySet() .stream() - .allMatch(k -> projectMap.get(k).stream().allMatch(p -> p.startsWith("40|")))); + .allMatch(k -> k.startsWith("40|"))); + + System.out.println(projectMap); + } + + @Test + void getCommunityOrganizations() throws Exception { + CommunityEntityMap organizationMap = Utils.getCommunityOrganization(true); + Assertions.assertTrue(organizationMap.keySet().stream().allMatch(k -> k.startsWith("20|"))); + } } diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareAssocTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareAssocTest.java new file mode 100644 index 000000000..6536ecc85 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareAssocTest.java @@ -0,0 +1,95 @@ + +package eu.dnetlib.dhp.resulttocommunityfromorganization; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.Gson; + +import eu.dnetlib.dhp.api.Utils; +import eu.dnetlib.dhp.api.model.CommunityEntityMap; +import eu.dnetlib.dhp.bulktag.BulkTagJobTest; +import eu.dnetlib.dhp.bulktag.SparkBulkTagJob; +import eu.dnetlib.dhp.schema.oaf.Dataset; + +/** + * @author miriam.baglioni + * @Date 13/10/23 + */ +public class PrepareAssocTest { + + private static SparkSession spark; + + private static Path workingDir; + + private static final Logger log = LoggerFactory.getLogger(PrepareAssocTest.class); + + @BeforeAll + public static void beforeAll() throws IOException { + workingDir = Files.createTempDirectory(BulkTagJobTest.class.getSimpleName()); + log.info("using work dir {}", workingDir); + + SparkConf conf = new SparkConf(); + conf.setAppName(BulkTagJobTest.class.getSimpleName()); + + conf.setMaster("local[*]"); + conf.set("spark.driver.host", "localhost"); + conf.set("hive.metastore.local", "true"); + conf.set("spark.ui.enabled", "false"); + conf.set("spark.sql.warehouse.dir", workingDir.toString()); + conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString()); + + spark = SparkSession + .builder() + .appName(PrepareAssocTest.class.getSimpleName()) + .config(conf) + .getOrCreate(); + } + + @AfterAll + public static void afterAll() throws IOException { + FileUtils.deleteDirectory(workingDir.toFile()); + spark.stop(); + } + + @Test + void test1() throws Exception { + + PrepareResultCommunitySet + .main( + new String[] { + + "-isSparkSessionManaged", Boolean.FALSE.toString(), + "-sourcePath", + getClass().getResource("/eu/dnetlib/dhp/resulttocommunityfromorganization/relation/").getPath(), + "-outputPath", workingDir.toString() + "/prepared", + "-production", Boolean.TRUE.toString(), + "-hive_metastore_uris", "" + }); + + final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .textFile(workingDir.toString() + "/prepared") + .map(item -> new ObjectMapper().readValue(item, ResultCommunityList.class)); + + tmp.foreach(r -> System.out.println(new ObjectMapper().writeValueAsString(r))); + } + +} diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareAssocTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareAssocTest.java new file mode 100644 index 000000000..0e10b3edf --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareAssocTest.java @@ -0,0 +1,88 @@ + +package eu.dnetlib.dhp.resulttocommunityfromproject; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.commons.io.FileUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.bulktag.BulkTagJobTest; +import eu.dnetlib.dhp.resulttocommunityfromorganization.ResultCommunityList; + +/** + * @author miriam.baglioni + * @Date 13/10/23 + */ +public class PrepareAssocTest { + + private static SparkSession spark; + + private static Path workingDir; + + private static final Logger log = LoggerFactory.getLogger(PrepareAssocTest.class); + + @BeforeAll + public static void beforeAll() throws IOException { + workingDir = Files.createTempDirectory(BulkTagJobTest.class.getSimpleName()); + log.info("using work dir {}", workingDir); + + SparkConf conf = new SparkConf(); + conf.setAppName(BulkTagJobTest.class.getSimpleName()); + + conf.setMaster("local[*]"); + conf.set("spark.driver.host", "localhost"); + conf.set("hive.metastore.local", "true"); + conf.set("spark.ui.enabled", "false"); + conf.set("spark.sql.warehouse.dir", workingDir.toString()); + conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString()); + + spark = SparkSession + .builder() + .appName(PrepareAssocTest.class.getSimpleName()) + .config(conf) + .getOrCreate(); + } + + @AfterAll + public static void afterAll() throws IOException { + FileUtils.deleteDirectory(workingDir.toFile()); + spark.stop(); + } + + @Test + void test1() throws Exception { + + PrepareResultCommunitySet + .main( + new String[] { + + "-isSparkSessionManaged", Boolean.FALSE.toString(), + "-sourcePath", + getClass().getResource("/eu/dnetlib/dhp/resulttocommunityfromproject/relation/").getPath(), + "-outputPath", workingDir.toString() + "/prepared", + "-production", Boolean.TRUE.toString(), + "-hive_metastore_uris", "" + }); + + final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .textFile(workingDir.toString() + "/prepared") + .map(item -> new ObjectMapper().readValue(item, ResultProjectList.class)); + + tmp.foreach(r -> System.out.println(new ObjectMapper().writeValueAsString(r))); + } + +} diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultToCommunityJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultToCommunityJobTest.java new file mode 100644 index 000000000..6a5726cbe --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultToCommunityJobTest.java @@ -0,0 +1,323 @@ + +package eu.dnetlib.dhp.resulttocommunityfromproject; + +import static org.apache.spark.sql.functions.desc; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.commons.io.FileUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.orcidtoresultfromsemrel.OrcidPropagationJobTest; +import eu.dnetlib.dhp.resulttocommunityfromorganization.SparkResultToCommunityFromOrganizationJob; +import eu.dnetlib.dhp.schema.oaf.Dataset; + +public class ResultToCommunityJobTest { + + private static final Logger log = LoggerFactory.getLogger(ResultToCommunityJobTest.class); + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static SparkSession spark; + + private static Path workingDir; + + @BeforeAll + public static void beforeAll() throws IOException { + workingDir = Files.createTempDirectory(ResultToCommunityJobTest.class.getSimpleName()); + log.info("using work dir {}", workingDir); + + SparkConf conf = new SparkConf(); + conf.setAppName(ResultToCommunityJobTest.class.getSimpleName()); + + conf.setMaster("local[*]"); + conf.set("spark.driver.host", "localhost"); + conf.set("hive.metastore.local", "true"); + conf.set("spark.ui.enabled", "false"); + conf.set("spark.sql.warehouse.dir", workingDir.toString()); + conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString()); + + spark = SparkSession + .builder() + .appName(OrcidPropagationJobTest.class.getSimpleName()) + .config(conf) + .getOrCreate(); + } + + @AfterAll + public static void afterAll() throws IOException { + FileUtils.deleteDirectory(workingDir.toFile()); + spark.stop(); + } + + @Test + void testSparkResultToCommunityFromProjectJob() throws Exception { + final String preparedInfoPath = getClass() + .getResource("/eu/dnetlib/dhp/resulttocommunityfromproject/preparedInfo") + .getPath(); + SparkResultToCommunityFromProject + .main( + new String[] { + + "-isSparkSessionManaged", Boolean.FALSE.toString(), + "-sourcePath", getClass() + .getResource("/eu/dnetlib/dhp/resulttocommunityfromproject/sample/") + .getPath(), + + "-outputPath", workingDir.toString() + "/", + "-preparedInfoPath", preparedInfoPath + }); + + final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .textFile(workingDir.toString() + "/dataset") + .map(item -> OBJECT_MAPPER.readValue(item, Dataset.class)); + + tmp.foreach(d -> System.out.println(new ObjectMapper().writeValueAsString(d))); +// Assertions.assertEquals(10, tmp.count()); +// org.apache.spark.sql.Dataset verificationDataset = spark +// .createDataset(tmp.rdd(), Encoders.bean(Dataset.class)); +// +// verificationDataset.createOrReplaceTempView("dataset"); +// +// String query = "select id, MyT.id community " +// + "from dataset " +// + "lateral view explode(context) c as MyT " +// + "lateral view explode(MyT.datainfo) d as MyD " +// + "where MyD.inferenceprovenance = 'propagation'"; +// +// org.apache.spark.sql.Dataset resultExplodedProvenance = spark.sql(query); +// Assertions.assertEquals(5, resultExplodedProvenance.count()); +// Assertions +// .assertEquals( +// 0, +// resultExplodedProvenance +// .filter("id = '50|dedup_wf_001::afaf128022d29872c4dad402b2db04fe'") +// .count()); +// Assertions +// .assertEquals( +// 1, +// resultExplodedProvenance +// .filter("id = '50|dedup_wf_001::3f62cfc27024d564ea86760c494ba93b'") +// .count()); +// Assertions +// .assertEquals( +// "beopen", +// resultExplodedProvenance +// .select("community") +// .where( +// resultExplodedProvenance +// .col("id") +// .equalTo( +// "50|dedup_wf_001::3f62cfc27024d564ea86760c494ba93b")) +// .collectAsList() +// .get(0) +// .getString(0)); +// +// Assertions +// .assertEquals( +// 2, +// resultExplodedProvenance +// .filter("id = '50|od________18::8887b1df8b563c4ea851eb9c882c9d7b'") +// .count()); +// Assertions +// .assertEquals( +// "mes", +// resultExplodedProvenance +// .select("community") +// .where( +// resultExplodedProvenance +// .col("id") +// .equalTo( +// "50|od________18::8887b1df8b563c4ea851eb9c882c9d7b")) +// .sort(desc("community")) +// .collectAsList() +// .get(0) +// .getString(0)); +// Assertions +// .assertEquals( +// "euromarine", +// resultExplodedProvenance +// .select("community") +// .where( +// resultExplodedProvenance +// .col("id") +// .equalTo( +// "50|od________18::8887b1df8b563c4ea851eb9c882c9d7b")) +// .sort(desc("community")) +// .collectAsList() +// .get(1) +// .getString(0)); +// +// Assertions +// .assertEquals( +// 1, +// resultExplodedProvenance +// .filter("id = '50|doajarticles::8d817039a63710fcf97e30f14662c6c8'") +// .count()); +// Assertions +// .assertEquals( +// "mes", +// resultExplodedProvenance +// .select("community") +// .where( +// resultExplodedProvenance +// .col("id") +// .equalTo( +// "50|doajarticles::8d817039a63710fcf97e30f14662c6c8")) +// .sort(desc("community")) +// .collectAsList() +// .get(0) +// .getString(0)); +// +// Assertions +// .assertEquals( +// 1, +// resultExplodedProvenance +// .filter("id = '50|doajarticles::3c98f0632f1875b4979e552ba3aa01e6'") +// .count()); +// Assertions +// .assertEquals( +// "mes", +// resultExplodedProvenance +// .select("community") +// .where( +// resultExplodedProvenance +// .col("id") +// .equalTo( +// "50|doajarticles::3c98f0632f1875b4979e552ba3aa01e6")) +// .sort(desc("community")) +// .collectAsList() +// .get(0) +// .getString(0)); +// +// query = "select id, MyT.id community " +// + "from dataset " +// + "lateral view explode(context) c as MyT " +// + "lateral view explode(MyT.datainfo) d as MyD "; +// +// org.apache.spark.sql.Dataset resultCommunityId = spark.sql(query); +// +// Assertions.assertEquals(10, resultCommunityId.count()); +// +// Assertions +// .assertEquals( +// 1, +// resultCommunityId +// .filter("id = '50|dedup_wf_001::afaf128022d29872c4dad402b2db04fe'") +// .count()); +// Assertions +// .assertEquals( +// "beopen", +// resultCommunityId +// .select("community") +// .where( +// resultCommunityId +// .col("id") +// .equalTo( +// "50|dedup_wf_001::afaf128022d29872c4dad402b2db04fe")) +// .collectAsList() +// .get(0) +// .getString(0)); +// +// Assertions +// .assertEquals( +// 1, +// resultCommunityId +// .filter("id = '50|dedup_wf_001::3f62cfc27024d564ea86760c494ba93b'") +// .count()); +// +// Assertions +// .assertEquals( +// 3, +// resultCommunityId +// .filter("id = '50|od________18::8887b1df8b563c4ea851eb9c882c9d7b'") +// .count()); +// Assertions +// .assertEquals( +// "beopen", +// resultCommunityId +// .select("community") +// .where( +// resultCommunityId +// .col("id") +// .equalTo( +// "50|od________18::8887b1df8b563c4ea851eb9c882c9d7b")) +// .sort(desc("community")) +// .collectAsList() +// .get(2) +// .getString(0)); +// +// Assertions +// .assertEquals( +// 2, +// resultCommunityId +// .filter("id = '50|doajarticles::8d817039a63710fcf97e30f14662c6c8'") +// .count()); +// Assertions +// .assertEquals( +// "euromarine", +// resultCommunityId +// .select("community") +// .where( +// resultCommunityId +// .col("id") +// .equalTo( +// "50|doajarticles::8d817039a63710fcf97e30f14662c6c8")) +// .sort(desc("community")) +// .collectAsList() +// .get(1) +// .getString(0)); +// +// Assertions +// .assertEquals( +// 3, +// resultCommunityId +// .filter("id = '50|doajarticles::3c98f0632f1875b4979e552ba3aa01e6'") +// .count()); +// Assertions +// .assertEquals( +// "euromarine", +// resultCommunityId +// .select("community") +// .where( +// resultCommunityId +// .col("id") +// .equalTo( +// "50|doajarticles::3c98f0632f1875b4979e552ba3aa01e6")) +// .sort(desc("community")) +// .collectAsList() +// .get(2) +// .getString(0)); +// Assertions +// .assertEquals( +// "ni", +// resultCommunityId +// .select("community") +// .where( +// resultCommunityId +// .col("id") +// .equalTo( +// "50|doajarticles::3c98f0632f1875b4979e552ba3aa01e6")) +// .sort(desc("community")) +// .collectAsList() +// .get(0) +// .getString(0)); + } +} diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/preparedInfo/resultCommunityList.json b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/preparedInfo/resultCommunityList.json new file mode 100644 index 000000000..bd4674c4f --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/preparedInfo/resultCommunityList.json @@ -0,0 +1,36 @@ +{"communityList":["beopen"],"resultId":"50|dedup_wf_001::afaf128022d29872c4dad402b2db04fe"} +{"communityList":["beopen"],"resultId":"50|dedup_wf_001::3f62cfc27024d564ea86760c494ba93b"} +{"communityList":["euromarine","mes"],"resultId":"50|od________18::8887b1df8b563c4ea851eb9c882c9d7b"} +{"communityList":["euromarine","mes"],"resultId":"50|doajarticles::8d817039a63710fcf97e30f14662c6c8"} +{"communityList":["euromarine","mes"],"resultId":"50|doajarticles::3c98f0632f1875b4979e552ba3aa01e6"} +{"communityList":["euromarine","mes"],"resultId":"50|doajarticles::53b70ea6e0769d02ddf93307ec8e3e92"} +{"communityList":["euromarine","mes"],"resultId":"50|dedup_wf_001::ef1ac6efc10f420fa9e190e49644f1f2"} +{"communityList":["euromarine","mes"],"resultId":"50|dedup_wf_001::b738aa93950dddfb0294df2e8fdf0579"} +{"communityList":["euromarine","mes"],"resultId":"50|doajarticles::befccb1f9e6b833fd82e587737ae9e7d"} +{"communityList":["euromarine","mes"],"resultId":"50|doajarticles::bf1cba621615e27db1692865a5f35a0b"} +{"communityList":["euromarine","mes"],"resultId":"50|dedup_wf_001::e105de571b336daae05f0e75cf740c5c"} +{"communityList":["euromarine","mes"],"resultId":"50|dedup_wf_001::1fe4f347c9df657b7ba520987d79436e"} +{"communityList":["euromarine","mes"],"resultId":"50|doajarticles::6d7c00a8c8e59f0215459e2e4ee3fd6c"} +{"communityList":["euromarine","mes"],"resultId":"50|dedup_wf_001::39ec88ef4127db0ea1b88938f1c52889"} +{"communityList":["euromarine","mes"],"resultId":"50|dedup_wf_001::3496709db804d98f76c45d7ed023dd95"} +{"communityList":["euromarine","mes"],"resultId":"50|doajarticles::dc97fffbdb6d35f792fc0ab428ff065c"} +{"communityList":["euromarine","mes"],"resultId":"50|dedup_wf_001::b61d082d96619d9b7a876e6dce44cf65"} +{"communityList":["euromarine","mes"],"resultId":"50|dedup_wf_001::32a96881c3036cf2d2165bb2d276ea82"} +{"communityList":["euromarine","mes"],"resultId":"50|dedup_wf_001::76e9e6a959ba588483c74ec580369864"} +{"communityList":["euromarine","mes"],"resultId":"50|dedup_wf_001::1487a0a92572376d95d6cc3f066504b7"} +{"communityList":["euromarine","mes"],"resultId":"50|dedup_wf_001::413a0a0656f888cce9c15f6be6df60e3"} +{"communityList":["euromarine","mes"],"resultId":"50|dedup_wf_001::25c3e91960cbd7a8f95a2e511cbffddd"} +{"communityList":["euromarine","mes"],"resultId":"50|dedup_wf_001::4d523b52094a689856e479bb99063c7a"} +{"communityList":["euromarine","mes"],"resultId":"50|od______2663::393c7262bb71642b7bb4c67cfeab02c5"} +{"communityList":["euromarine","mes"],"resultId":"50|dedup_wf_001::0b3333d875b91ffa4db0735efec94e7a"} +{"communityList":["euromarine","mes"],"resultId":"50|dedup_wf_001::0699c30043edfae40786d80acd20d300"} +{"communityList":["euromarine","mes"],"resultId":"50|doajarticles::7f29ade677e66ffbf1312fa837bc73ca"} +{"communityList":["euromarine","mes"],"resultId":"50|doajarticles::7ba6627ac7590d367cc01bbac4d518e8"} +{"communityList":["euromarine","mes"],"resultId":"50|dedup_wf_001::8ad9bc047433401947dc0cdb4a989cee"} +{"communityList":["euromarine","mes"],"resultId":"50|dedup_wf_001::cce70f27d85df658479d0ec0046a4eb3"} +{"communityList":["euromarine","mes"],"resultId":"50|dedup_wf_001::74304be834b7013dbaeb73c3a19a654b"} +{"communityList":["euromarine","mes"],"resultId":"50|dedup_wf_001::46fc13a87befb6a83ac9c63580528ab0"} +{"communityList":["euromarine","mes"],"resultId":"50|doajarticles::66f1867488b62d9c9fb734273775e203"} +{"communityList":["euromarine","mes"],"resultId":"50|dedup_wf_001::79236995d5c30e5234a47cee4a728cae"} +{"communityList":["euromarine","mes"],"resultId":"50|od______2386::cb7f6cb01d1a835612731d645842f699"} +{"communityList":["euromarine","mes"],"resultId":"50|dedup_wf_001::d424daa43f97a434eb0a12289410cade"} \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/preparedInfo/resultCommunityList.json.gz b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/preparedInfo/resultCommunityList.json.gz deleted file mode 100644 index 8b452d0e17e06baf8f3ac6d62277c2a08d88a13c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 939 zcmV;c162GUiwFqE4x3&819D|^b!>D)Z*6ULZfSIROlfm;E^2dcZUC)UNs44g4Bh(} zx$cD`LZP8@l!t}WKNm@oi~mx3dP9D_`Sa7$m-mmK zfBoU{)92g6kGGT0AM*X}r#H8c`RU8M&)>)G;kMzo$oTT%?JxBfU_3lnS&b<$XG|5I zj$wf?&x`|DdHeR=*L$D6il&!OhKaEW(}bA7QSq3r^Uu%DFCU+u?BnD6yuG~3&%s|TnnNK`39&|9gmQR^^)cy&=s6Bd#i*H5QLtIB+1TkZ{EX71BWqR2%q`4|Ej2kV zLaE`$Xbu`XC&wA2Q)t_+wGHmSqYzhtV#169)z=f?4T2SrFGW$MVAci z#cRb&=If|3Y@6Y=v(gpw{#pKlLaMD5z?D&!2rLu=<}o4q`)eGAYpO*Oa}u@F)L3Jd z$>QU@{Qg;9qjXTxWOLryKfW1p80T2`*F7Zf?_;kvCf4TVfQ)M)3`GtHkAtS7p7*EN zE0oqDLv^;uehEwIGY-fOHYCG57pbymXd_8k?Iti*9Lp|GzHD@<-FQ*EK Date: Mon, 16 Oct 2023 11:46:12 +0200 Subject: [PATCH 064/148] new spark parrameter updated --- .../oozie_app/workflow.xml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml index 55490e25c..7a5fd7434 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml @@ -90,14 +90,16 @@ eu.dnetlib.dhp.resulttocommunityfromorganization.PrepareResultCommunitySet dhp-enrichment-${projectVersion}.jar - --executor-cores=4 - --executor-memory=10G + --executor-cores=6 + --executor-memory=5G + --spark.executor.memoryOverhead=3g + --conf spark.sql.shuffle.partitions=3284 --driver-memory=${sparkDriverMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.dynamicAllocation.enabled=true + --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors} --sourcePath${sourcePath}/relation @@ -117,14 +119,15 @@ eu.dnetlib.dhp.resulttocommunityfromorganization.SparkResultToCommunityFromOrganizationJob dhp-enrichment-${projectVersion}.jar - --executor-cores=${sparkExecutorCores} - --executor-memory=${sparkExecutorMemory} + --executor-cores=6 + --executor-memory=5G + --spark.executor.memoryOverhead=3g + --conf spark.sql.shuffle.partitions=3284 --driver-memory=${sparkDriverMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.dynamicAllocation.enabled=true --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors} --preparedInfoPath${workingDir}/preparedInfo/resultCommunityList From 6dfcd0c9a2072c589709a97769185a966def58b4 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Mon, 16 Oct 2023 12:57:18 +0200 Subject: [PATCH 065/148] [raw graph] mapping original resource types --- .../raw/AbstractMdRecordToOafMapper.java | 63 ++++--- .../dhp/oa/graph/raw/OafToOafMapper.java | 41 +++-- .../dhp/oa/graph/raw/OdfToOafMapper.java | 22 ++- .../oa/graph/raw/OriginalTypeComparator.java | 43 ++--- .../dnetlib/dhp/oa/graph/raw/MappersTest.java | 168 ++++++++++++++++-- .../dnetlib/dhp/oa/graph/raw/oaf_crossref.xml | 68 +++++++ 6 files changed, 308 insertions(+), 97 deletions(-) create mode 100644 dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/oaf_crossref.xml diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java index 5f0acbab7..49133cedb 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java @@ -157,20 +157,20 @@ public abstract class AbstractMdRecordToOafMapper { if (vocs.vocabularyExists(OPENAIRE_META_RESOURCE_TYPE)) { Optional instanceTypeMapping = instances - .stream() - .flatMap(i -> i.getInstanceTypeMapping().stream()) - .filter(t -> OPENAIRE_COAR_RESOURCE_TYPES_3_1.equals(t.getVocabularyName())) - .findFirst(); + .stream() + .flatMap(i -> i.getInstanceTypeMapping().stream()) + .filter(t -> OPENAIRE_COAR_RESOURCE_TYPES_3_1.equals(t.getVocabularyName())) + .findFirst(); if (!instanceTypeMapping.isPresent()) { - throw new IllegalStateException("unable to find an instance from " + OPENAIRE_COAR_RESOURCE_TYPES_3_1); + return null; } else { final String typeCode = instanceTypeMapping.get().getTypeCode(); return Optional - .ofNullable(vocs.lookupTermBySynonym(OPENAIRE_META_RESOURCE_TYPE, typeCode)) - .orElseThrow(() -> - new IllegalStateException("unable to find a synonym for '" + typeCode + "' in " + - OPENAIRE_META_RESOURCE_TYPE)); + .ofNullable(vocs.lookupTermBySynonym(OPENAIRE_META_RESOURCE_TYPE, typeCode)) + .orElseThrow( + () -> new IllegalStateException("unable to find a synonym for '" + typeCode + "' in " + + OPENAIRE_META_RESOURCE_TYPE)); } } else { throw new IllegalStateException("vocabulary '" + OPENAIRE_META_RESOURCE_TYPE + "' not available"); @@ -197,7 +197,8 @@ public abstract class AbstractMdRecordToOafMapper { final DataInfo info, final long lastUpdateTimestamp) { - final OafEntity entity = createEntity(doc, type, metaResourceType, instances, collectedFrom, info, lastUpdateTimestamp); + final OafEntity entity = createEntity( + doc, type, metaResourceType, instances, collectedFrom, info, lastUpdateTimestamp); final Set originalId = Sets.newHashSet(entity.getOriginalId()); originalId.add(entity.getId()); @@ -550,29 +551,33 @@ public abstract class AbstractMdRecordToOafMapper { protected abstract String findOriginalType(Document doc); protected List prepareInstanceTypeMapping(Document doc) { - return Optional.ofNullable(findOriginalType(doc)) - .map(originalType -> { - final List mappings = Lists.newArrayList(); + return Optional + .ofNullable(findOriginalType(doc)) + .map(originalType -> { + final List mappings = Lists.newArrayList(); - if (vocs.vocabularyExists(OPENAIRE_COAR_RESOURCE_TYPES_3_1)) { + if (vocs.vocabularyExists(OPENAIRE_COAR_RESOURCE_TYPES_3_1)) { - // TODO verify what the vocabs return when a synonym is not defined - Optional.ofNullable(vocs.lookupTermBySynonym(OPENAIRE_COAR_RESOURCE_TYPES_3_1, originalType)) - .ifPresent(coarTerm -> { - mappings.add(OafMapperUtils.instanceTypeMapping(originalType, coarTerm)); - if (vocs.vocabularyExists(OPENAIRE_USER_RESOURCE_TYPES)) { + // TODO verify what the vocabs return when a synonym is not defined + Optional + .ofNullable(vocs.lookupTermBySynonym(OPENAIRE_COAR_RESOURCE_TYPES_3_1, originalType)) + .ifPresent(coarTerm -> { + mappings.add(OafMapperUtils.instanceTypeMapping(originalType, coarTerm)); + if (vocs.vocabularyExists(OPENAIRE_USER_RESOURCE_TYPES)) { - // TODO verify what the vocabs return when a synonym is not defined - Optional - .ofNullable(vocs.lookupTermBySynonym(OPENAIRE_USER_RESOURCE_TYPES, coarTerm.getClassid())) - .ifPresent(type -> mappings.add(OafMapperUtils.instanceTypeMapping(originalType, type))); - } - }); - } + // TODO verify what the vocabs return when a synonym is not defined + Optional + .ofNullable( + vocs.lookupTermBySynonym(OPENAIRE_USER_RESOURCE_TYPES, coarTerm.getClassid())) + .ifPresent( + type -> mappings.add(OafMapperUtils.instanceTypeMapping(originalType, type))); + } + }); + } - return mappings; - }) - .orElse(new ArrayList<>()); + return mappings; + }) + .orElse(new ArrayList<>()); } private Journal prepareJournal(final Document doc, final DataInfo info) { diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java index 6bcc8ec44..a63296d18 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OafToOafMapper.java @@ -1,17 +1,8 @@ package eu.dnetlib.dhp.oa.graph.raw; -import com.google.common.collect.Lists; -import eu.dnetlib.dhp.common.PacePerson; -import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup; -import eu.dnetlib.dhp.schema.oaf.*; -import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions; -import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; -import eu.dnetlib.dhp.schema.oaf.utils.ModelHardLimits; -import org.apache.commons.lang3.StringUtils; -import org.dom4j.Document; -import org.dom4j.Element; -import org.dom4j.Node; +import static eu.dnetlib.dhp.schema.common.ModelConstants.*; +import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.*; import java.net.URLDecoder; import java.util.ArrayList; @@ -20,8 +11,19 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import static eu.dnetlib.dhp.schema.common.ModelConstants.*; -import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.*; +import org.apache.commons.lang3.StringUtils; +import org.dom4j.Document; +import org.dom4j.Element; +import org.dom4j.Node; + +import com.google.common.collect.Lists; + +import eu.dnetlib.dhp.common.PacePerson; +import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup; +import eu.dnetlib.dhp.schema.oaf.*; +import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions; +import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; +import eu.dnetlib.dhp.schema.oaf.utils.ModelHardLimits; public class OafToOafMapper extends AbstractMdRecordToOafMapper { @@ -201,12 +203,13 @@ public class OafToOafMapper extends AbstractMdRecordToOafMapper { */ @Override protected String findOriginalType(Document doc) { - return (String) doc.selectNodes("//dc:type") - .stream() - .map(o -> "" + ((Node) o).getText().trim()) - .sorted(new OriginalTypeComparator()) - .findFirst() - .orElse(null); + return (String) doc + .selectNodes("//dc:type") + .stream() + .map(o -> "" + ((Node) o).getText().trim()) + .sorted(new OriginalTypeComparator()) + .findFirst() + .orElse(null); } @Override diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java index c383e9300..e63b01a00 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OdfToOafMapper.java @@ -230,20 +230,18 @@ public class OdfToOafMapper extends AbstractMdRecordToOafMapper { */ @Override protected String findOriginalType(Document doc) { - String resourceType = Optional.ofNullable((Element) doc.selectSingleNode( + return Optional + .ofNullable( + (Element) doc + .selectSingleNode( "//*[local-name()='metadata']/*[local-name() = 'resource']/*[local-name() = 'resourceType']")) - .map(element -> { - final String resourceTypeURI = element.attributeValue("anyURI"); - final String resourceTypeTxt = element.getText(); - - return ObjectUtils.firstNonNull(resourceTypeURI, resourceTypeTxt); - }) - .orElse(doc.valueOf( - "//*[local-name()='metadata']/*[local-name() = 'resource']/*[local-name() = 'CobjCategory']/text()") - ); - - return resourceType; + .map(element -> { + final String resourceTypeURI = element.attributeValue("anyURI"); + final String resourceTypeTxt = element.getText(); + return ObjectUtils.firstNonNull(resourceTypeURI, resourceTypeTxt); + }) + .orElse(null); } @Override diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OriginalTypeComparator.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OriginalTypeComparator.java index 2eeead32e..c3d8b4789 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OriginalTypeComparator.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/OriginalTypeComparator.java @@ -1,32 +1,33 @@ -package eu.dnetlib.dhp.oa.graph.raw; -import java.util.Comparator; +package eu.dnetlib.dhp.oa.graph.raw; import static org.apache.commons.lang3.StringUtils.contains; import static org.apache.commons.lang3.StringUtils.startsWith; +import java.util.Comparator; + public class OriginalTypeComparator implements Comparator { - @Override - public int compare(String t1, String t2) { + @Override + public int compare(String t1, String t2) { - if (t1.equals(t2)) { - return 0; - } - if (startsWith(t1, "http") && contains(t1, "coar") && contains(t1, "resource_type")) { - return -1; - } - if (startsWith(t2, "http") && contains(t2, "coar") && contains(t2, "resource_type")) { - return 1; - } - if (startsWith(t1, "info:eu-repo/semantics")) { - return -1; - } - if (startsWith(t2, "info:eu-repo/semantics")) { - return 1; - } + if (t1.equals(t2)) { + return 0; + } + if (startsWith(t1, "http") && contains(t1, "coar") && contains(t1, "resource_type")) { + return -1; + } + if (startsWith(t2, "http") && contains(t2, "coar") && contains(t2, "resource_type")) { + return 1; + } + if (startsWith(t1, "info:eu-repo/semantics")) { + return -1; + } + if (startsWith(t2, "info:eu-repo/semantics")) { + return 1; + } - return t1.compareTo(t2); - } + return t1.compareTo(t2); + } } diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java index 4d30a3fa8..e6997ac1d 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java @@ -14,6 +14,8 @@ import java.util.stream.Collectors; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.sql.Encoders; import org.dom4j.DocumentException; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -25,8 +27,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import eu.dnetlib.dhp.common.Constants; import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup; +import eu.dnetlib.dhp.oa.graph.clean.CleaningRuleMap; +import eu.dnetlib.dhp.oa.graph.clean.OafCleaner; import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.oaf.*; +import eu.dnetlib.dhp.schema.oaf.utils.GraphCleaningFunctions; import eu.dnetlib.dhp.schema.oaf.utils.IdentifierFactory; import eu.dnetlib.dhp.schema.oaf.utils.PidType; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; @@ -119,19 +124,21 @@ class MappersTest { assertNotNull(instance.getInstanceTypeMapping()); assertEquals(2, instance.getInstanceTypeMapping().size()); - Optional coarType = instance.getInstanceTypeMapping() - .stream() - .filter(itm -> ModelConstants.OPENAIRE_COAR_RESOURCE_TYPES_3_1.equals(itm.getVocabularyName())) - .findFirst(); + Optional coarType = instance + .getInstanceTypeMapping() + .stream() + .filter(itm -> ModelConstants.OPENAIRE_COAR_RESOURCE_TYPES_3_1.equals(itm.getVocabularyName())) + .findFirst(); assertTrue(coarType.isPresent()); assertEquals("http://purl.org/coar/resource_type/c_5794", coarType.get().getTypeCode()); assertEquals("conference paper", coarType.get().getTypeLabel()); - Optional userType = instance.getInstanceTypeMapping() - .stream() - .filter(itm -> ModelConstants.OPENAIRE_USER_RESOURCE_TYPES.equals(itm.getVocabularyName())) - .findFirst(); + Optional userType = instance + .getInstanceTypeMapping() + .stream() + .filter(itm -> ModelConstants.OPENAIRE_USER_RESOURCE_TYPES.equals(itm.getVocabularyName())) + .findFirst(); assertTrue(userType.isPresent()); assertEquals("Article", userType.get().getTypeCode()); @@ -266,8 +273,8 @@ class MappersTest { }); Publication p_cleaned = cleanup(p, vocs); - assertEquals("0000", p_cleaned.getInstance().get(0).getRefereed().getClassid()); - assertEquals("Unknown", p_cleaned.getInstance().get(0).getRefereed().getClassname()); + assertEquals("0002", p_cleaned.getInstance().get(0).getRefereed().getClassid()); + assertEquals("nonPeerReviewed", p_cleaned.getInstance().get(0).getRefereed().getClassname()); assertNotNull(p.getInstance().get(0).getPid()); assertEquals(2, p.getInstance().get(0).getPid().size()); @@ -485,8 +492,8 @@ class MappersTest { }); Publication p_cleaned = cleanup(p, vocs); - assertEquals("0000", p_cleaned.getInstance().get(0).getRefereed().getClassid()); - assertEquals("Unknown", p_cleaned.getInstance().get(0).getRefereed().getClassname()); + assertEquals("0002", p_cleaned.getInstance().get(0).getRefereed().getClassid()); + assertEquals("nonPeerReviewed", p_cleaned.getInstance().get(0).getRefereed().getClassname()); } @Test @@ -604,8 +611,137 @@ class MappersTest { assertTrue(i.getUrl().contains("https://clinicaltrials.gov/ct2/show/NCT02321059")); Dataset d_cleaned = cleanup(d, vocs); - assertEquals("0000", d_cleaned.getInstance().get(0).getRefereed().getClassid()); - assertEquals("Unknown", d_cleaned.getInstance().get(0).getRefereed().getClassname()); + assertEquals("0002", d_cleaned.getInstance().get(0).getRefereed().getClassid()); + assertEquals("nonPeerReviewed", d_cleaned.getInstance().get(0).getRefereed().getClassname()); + } + + @Test + void test_record_from_Crossref() throws IOException { + + final CleaningRuleMap mapping = CleaningRuleMap.create(vocs); + + final String xml = IOUtils + .toString(Objects.requireNonNull(getClass().getResourceAsStream("oaf_crossref.xml"))); + final List list = new OafToOafMapper(vocs, false, true).processMdRecord(xml); + + assertEquals(1, list.size()); + assertTrue(list.get(0) instanceof Publication); + + final Publication p = OafCleaner.apply(fixVocabularyNames((Publication) list.get(0)), mapping); + + assertNotNull(p.getDateofcollection()); + assertEquals("2020-08-06T07:04:09.62Z", p.getDateofcollection()); + + assertNotNull(p.getDateoftransformation()); + assertEquals("2020-08-06T07:20:57.911Z", p.getDateoftransformation()); + + assertNotNull(p.getDataInfo()); + assertFalse(p.getDataInfo().getInvisible()); + assertFalse(p.getDataInfo().getDeletedbyinference()); + assertEquals("0.9", p.getDataInfo().getTrust()); + + assertValidId(p.getId()); + assertEquals(2, p.getOriginalId().size()); + + assertEquals("50|doi_________::7f0f7807f17db50e5c2b5c452ccaf06d", p.getOriginalId().get(0)); + assertValidId(p.getCollectedfrom().get(0).getKey()); + + assertNotNull(p.getTitle()); + assertEquals(1, p.getTitle().size()); + assertEquals( + "A case report of serious haemolysis in a glucose-6-phosphate dehydrogenase-deficient COVID-19 patient receiving hydroxychloroquine", + p + .getTitle() + .get(0) + .getValue()); + + assertNotNull(p.getDescription()); + assertEquals(0, p.getDescription().size()); + + assertEquals(8, p.getAuthor().size()); + + assertNotNull(p.getInstance()); + assertEquals(1, p.getInstance().size()); + + final Instance i = p.getInstance().get(0); + + assertNotNull(i.getAccessright()); + assertEquals(ModelConstants.DNET_ACCESS_MODES, i.getAccessright().getSchemeid()); + assertEquals(ModelConstants.DNET_ACCESS_MODES, i.getAccessright().getSchemename()); + assertEquals("OPEN", i.getAccessright().getClassid()); + assertEquals("Open Access", i.getAccessright().getClassname()); + + assertNotNull(i.getCollectedfrom()); + assertEquals("10|openaire____::081b82f96300b6a6e3d282bad31cb6e2", i.getCollectedfrom().getKey()); + assertEquals("Crossref", i.getCollectedfrom().getValue()); + + assertNotNull(i.getHostedby()); + assertEquals("10|openaire____::55045bd2a65019fd8e6741a755395c8c", i.getHostedby().getKey()); + assertEquals("Unknown Repository", i.getHostedby().getValue()); + + assertNotNull(i.getInstancetype()); + assertEquals("0001", i.getInstancetype().getClassid()); + assertEquals("Article", i.getInstancetype().getClassname()); + assertEquals(ModelConstants.DNET_PUBLICATION_RESOURCE, i.getInstancetype().getSchemeid()); + assertEquals(ModelConstants.DNET_PUBLICATION_RESOURCE, i.getInstancetype().getSchemename()); + + assertNull(i.getLicense()); + assertNotNull(i.getDateofacceptance()); + assertEquals("2020-06-04", i.getDateofacceptance().getValue()); + + assertNull(i.getProcessingchargeamount()); + assertNull(i.getProcessingchargecurrency()); + + assertNotNull(i.getPid()); + assertEquals(1, i.getPid().size()); + + assertNotNull(i.getAlternateIdentifier()); + assertEquals(0, i.getAlternateIdentifier().size()); + + assertNotNull(i.getUrl()); + assertEquals(1, i.getUrl().size()); + assertTrue(i.getUrl().contains("http://dx.doi.org/10.1080/23744235.2020.1774644")); + + assertEquals("", p.getInstance().get(0).getRefereed().getClassid()); + assertEquals("", p.getInstance().get(0).getRefereed().getClassname()); + + Publication p_cleaned = cleanup(p, vocs); + + assertEquals("0001", p_cleaned.getInstance().get(0).getRefereed().getClassid()); + assertEquals("peerReviewed", p_cleaned.getInstance().get(0).getRefereed().getClassname()); + + assertNotNull(p_cleaned.getMetaResourceType()); + assertEquals("Research Literature", p_cleaned.getMetaResourceType().getClassid()); + assertEquals("Research Literature", p_cleaned.getMetaResourceType().getClassname()); + assertEquals(ModelConstants.OPENAIRE_META_RESOURCE_TYPE, p_cleaned.getMetaResourceType().getSchemeid()); + assertEquals(ModelConstants.OPENAIRE_META_RESOURCE_TYPE, p_cleaned.getMetaResourceType().getSchemename()); + + assertNotNull(p_cleaned.getInstance().get(0).getInstanceTypeMapping()); + assertEquals(2, p_cleaned.getInstance().get(0).getInstanceTypeMapping().size()); + + assertTrue( + p_cleaned + .getInstance() + .get(0) + .getInstanceTypeMapping() + .stream() + .anyMatch( + t -> "journal-article".equals(t.getOriginalType()) && + ModelConstants.OPENAIRE_COAR_RESOURCE_TYPES_3_1.equals(t.getVocabularyName()) && + "http://purl.org/coar/resource_type/c_2df8fbb1".equals(t.getTypeCode()) && + "research article".equals(t.getTypeLabel()))); + + assertTrue( + p_cleaned + .getInstance() + .get(0) + .getInstanceTypeMapping() + .stream() + .anyMatch( + t -> "journal-article".equals(t.getOriginalType()) && + ModelConstants.OPENAIRE_USER_RESOURCE_TYPES.equals(t.getVocabularyName()) && + "Article".equals(t.getTypeCode()) && + "Article".equals(t.getTypeLabel()))); } @Test @@ -908,8 +1044,8 @@ class MappersTest { }); Dataset p_cleaned = cleanup(p, vocs); - assertEquals("0000", p_cleaned.getInstance().get(0).getRefereed().getClassid()); - assertEquals("Unknown", p_cleaned.getInstance().get(0).getRefereed().getClassname()); + assertEquals("0002", p_cleaned.getInstance().get(0).getRefereed().getClassid()); + assertEquals("nonPeerReviewed", p_cleaned.getInstance().get(0).getRefereed().getClassname()); } @Test diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/oaf_crossref.xml b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/oaf_crossref.xml new file mode 100644 index 000000000..182820a08 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/oaf_crossref.xml @@ -0,0 +1,68 @@ + + +
+ doi_________::7f0f7807f17db50e5c2b5c452ccaf06d + doi_________::7f0f7807f17db50e5c2b5c452ccaf06d + 2020-08-06T07:04:09.62Z + + + + + + 2020-08-06T07:20:57.911Z + openaire____ +
+ + A case report of serious haemolysis in a glucose-6-phosphate dehydrogenase-deficient COVID-19 patient receiving hydroxychloroquine + Maillart, E. + Leemans, S. + Van Noten, H. + Vandergraesen, T. + Mahadeb, B. + Salaouatchi, M. T. + De Bels, D. + Clevenbergh, P. + + http://dx.doi.org/10.1080/23744235.2020.1774644 + + Informa UK Limited + Crossref + Infectious Diseases + Microbiology (medical) + General Immunology and Microbiology + Infectious Diseases + General Medicine + journal-article + 0001 + 2020-06-04 + + OPEN + + + 10.1080/23744235.2020.1774644 + Infectious Diseases + + + + + file%3A%2F%2F%2Fsrv%2Fclaims%2Frecords%2Fpublication%2Fcrossref + + + + + + + false + false + 0.9 + + + + + \ No newline at end of file From 0e44b037a52558e20bbe418a5d313fc7fd8e966f Mon Sep 17 00:00:00 2001 From: Giambattista Bloisi Date: Tue, 17 Oct 2023 07:54:01 +0200 Subject: [PATCH 066/148] FIX: GroupEntitiesSparkJob deletes whole graph outputPath instead of its temporary folder --- .../java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java index 99981bf6a..f5c8eea19 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java @@ -71,7 +71,7 @@ public class GroupEntitiesSparkJob { conf, isSparkSessionManaged, spark -> { - HdfsSupport.remove(outputPath, spark.sparkContext().hadoopConfiguration()); + HdfsSupport.remove(checkpointPath, spark.sparkContext().hadoopConfiguration()); groupEntities(spark, graphInputPath, checkpointPath, outputPath, filterInvisible); }); } From d28b7085f6f13a644926302e47da314dcb22de5d Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 17 Oct 2023 11:09:31 +0200 Subject: [PATCH 067/148] more NPE checks --- .../common/vocabulary/VocabularyGroup.java | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/vocabulary/VocabularyGroup.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/vocabulary/VocabularyGroup.java index 4c1feac45..64b6f91af 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/vocabulary/VocabularyGroup.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/vocabulary/VocabularyGroup.java @@ -136,24 +136,21 @@ public class VocabularyGroup implements Serializable { } public Qualifier lookupTermBySynonym(final String vocId, final String syn) { - if (StringUtils.isBlank(vocId)) { - return OafMapperUtils.unknown("", ""); - } - - final Vocabulary vocabulary = vocs.get(vocId.toLowerCase()); - - return Optional - .ofNullable(vocabulary.getTerm(syn)) + return find(vocId) .map( - term -> OafMapperUtils - .qualifier(term.getId(), term.getName(), vocabulary.getId(), vocabulary.getName())) - .orElse( - Optional - .ofNullable(vocabulary.getTermBySynonym(syn)) + vocabulary -> Optional + .ofNullable(vocabulary.getTerm(syn)) .map( term -> OafMapperUtils .qualifier(term.getId(), term.getName(), vocabulary.getId(), vocabulary.getName())) - .orElse(null)); + .orElse( + Optional + .ofNullable(vocabulary.getTermBySynonym(syn)) + .map( + term -> OafMapperUtils + .qualifier(term.getId(), term.getName(), vocabulary.getId(), vocabulary.getName())) + .orElse(null))) + .orElse(null); } /** From b0fed1725edc8c000619906751ad46a105c9449b Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Thu, 19 Oct 2023 12:13:45 +0200 Subject: [PATCH 068/148] avoid NPEs --- .../oaf/utils/GraphCleaningFunctions.java | 10 ++++++ .../dnetlib/dhp/bulktag/SparkBulkTagJob.java | 7 ++-- .../PrepareResultCountrySet.java | 32 ++++++++++++++----- 3 files changed, 39 insertions(+), 10 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java index 324e3dd58..b4402a2fb 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java @@ -286,6 +286,12 @@ public class GraphCleaningFunctions extends CleaningFunctions { public static T cleanup(T value, VocabularyGroup vocs) { + if (Objects.isNull(value.getDataInfo())) { + final DataInfo d = new DataInfo(); + d.setDeletedbyinference(false); + value.setDataInfo(d); + } + if (value instanceof OafEntity) { OafEntity e = (OafEntity) value; @@ -305,6 +311,10 @@ public class GraphCleaningFunctions extends CleaningFunctions { } else if (value instanceof Result) { Result r = (Result) value; + if (Objects.isNull(r.getContext())) { + r.setContext(new ArrayList<>()); + } + if (Objects.nonNull(r.getFulltext()) && (ModelConstants.SOFTWARE_RESULTTYPE_CLASSID.equals(r.getResulttype().getClassid()) || ModelConstants.DATASET_RESULTTYPE_CLASSID.equals(r.getResulttype().getClassid()))) { diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java index 3186ed5c0..fc3882b73 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java @@ -25,6 +25,7 @@ import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.bulktag.community.*; import eu.dnetlib.dhp.schema.oaf.Datasource; import eu.dnetlib.dhp.schema.oaf.Result; +import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils; public class SparkBulkTagJob { @@ -170,10 +171,12 @@ public class SparkBulkTagJob { // TODO remove this hack as soon as the values fixed by this method will be provided as NON null private static MapFunction patchResult() { return r -> { - if (r.getDataInfo().getDeletedbyinference() == null) { + if (Objects.isNull(r.getDataInfo())) { + r.setDataInfo(OafMapperUtils.dataInfo(false, "", false, false, OafMapperUtils.unknown("", ""), "")); + } else if (r.getDataInfo().getDeletedbyinference() == null) { r.getDataInfo().setDeletedbyinference(false); } - if (r.getContext() == null) { + if (Objects.isNull(r.getContext())) { r.setContext(new ArrayList<>()); } return r; diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/countrypropagation/PrepareResultCountrySet.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/countrypropagation/PrepareResultCountrySet.java index 28b6f616d..184d24751 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/countrypropagation/PrepareResultCountrySet.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/countrypropagation/PrepareResultCountrySet.java @@ -5,10 +5,7 @@ import static eu.dnetlib.dhp.PropagationConstant.*; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; import org.apache.commons.io.IOUtils; @@ -88,14 +85,33 @@ public class PrepareResultCountrySet { // selects all the results non deleted by inference and non invisible Dataset result = readPath(spark, inputPath, resultClazz) .filter( - (FilterFunction) r -> !r.getDataInfo().getDeletedbyinference() && - !r.getDataInfo().getInvisible()); + (FilterFunction) r -> Optional + .ofNullable(r.getDataInfo()) + .map(dataInfo -> !dataInfo.getDeletedbyinference() && !dataInfo.getInvisible()) + .orElse(true)); // of the results collects the distinct keys for collected from (at the level of the result) and hosted by // and produces pairs resultId, key for each distinct key associated to the result result.flatMap((FlatMapFunction) r -> { - Set cfhb = r.getCollectedfrom().stream().map(cf -> cf.getKey()).collect(Collectors.toSet()); - cfhb.addAll(r.getInstance().stream().map(i -> i.getHostedby().getKey()).collect(Collectors.toSet())); + Set cfhb = Optional + .ofNullable(r.getCollectedfrom()) + .map(cf -> cf.stream().map(KeyValue::getKey).collect(Collectors.toSet())) + .orElse(new HashSet<>()); + cfhb + .addAll( + Optional + .ofNullable(r.getInstance()) + .map( + i -> i + .stream() + .map( + ii -> Optional + .ofNullable(ii.getHostedby()) + .map(KeyValue::getKey) + .orElse(null)) + .filter(Objects::nonNull) + .collect(Collectors.toSet())) + .orElse(new HashSet<>())); return cfhb .stream() .map(value -> EntityEntityRel.newInstance(r.getId(), value)) From 2b9d0416eca2b0b1fc5b9a2b7ed19ca0868ddd2c Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Thu, 19 Oct 2023 16:26:37 +0200 Subject: [PATCH 069/148] [graph raw] URL Validator to accept double slashes --- .../raw/AbstractMdRecordToOafMapper.java | 2 +- .../dnetlib/dhp/oa/graph/raw/MappersTest.java | 14 ++++ .../dnetlib/dhp/oa/graph/raw/idus_sevilla.xml | 65 +++++++++++++++++++ 3 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/idus_sevilla.xml diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java index b37e6a755..bca6a2aae 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java @@ -29,7 +29,7 @@ public abstract class AbstractMdRecordToOafMapper { protected final VocabularyGroup vocs; - protected static final UrlValidator URL_VALIDATOR = UrlValidator.getInstance(); + protected static final UrlValidator URL_VALIDATOR = new UrlValidator(UrlValidator.ALLOW_2_SLASHES); private final boolean invisible; diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java index b506d3a62..da7a890ee 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java @@ -797,6 +797,20 @@ class MappersTest { assertFalse(p_cleaned.getTitle().isEmpty()); } + @Test + void test_instance_url_validation() throws IOException { + final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("idus_sevilla.xml"))); + final List list = new OafToOafMapper(vocs, false, true).processMdRecord(xml); + + final Publication p = (Publication) list.get(0); + + assertNotNull(p.getInstance()); + assertFalse(p.getInstance().isEmpty()); + assertNotNull(p.getInstance().get(0).getUrl()); + assertFalse(p.getInstance().get(0).getUrl().isEmpty()); + assertEquals("https://idus.us.es/handle//11441/118940", p.getInstance().get(0).getUrl().get(0)); + } + @Test void testZenodo() throws IOException, DocumentException { final String xml = IOUtils.toString(Objects.requireNonNull(getClass().getResourceAsStream("odf_zenodo.xml"))); diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/idus_sevilla.xml b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/idus_sevilla.xml new file mode 100644 index 000000000..1bfa3c7c3 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/raw/idus_sevilla.xml @@ -0,0 +1,65 @@ + + +
+ od______3272::6a4d00217a024a46ce9697ce98b13c2a + oai:idus.us.es:11441/118940 + + + + + + 2021-08-20T12:32:32.826Z + 2023-07-04T15:47:55.397Z + od______3272 +
+ + El museo pictorico y escala optica : tomo I : theorica de la pintura en que se describe su origen ... y se aprueban con demonstraciomes mathematicas y filosoficas, sus mas radicales fundamentos + Palomino de Castro y Velasco, Antonio, 1653-1726 + Rovira y Brocandel, Hipólito, 1693-1765 + Palomino de Castro y Velasco, Antonio, 1653-1726 + 2021-08-12T08:59:53Z + 1715 + A 042(a)/063 + application/pdf + https://idus.us.es/handle//11441/118940 + spa + En Madrid : por Lucas Antonio de Bedmar ... : vendese en casa de Don Joseph de Villar y Villanueva, 1715 + info:eu-repo/semantics/book + info:eu-repo/semantics/publishedVersion + 0002 + 1715-01-01 + + opendoar____::3272 + OPEN + + + https://idus.us.es/handle//11441/118940 + + http://creativecommons.org/licenses/by-nc-nd/4.0/ + + + + + http%3A%2F%2Fidus.us.es%2Foai%2Fdriver + oai:idus.us.es:11441/118940 + 2021-08-12T08:59:54Z + http://www.openarchives.org/OAI/2.0/oai_dc/ + + + + false + false + 0.9 + + + + + \ No newline at end of file From 6b19dcee80cdc19b6acdd95e6b05e5a4093b3ba7 Mon Sep 17 00:00:00 2001 From: Serafeim Chatzopoulos Date: Thu, 19 Oct 2023 19:58:25 +0300 Subject: [PATCH 070/148] Add actionset creation for pubmed affiliations --- .../PrepareAffiliationRelations.java | 32 ++++++++++++------- .../input_actionset_parameter.json | 8 ++++- .../bipaffiliations/job.properties | 1 + .../bipaffiliations/oozie_app/workflow.xml | 7 +++- 4 files changed, 35 insertions(+), 13 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java index 603ad6339..cbfba30c5 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java @@ -12,6 +12,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.io.compress.GzipCodec; import org.apache.hadoop.mapred.SequenceFileOutputFormat; import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.sql.*; import org.apache.spark.sql.Dataset; @@ -58,10 +59,13 @@ public class PrepareAffiliationRelations implements Serializable { log.info("isSparkSessionManaged: {}", isSparkSessionManaged); final String inputPath = parser.get("inputPath"); - log.info("inputPath {}: ", inputPath); + log.info("inputPath: {}", inputPath); + + final String pubmedInputPath = parser.get("pubmedInputPath"); + log.info("pubmedInputPath: {}", pubmedInputPath); final String outputPath = parser.get("outputPath"); - log.info("outputPath {}: ", outputPath); + log.info("outputPath: {}", outputPath); SparkConf conf = new SparkConf(); @@ -70,12 +74,22 @@ public class PrepareAffiliationRelations implements Serializable { isSparkSessionManaged, spark -> { Constants.removeOutputDir(spark, outputPath); - prepareAffiliationRelations(spark, inputPath, outputPath); + + List collectedFromCrossref = OafMapperUtils.listKeyValues(ModelConstants.CROSSREF_ID, "Crossref"); + JavaPairRDD crossrefRelations = prepareAffiliationRelations(spark, inputPath, collectedFromCrossref); + + List collectedFromPubmed = OafMapperUtils.listKeyValues(ModelConstants.PUBMED_CENTRAL_ID, "Pubmed"); + JavaPairRDD pubmedRelations = prepareAffiliationRelations(spark, inputPath, collectedFromPubmed); + + crossrefRelations + .union(pubmedRelations) + .saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, GzipCodec.class); + }); } - private static void prepareAffiliationRelations(SparkSession spark, String inputPath, - String outputPath) { + private static JavaPairRDD prepareAffiliationRelations(SparkSession spark, String inputPath, + List collectedfrom) { // load and parse affiliation relations from HDFS Dataset df = spark @@ -92,7 +106,7 @@ public class PrepareAffiliationRelations implements Serializable { new Column("matching.Confidence").as("confidence")); // prepare action sets for affiliation relations - df + return df .toJavaRDD() .flatMap((FlatMapFunction) row -> { @@ -120,8 +134,6 @@ public class PrepareAffiliationRelations implements Serializable { qualifier, Double.toString(row.getAs("confidence"))); - List collectedfrom = OafMapperUtils.listKeyValues(ModelConstants.CROSSREF_ID, "Crossref"); - // return bi-directional relations return getAffiliationRelationPair(paperId, affId, collectedfrom, dataInfo).iterator(); @@ -129,9 +141,7 @@ public class PrepareAffiliationRelations implements Serializable { .map(p -> new AtomicAction(Relation.class, p)) .mapToPair( aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()), - new Text(OBJECT_MAPPER.writeValueAsString(aa)))) - .saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, GzipCodec.class); - + new Text(OBJECT_MAPPER.writeValueAsString(aa)))); } private static List getAffiliationRelationPair(String paperId, String affId, List collectedfrom, diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/input_actionset_parameter.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/input_actionset_parameter.json index 7663a454b..96dcc3b32 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/input_actionset_parameter.json +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/input_actionset_parameter.json @@ -8,7 +8,13 @@ { "paramName": "ip", "paramLongName": "inputPath", - "paramDescription": "the URL from where to get the programme file", + "paramDescription": "the path to get the input data from Crossref", + "paramRequired": true + }, + { + "paramName": "pip", + "paramLongName": "pubmedInputPath", + "paramDescription": "the path to get the input data from Pubmed", "paramRequired": true }, { diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/job.properties b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/job.properties index d942e6772..fe3cbb633 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/job.properties +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/job.properties @@ -32,4 +32,5 @@ spark2SqlQueryExecutionListeners=com.cloudera.spark.lineage.NavigatorQueryListen oozie.wf.application.path=${oozieTopWfApplicationPath} inputPath=/data/bip-affiliations/data.json +pubmedInputPath=/data/bip-affiiations/pubmed-data.json outputPath=/tmp/crossref-affiliations-output-v5 diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/oozie_app/workflow.xml index 9930cfe17..c0a6bfc52 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/oozie_app/workflow.xml @@ -3,7 +3,11 @@ inputPath - the path where to find the inferred affiliation relations + the path where to find the inferred affiliation relations from Crossref + + + pubmedInputPath + the path where to find the inferred affiliation relations from Pubmed outputPath @@ -97,6 +101,7 @@ --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} --inputPath${inputPath} + --pubmedInputPath${pubmedInputPath} --outputPath${outputPath} From a4214ced1e0b71b4af87cf4cdddf05b30f92afb1 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 20 Oct 2023 10:14:20 +0200 Subject: [PATCH 071/148] fixing issue on propagation organization. added --config to workflow definition. added oozie_app to communtiy project --- .../PrepareResultCommunitySet.java | 4 +- .../SparkResultToCommunityFromProject.java | 2 - .../oozie_app/workflow.xml | 4 +- ...t_preparecommunitytoresult_parameters.json | 7 +- .../oozie_app/config-default.xml | 58 +++++++ .../oozie_app/workflow.xml | 144 ++++++++++++++++++ 6 files changed, 207 insertions(+), 12 deletions(-) create mode 100644 dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/oozie_app/config-default.xml create mode 100644 dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/oozie_app/workflow.xml diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java index 883f5ca86..c47075272 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java @@ -109,12 +109,12 @@ public class PrepareResultCommunitySet { }); }); - if(cl.size() == 0) + if (cl.size() == 0) return null; rpl.setCommunityList(cl); return rpl; }, Encoders.bean(ResultProjectList.class)) - .filter(Objects::nonNull) + .filter(Objects::nonNull) .write() .mode(SaveMode.Overwrite) .option("compression", "gzip") diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java index daef6a317..90c30f812 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java @@ -62,10 +62,8 @@ public class SparkResultToCommunityFromProject implements Serializable { final String possibleupdatespath = parser.get("preparedInfoPath"); log.info("preparedInfoPath: {}", possibleupdatespath); - SparkConf conf = new SparkConf(); - runWithSparkSession( conf, isSparkSessionManaged, diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml index 7a5fd7434..d5d75fbc1 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml @@ -92,7 +92,7 @@ --executor-cores=6 --executor-memory=5G - --spark.executor.memoryOverhead=3g + --conf spark.executor.memoryOverhead=3g --conf spark.sql.shuffle.partitions=3284 --driver-memory=${sparkDriverMemory} --conf spark.extraListeners=${spark2ExtraListeners} @@ -121,7 +121,7 @@ --executor-cores=6 --executor-memory=5G - --spark.executor.memoryOverhead=3g + --conf spark.executor.memoryOverhead=3g --conf spark.sql.shuffle.partitions=3284 --driver-memory=${sparkDriverMemory} --conf spark.extraListeners=${spark2ExtraListeners} diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/input_preparecommunitytoresult_parameters.json b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/input_preparecommunitytoresult_parameters.json index 8b6291e5d..9a50c79fa 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/input_preparecommunitytoresult_parameters.json +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/input_preparecommunitytoresult_parameters.json @@ -5,12 +5,7 @@ "paramDescription": "the path of the sequencial file to read", "paramRequired": true }, - { - "paramName":"h", - "paramLongName":"hive_metastore_uris", - "paramDescription": "the hive metastore uris", - "paramRequired": true - }, + { "paramName": "ssm", "paramLongName": "isSparkSessionManaged", diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/oozie_app/config-default.xml b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/oozie_app/config-default.xml new file mode 100644 index 000000000..2744ea92b --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/oozie_app/config-default.xml @@ -0,0 +1,58 @@ + + + jobTracker + yarnRM + + + nameNode + hdfs://nameservice1 + + + oozie.use.system.libpath + true + + + oozie.action.sharelib.for.spark + spark2 + + + hive_metastore_uris + thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083 + + + spark2YarnHistoryServerAddress + http://iis-cdh5-test-gw.ocean.icm.edu.pl:18089 + + + spark2EventLogDir + /user/spark/spark2ApplicationHistory + + + spark2ExtraListeners + com.cloudera.spark.lineage.NavigatorAppListener + + + spark2SqlQueryExecutionListeners + com.cloudera.spark.lineage.NavigatorQueryListener + + + sparkExecutorNumber + 4 + + + sparkDriverMemory + 15G + + + sparkExecutorMemory + 6G + + + sparkExecutorCores + 1 + + + spark2MaxExecutors + 50 + + \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/oozie_app/workflow.xml b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/oozie_app/workflow.xml new file mode 100644 index 000000000..21cc2d887 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/oozie_app/workflow.xml @@ -0,0 +1,144 @@ + + + + sourcePath + the source path + + + + outputPath + the output path + + + + + ${jobTracker} + ${nameNode} + + + oozie.action.sharelib.for.spark + ${oozieActionShareLibForSpark2} + + + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + + + + + + + + + + + + + + + + + ${nameNode}/${sourcePath}/relation + ${nameNode}/${outputPath}/relation + + + + + + + + ${nameNode}/${sourcePath}/organization + ${nameNode}/${outputPath}/organization + + + + + + + + ${nameNode}/${sourcePath}/project + ${nameNode}/${outputPath}/project + + + + + + + + ${nameNode}/${sourcePath}/datasource + ${nameNode}/${outputPath}/datasource + + + + + + + + + + yarn + cluster + Prepare-Community-Result-Organization + eu.dnetlib.dhp.resulttocommunityfromproject.PrepareResultCommunitySet + dhp-enrichment-${projectVersion}.jar + + --executor-cores=6 + --executor-memory=5G + --conf spark.executor.memoryOverhead=3g + --conf spark.sql.shuffle.partitions=3284 + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + + --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors} + + --sourcePath${sourcePath}/relation + --outputPath${workingDir}/preparedInfo/resultCommunityList + --production${production} + + + + + + + + yarn + cluster + community2resultfromproject + eu.dnetlib.dhp.resulttocommunityfromproject.SparkResultToCommunityFromProject + dhp-enrichment-${projectVersion}.jar + + --executor-cores=6 + --executor-memory=5G + --conf spark.executor.memoryOverhead=3g + --conf spark.sql.shuffle.partitions=3284 + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors} + + --preparedInfoPath${workingDir}/preparedInfo/resultCommunityList + --sourcePath${sourcePath}/ + --outputPath${outputPath}/ + + + + + + + + + + \ No newline at end of file From aad5982bf171d4504de660afa1a4350bd7e761f9 Mon Sep 17 00:00:00 2001 From: Serafeim Chatzopoulos Date: Fri, 20 Oct 2023 12:48:21 +0300 Subject: [PATCH 072/148] Change the description of the workflow --- .../PrepareAffiliationRelations.java | 22 ++++++++++++------- .../bipaffiliations/oozie_app/workflow.xml | 2 +- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java index cbfba30c5..18d98be54 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java @@ -75,21 +75,27 @@ public class PrepareAffiliationRelations implements Serializable { spark -> { Constants.removeOutputDir(spark, outputPath); - List collectedFromCrossref = OafMapperUtils.listKeyValues(ModelConstants.CROSSREF_ID, "Crossref"); - JavaPairRDD crossrefRelations = prepareAffiliationRelations(spark, inputPath, collectedFromCrossref); + List collectedFromCrossref = OafMapperUtils + .listKeyValues(ModelConstants.CROSSREF_ID, "Crossref"); + JavaPairRDD crossrefRelations = prepareAffiliationRelations( + spark, inputPath, collectedFromCrossref); - List collectedFromPubmed = OafMapperUtils.listKeyValues(ModelConstants.PUBMED_CENTRAL_ID, "Pubmed"); - JavaPairRDD pubmedRelations = prepareAffiliationRelations(spark, inputPath, collectedFromPubmed); + List collectedFromPubmed = OafMapperUtils + .listKeyValues(ModelConstants.PUBMED_CENTRAL_ID, "Pubmed"); + JavaPairRDD pubmedRelations = prepareAffiliationRelations( + spark, inputPath, collectedFromPubmed); crossrefRelations - .union(pubmedRelations) - .saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, GzipCodec.class); + .union(pubmedRelations) + .saveAsHadoopFile( + outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, GzipCodec.class); }); } - private static JavaPairRDD prepareAffiliationRelations(SparkSession spark, String inputPath, - List collectedfrom) { + private static JavaPairRDD prepareAffiliationRelations(SparkSession spark, + String inputPath, + List collectedfrom) { // load and parse affiliation relations from HDFS Dataset df = spark diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/oozie_app/workflow.xml index c0a6bfc52..e3fdddfd6 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/oozie_app/workflow.xml @@ -87,7 +87,7 @@ yarn cluster - Produces the atomic action with the inferred by BIP! affiliation relations from Crossref + Produces the atomic action with the inferred by BIP! affiliation relations (from Crossref and Pubmed) eu.dnetlib.dhp.actionmanager.bipaffiliations.PrepareAffiliationRelations dhp-aggregation-${projectVersion}.jar From 69dac916590e98327ba7f7a13c1946bc0423b3ff Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 20 Oct 2023 15:45:52 +0200 Subject: [PATCH 073/148] adding the new code to use the API instead of the Information Service --- .../eu/dnetlib/dhp/api/QueryCommunityAPI.java | 83 +++++++++ .../main/java/eu/dnetlib/dhp/api/Utils.java | 169 ++++++++++++++++++ .../api/model/CommunityContentprovider.java | 43 +++++ .../dhp/api/model/CommunityEntityMap.java | 21 +++ .../dnetlib/dhp/api/model/CommunityModel.java | 108 +++++++++++ .../dhp/api/model/CommunitySummary.java | 15 ++ .../dnetlib/dhp/api/model/ContentModel.java | 51 ++++++ .../dnetlib/dhp/api/model/DatasourceList.java | 13 ++ .../dhp/api/model/OrganizationList.java | 16 ++ .../dnetlib/dhp/api/model/ProjectModel.java | 24 +++ .../community/QueryInformationSystem.java | 34 ---- .../dhp/api/QueryCommunityAPITest.java | 9 + 12 files changed, 552 insertions(+), 34 deletions(-) create mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java create mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java create mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityContentprovider.java create mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityEntityMap.java create mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityModel.java create mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunitySummary.java create mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/ContentModel.java create mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/DatasourceList.java create mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/OrganizationList.java create mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/ProjectModel.java delete mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/QueryInformationSystem.java create mode 100644 dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/api/QueryCommunityAPITest.java diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java new file mode 100644 index 000000000..262ca0290 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java @@ -0,0 +1,83 @@ + +package eu.dnetlib.dhp.api; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; + +import org.jetbrains.annotations.NotNull; + +/** + * @author miriam.baglioni + * @Date 06/10/23 + */ +public class QueryCommunityAPI { + private static final String PRODUCTION_BASE_URL = "https://services.openaire.eu/openaire/"; + private static final String BETA_BASE_URL = "https://beta.services.openaire.eu/openaire/"; + + private static String get(String geturl) throws IOException { + URL url = new URL(geturl); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setDoOutput(true); + conn.setRequestMethod("GET"); + + int responseCode = conn.getResponseCode(); + String body = getBody(conn); + conn.disconnect(); + if (responseCode != HttpURLConnection.HTTP_OK) + throw new IOException("Unexpected code " + responseCode + body); + + return body; + } + + public static String communities(boolean production) throws IOException { + if (production) + return get(PRODUCTION_BASE_URL + "community/communities"); + return get(BETA_BASE_URL + "community/communities"); + } + + public static String community(String id, boolean production) throws IOException { + if (production) + return get(PRODUCTION_BASE_URL + "community/" + id); + return get(BETA_BASE_URL + "community/" + id); + } + + public static String communityDatasource(String id, boolean production) throws IOException { + if (production) + return get(PRODUCTION_BASE_URL + "community/" + id + "/contentproviders"); + return (BETA_BASE_URL + "community/" + id + "/contentproviders"); + + } + + public static String communityPropagationOrganization(String id, boolean production) throws IOException { + if (production) + return get(PRODUCTION_BASE_URL + "community/" + id + "/propagationOrganizations"); + return get(BETA_BASE_URL + "community/" + id + "/propagationOrganizations"); + } + + public static String communityProjects(String id, String page, String size, boolean production) throws IOException { + if (production) + return get(PRODUCTION_BASE_URL + "community/" + id + "/projects/" + page + "/" + size); + return get(BETA_BASE_URL + "community/" + id + "/projects/" + page + "/" + size); + } + + @NotNull + private static String getBody(HttpURLConnection conn) throws IOException { + String body = "{}"; + try (BufferedReader br = new BufferedReader( + new InputStreamReader(conn.getInputStream(), "utf-8"))) { + StringBuilder response = new StringBuilder(); + String responseLine = null; + while ((responseLine = br.readLine()) != null) { + response.append(responseLine.trim()); + } + + body = response.toString(); + + } + return body; + } + +} diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java new file mode 100644 index 000000000..43d5e7e98 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java @@ -0,0 +1,169 @@ + +package eu.dnetlib.dhp.api; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +import javax.management.Query; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.amazonaws.util.StringUtils; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Maps; + +import eu.dnetlib.dhp.api.model.*; +import eu.dnetlib.dhp.bulktag.community.Community; +import eu.dnetlib.dhp.bulktag.community.CommunityConfiguration; +import eu.dnetlib.dhp.bulktag.community.Provider; +import eu.dnetlib.dhp.bulktag.criteria.VerbResolver; +import eu.dnetlib.dhp.bulktag.criteria.VerbResolverFactory; +import eu.dnetlib.dhp.resulttocommunityfromorganization.SparkResultToCommunityFromOrganizationJob; + +/** + * @author miriam.baglioni + * @Date 09/10/23 + */ +public class Utils implements Serializable { + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final VerbResolver resolver = VerbResolverFactory.newInstance(); + + private static final Logger log = LoggerFactory.getLogger(Utils.class); + + public static CommunityConfiguration getCommunityConfiguration(boolean production) throws IOException { + final Map communities = Maps.newHashMap(); + List validCommunities = new ArrayList<>(); + getValidCommunities(production) + .forEach(community -> { + try { + CommunityModel cm = MAPPER + .readValue(QueryCommunityAPI.community(community.getId(), production), CommunityModel.class); + validCommunities.add(getCommunity(cm)); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + validCommunities.forEach(community -> { + try { + DatasourceList dl = MAPPER + .readValue( + QueryCommunityAPI.communityDatasource(community.getId(), production), DatasourceList.class); + community.setProviders(dl.stream().map(d -> { + if (d.getEnabled() == null || Boolean.FALSE.equals(d.getEnabled())) + return null; + Provider p = new Provider(); + p.setOpenaireId("10|" + d.getOpenaireId()); + p.setSelectionConstraints(d.getSelectioncriteria()); + if (p.getSelectionConstraints() != null) + p.getSelectionConstraints().setSelection(resolver); + return p; + }) + .filter(Objects::nonNull) + .collect(Collectors.toList())); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + validCommunities.forEach(community -> { + if (community.isValid()) + communities.put(community.getId(), community); + }); + return new CommunityConfiguration(communities); + } + + private static Community getCommunity(CommunityModel cm) { + Community c = new Community(); + c.setId(cm.getId()); + c.setZenodoCommunities(cm.getOtherZenodoCommunities()); + if (!StringUtils.isNullOrEmpty(cm.getZenodoCommunity())) + c.getZenodoCommunities().add(cm.getZenodoCommunity()); + c.setSubjects(cm.getSubjects()); + c.getSubjects().addAll(cm.getFos()); + c.getSubjects().addAll(cm.getSdg()); + if (cm.getAdvancedConstraints() != null) { + c.setConstraints(cm.getAdvancedConstraints()); + c.getConstraints().setSelection(resolver); + } + if (cm.getRemoveConstraints() != null) { + c.setRemoveConstraints(cm.getRemoveConstraints()); + c.getRemoveConstraints().setSelection(resolver); + } + return c; + } + + public static List getValidCommunities(boolean production) throws IOException { + return MAPPER + .readValue(QueryCommunityAPI.communities(production), CommunitySummary.class) + .stream() + .filter( + community -> !community.getStatus().equals("hidden") && + (community.getType().equals("ri") || community.getType().equals("community"))) + .collect(Collectors.toList()); + } + + /** + * it returns for each organization the list of associated communities + */ + public static CommunityEntityMap getCommunityOrganization(boolean production) throws IOException { + CommunityEntityMap organizationMap = new CommunityEntityMap(); + getValidCommunities(production) + .forEach(community -> { + String id = community.getId(); + try { + List associatedOrgs = MAPPER + .readValue( + QueryCommunityAPI.communityPropagationOrganization(id, production), OrganizationList.class); + associatedOrgs.forEach(o -> { + if (!organizationMap + .keySet() + .contains( + "20|" + o)) + organizationMap.put("20|" + o, new ArrayList<>()); + organizationMap.get("20|" + o).add(community.getId()); + }); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + return organizationMap; + } + + public static CommunityEntityMap getCommunityProjects(boolean production) throws IOException { + CommunityEntityMap projectMap = new CommunityEntityMap(); + getValidCommunities(production) + .forEach(community -> { + int page = -1; + int size = 100; + ContentModel cm = new ContentModel(); + do { + page++; + try { + cm = MAPPER + .readValue( + QueryCommunityAPI + .communityProjects( + community.getId(), String.valueOf(page), String.valueOf(size), production), + ContentModel.class); + if (cm.getContent().size() > 0) { + cm.getContent().forEach(p -> { + if (!projectMap.keySet().contains("40|" + p.getOpenaireId())) + projectMap.put("40|" + p.getOpenaireId(), new ArrayList<>()); + projectMap.get("40|" + p.getOpenaireId()).add(community.getId()); + }); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } while (!cm.getLast()); + }); + return projectMap; + } +} diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityContentprovider.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityContentprovider.java new file mode 100644 index 000000000..9fab5a80c --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityContentprovider.java @@ -0,0 +1,43 @@ + +package eu.dnetlib.dhp.api.model; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.google.gson.Gson; + +import eu.dnetlib.dhp.bulktag.community.SelectionConstraints; + +@JsonAutoDetect +@JsonIgnoreProperties(ignoreUnknown = true) +public class CommunityContentprovider { + private String openaireId; + private SelectionConstraints selectioncriteria; + + private String enabled; + + public String getEnabled() { + return enabled; + } + + public void setEnabled(String enabled) { + this.enabled = enabled; + } + + public String getOpenaireId() { + return openaireId; + } + + public void setOpenaireId(final String openaireId) { + this.openaireId = openaireId; + } + + public SelectionConstraints getSelectioncriteria() { + + return this.selectioncriteria; + } + + public void setSelectioncriteria(SelectionConstraints selectioncriteria) { + this.selectioncriteria = selectioncriteria; + + } +} diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityEntityMap.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityEntityMap.java new file mode 100644 index 000000000..ca3eb2857 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityEntityMap.java @@ -0,0 +1,21 @@ + +package eu.dnetlib.dhp.api.model; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +public class CommunityEntityMap extends HashMap> { + + public CommunityEntityMap() { + super(); + } + + public List get(String key) { + + if (super.get(key) == null) { + return new ArrayList<>(); + } + return super.get(key); + } +} diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityModel.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityModel.java new file mode 100644 index 000000000..745e7efc2 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunityModel.java @@ -0,0 +1,108 @@ + +package eu.dnetlib.dhp.api.model; + +import java.io.Serializable; +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +import eu.dnetlib.dhp.bulktag.community.SelectionConstraints; + +/** + * @author miriam.baglioni + * @Date 06/10/23 + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class CommunityModel implements Serializable { + private String id; + private String type; + private String status; + + private String zenodoCommunity; + private List subjects; + private List otherZenodoCommunities; + private List fos; + private List sdg; + private SelectionConstraints advancedConstraints; + private SelectionConstraints removeConstraints; + + public String getZenodoCommunity() { + return zenodoCommunity; + } + + public void setZenodoCommunity(String zenodoCommunity) { + this.zenodoCommunity = zenodoCommunity; + } + + public List getSubjects() { + return subjects; + } + + public void setSubjects(List subjects) { + this.subjects = subjects; + } + + public List getOtherZenodoCommunities() { + return otherZenodoCommunities; + } + + public void setOtherZenodoCommunities(List otherZenodoCommunities) { + this.otherZenodoCommunities = otherZenodoCommunities; + } + + public List getFos() { + return fos; + } + + public void setFos(List fos) { + this.fos = fos; + } + + public List getSdg() { + return sdg; + } + + public void setSdg(List sdg) { + this.sdg = sdg; + } + + public SelectionConstraints getRemoveConstraints() { + return removeConstraints; + } + + public void setRemoveConstraints(SelectionConstraints removeConstraints) { + this.removeConstraints = removeConstraints; + } + + public SelectionConstraints getAdvancedConstraints() { + return advancedConstraints; + } + + public void setAdvancedConstraints(SelectionConstraints advancedConstraints) { + this.advancedConstraints = advancedConstraints; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } +} diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunitySummary.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunitySummary.java new file mode 100644 index 000000000..a0541f7ee --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/CommunitySummary.java @@ -0,0 +1,15 @@ + +package eu.dnetlib.dhp.api.model; + +import java.io.Serializable; +import java.util.ArrayList; + +/** + * @author miriam.baglioni + * @Date 06/10/23 + */ +public class CommunitySummary extends ArrayList implements Serializable { + public CommunitySummary() { + super(); + } +} diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/ContentModel.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/ContentModel.java new file mode 100644 index 000000000..469709f59 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/ContentModel.java @@ -0,0 +1,51 @@ + +package eu.dnetlib.dhp.api.model; + +import java.io.Serializable; +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** + * @author miriam.baglioni + * @Date 09/10/23 + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class ContentModel implements Serializable { + private List content; + private Integer totalPages; + private Boolean last; + private Integer number; + + public List getContent() { + return content; + } + + public void setContent(List content) { + this.content = content; + } + + public Integer getTotalPages() { + return totalPages; + } + + public void setTotalPages(Integer totalPages) { + this.totalPages = totalPages; + } + + public Boolean getLast() { + return last; + } + + public void setLast(Boolean last) { + this.last = last; + } + + public Integer getNumber() { + return number; + } + + public void setNumber(Integer number) { + this.number = number; + } +} diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/DatasourceList.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/DatasourceList.java new file mode 100644 index 000000000..30d0241c3 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/DatasourceList.java @@ -0,0 +1,13 @@ + +package eu.dnetlib.dhp.api.model; + +import java.io.Serializable; +import java.util.ArrayList; + +import eu.dnetlib.dhp.api.model.CommunityContentprovider; + +public class DatasourceList extends ArrayList implements Serializable { + public DatasourceList() { + super(); + } +} diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/OrganizationList.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/OrganizationList.java new file mode 100644 index 000000000..3c81ad179 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/OrganizationList.java @@ -0,0 +1,16 @@ + +package eu.dnetlib.dhp.api.model; + +import java.io.Serializable; +import java.util.ArrayList; + +/** + * @author miriam.baglioni + * @Date 09/10/23 + */ +public class OrganizationList extends ArrayList implements Serializable { + + public OrganizationList() { + super(); + } +} diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/ProjectModel.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/ProjectModel.java new file mode 100644 index 000000000..3495d6a63 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/model/ProjectModel.java @@ -0,0 +1,24 @@ + +package eu.dnetlib.dhp.api.model; + +import java.io.Serializable; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** + * @author miriam.baglioni + * @Date 09/10/23 + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class ProjectModel implements Serializable { + + private String openaireId; + + public String getOpenaireId() { + return openaireId; + } + + public void setOpenaireId(String openaireId) { + this.openaireId = openaireId; + } +} diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/QueryInformationSystem.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/QueryInformationSystem.java deleted file mode 100644 index 5fe3cf81f..000000000 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/QueryInformationSystem.java +++ /dev/null @@ -1,34 +0,0 @@ - -package eu.dnetlib.dhp.bulktag.community; - -import java.io.IOException; -import java.util.List; - -import org.apache.commons.io.IOUtils; -import org.dom4j.DocumentException; -import org.xml.sax.SAXException; - -import com.google.common.base.Joiner; - -import eu.dnetlib.dhp.utils.ISLookupClientFactory; -import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; -import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; - -public class QueryInformationSystem { - - public static CommunityConfiguration getCommunityConfiguration(final String isLookupUrl) - throws ISLookUpException, DocumentException, SAXException, IOException { - ISLookUpService isLookUp = ISLookupClientFactory.getLookUpService(isLookupUrl); - final List res = isLookUp - .quickSearchProfile( - IOUtils - .toString( - QueryInformationSystem.class - .getResourceAsStream( - "/eu/dnetlib/dhp/bulktag/query.xq"))); - - final String xmlConf = "" + Joiner.on(" ").join(res) + ""; - - return CommunityConfigurationFactory.newInstance(xmlConf); - } -} diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/api/QueryCommunityAPITest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/api/QueryCommunityAPITest.java new file mode 100644 index 000000000..03084301e --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/api/QueryCommunityAPITest.java @@ -0,0 +1,9 @@ +package eu.dnetlib.dhp.api;/** + + * @author miriam.baglioni + + * @Date 20/10/23 + + */ +public class QueryCommunityAPITest { +} From 18bfff8af303ae1ecc4e5eb2d5118d64b3d2e56a Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 20 Oct 2023 15:47:03 +0200 Subject: [PATCH 074/148] adding test classes and modifying test for bulktag --- .../dhp/api/QueryCommunityAPITest.java | 115 +++++++++- .../dnetlib/dhp/bulktag/BulkTagJobTest.java | 215 ++++++++++-------- .../CommunityConfigurationFactoryTest.java | 2 +- 3 files changed, 236 insertions(+), 96 deletions(-) diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/api/QueryCommunityAPITest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/api/QueryCommunityAPITest.java index 03084301e..0cdf0f50b 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/api/QueryCommunityAPITest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/api/QueryCommunityAPITest.java @@ -1,9 +1,116 @@ -package eu.dnetlib.dhp.api;/** +package eu.dnetlib.dhp.api; + + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.api.model.CommunityEntityMap; +import eu.dnetlib.dhp.api.model.CommunityModel; +import eu.dnetlib.dhp.api.model.CommunitySummary; +import eu.dnetlib.dhp.api.model.DatasourceList; +import eu.dnetlib.dhp.bulktag.community.Community; +import eu.dnetlib.dhp.bulktag.community.CommunityConfiguration; + +/** * @author miriam.baglioni - - * @Date 20/10/23 - + * @Date 06/10/23 */ public class QueryCommunityAPITest { + + @Test + void communityList() throws Exception { + String body = QueryCommunityAPI.communities(true); + new ObjectMapper() + .readValue(body, CommunitySummary.class) + .forEach(p -> { + try { + System.out.println(new ObjectMapper().writeValueAsString(p)); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }); + } + + @Test + void community() throws Exception { + String id = "dh-ch"; + String body = QueryCommunityAPI.community(id, true); + System.out + .println( + new ObjectMapper() + .writeValueAsString( + new ObjectMapper() + .readValue(body, CommunityModel.class))); + } + + @Test + void communityDatasource() throws Exception { + String id = "dh-ch"; + String body = QueryCommunityAPI.communityDatasource(id, true); + new ObjectMapper() + .readValue(body, DatasourceList.class) + .forEach(ds -> { + try { + System.out.println(new ObjectMapper().writeValueAsString(ds)); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }); + ; + } + + @Test + void validCommunities() throws Exception { + CommunityConfiguration cc = Utils.getCommunityConfiguration(true); + System.out.println(cc.getCommunities().keySet()); + Community community = cc.getCommunities().get("aurora"); + Assertions.assertEquals(0, community.getSubjects().size()); + Assertions.assertEquals(null, community.getConstraints()); + Assertions.assertEquals(null, community.getRemoveConstraints()); + Assertions.assertEquals(2, community.getZenodoCommunities().size()); + Assertions + .assertTrue( + community.getZenodoCommunities().stream().anyMatch(c -> c.equals("aurora-universities-network"))); + Assertions + .assertTrue(community.getZenodoCommunities().stream().anyMatch(c -> c.equals("university-of-innsbruck"))); + Assertions.assertEquals(35, community.getProviders().size()); + Assertions + .assertEquals( + 35, community.getProviders().stream().filter(p -> p.getSelectionConstraints() == null).count()); + + } + + @Test + void eutopiaCommunityConfiguration() throws Exception { + CommunityConfiguration cc = Utils.getCommunityConfiguration(true); + System.out.println(cc.getCommunities().keySet()); + Community community = cc.getCommunities().get("eutopia"); + community.getProviders().forEach(p -> System.out.println(p.getOpenaireId())); + } + + @Test + void getCommunityProjects() throws Exception { + CommunityEntityMap projectMap = Utils.getCommunityProjects(true); + + Assertions + .assertTrue( + projectMap + .keySet() + .stream() + .allMatch(k -> k.startsWith("40|"))); + + System.out.println(projectMap); + } + + @Test + void getCommunityOrganizations() throws Exception { + CommunityEntityMap organizationMap = Utils.getCommunityOrganization(true); + Assertions.assertTrue(organizationMap.keySet().stream().allMatch(k -> k.startsWith("20|"))); + + } + } diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java index 660a55472..7cbbcaafb 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java @@ -6,6 +6,7 @@ import static eu.dnetlib.dhp.bulktag.community.TaggingConstants.ZENODO_COMMUNITY import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.HashMap; import java.util.List; import org.apache.commons.io.FileUtils; @@ -98,14 +99,11 @@ public class BulkTagJobTest { SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", - getClass().getResource("/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates").getPath(), + getClass().getResource("/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates/").getPath(), "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Dataset", - "-outputPath", workingDir.toString() + "/dataset", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + "-outputPath", workingDir.toString() + "/", "-pathMap", pathMap }); @@ -133,19 +131,16 @@ public class BulkTagJobTest { @Test void bulktagBySubjectNoPreviousContextTest() throws Exception { final String sourcePath = getClass() - .getResource("/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/nocontext") + .getResource("/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/nocontext/") .getPath(); final String pathMap = BulkTagJobTest.pathMap; SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Dataset", - "-outputPath", workingDir.toString() + "/dataset", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + "-outputPath", workingDir.toString() + "/", "-pathMap", pathMap }); @@ -230,19 +225,19 @@ public class BulkTagJobTest { void bulktagBySubjectPreviousContextNoProvenanceTest() throws Exception { final String sourcePath = getClass() .getResource( - "/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/contextnoprovenance") + "/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject/contextnoprovenance/") .getPath(); final String pathMap = BulkTagJobTest.pathMap; SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Dataset", - "-outputPath", workingDir.toString() + "/dataset", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -311,18 +306,18 @@ public class BulkTagJobTest { @Test void bulktagByDatasourceTest() throws Exception { final String sourcePath = getClass() - .getResource("/eu/dnetlib/dhp/bulktag/sample/publication/update_datasource") + .getResource("/eu/dnetlib/dhp/bulktag/sample/publication/update_datasource/") .getPath(); SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Publication", - "-outputPath", workingDir.toString() + "/publication", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -384,25 +379,25 @@ public class BulkTagJobTest { void bulktagByZenodoCommunityTest() throws Exception { final String sourcePath = getClass() .getResource( - "/eu/dnetlib/dhp/bulktag/sample/otherresearchproduct/update_zenodocommunity") + "/eu/dnetlib/dhp/bulktag/sample/otherresearchproduct/update_zenodocommunity/") .getPath(); SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.OtherResearchProduct", - "-outputPath", workingDir.toString() + "/orp", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); JavaRDD tmp = sc - .textFile(workingDir.toString() + "/orp") + .textFile(workingDir.toString() + "/otherresearchproduct") .map(item -> OBJECT_MAPPER.readValue(item, OtherResearchProduct.class)); Assertions.assertEquals(10, tmp.count()); @@ -505,18 +500,18 @@ public class BulkTagJobTest { @Test void bulktagBySubjectDatasourceTest() throws Exception { final String sourcePath = getClass() - .getResource("/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject_datasource") + .getResource("/eu/dnetlib/dhp/bulktag/sample/dataset/update_subject_datasource/") .getPath(); SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Dataset", - "-outputPath", workingDir.toString() + "/dataset", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -636,14 +631,14 @@ public class BulkTagJobTest { SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", - getClass().getResource("/eu/dnetlib/dhp/bulktag/sample/software/software_10.json.gz").getPath(), + getClass().getResource("/eu/dnetlib/dhp/bulktag/sample/software/").getPath(), "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Software", - "-outputPath", workingDir.toString() + "/software", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -732,18 +727,18 @@ public class BulkTagJobTest { final String sourcePath = getClass() .getResource( - "/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints") + "/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints/") .getPath(); SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Dataset", - "-outputPath", workingDir.toString() + "/dataset", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -774,19 +769,19 @@ public class BulkTagJobTest { void bulkTagOtherJupyter() throws Exception { final String sourcePath = getClass() .getResource( - "/eu/dnetlib/dhp/eosctag/jupyter/otherresearchproduct") + "/eu/dnetlib/dhp/eosctag/jupyter/") .getPath(); SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.OtherResearchProduct", - "-outputPath", workingDir.toString() + "/otherresearchproduct", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -829,18 +824,18 @@ public class BulkTagJobTest { public void bulkTagDatasetJupyter() throws Exception { final String sourcePath = getClass() .getResource( - "/eu/dnetlib/dhp/eosctag/jupyter/dataset") + "/eu/dnetlib/dhp/eosctag/jupyter/") .getPath(); SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Dataset", - "-outputPath", workingDir.toString() + "/dataset", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -878,18 +873,18 @@ public class BulkTagJobTest { final String sourcePath = getClass() .getResource( - "/eu/dnetlib/dhp/eosctag/jupyter/software") + "/eu/dnetlib/dhp/eosctag/jupyter/") .getPath(); SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Software", - "-outputPath", workingDir.toString() + "/software", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -1096,18 +1091,18 @@ public class BulkTagJobTest { void galaxyOtherTest() throws Exception { final String sourcePath = getClass() .getResource( - "/eu/dnetlib/dhp/eosctag/galaxy/otherresearchproduct") + "/eu/dnetlib/dhp/eosctag/galaxy/") .getPath(); SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.OtherResearchProduct", - "-outputPath", workingDir.toString() + "/otherresearchproduct", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -1214,18 +1209,18 @@ public class BulkTagJobTest { void galaxySoftwareTest() throws Exception { final String sourcePath = getClass() .getResource( - "/eu/dnetlib/dhp/eosctag/galaxy/software") + "/eu/dnetlib/dhp/eosctag/galaxy/") .getPath(); SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Software", - "-outputPath", workingDir.toString() + "/software", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -1333,19 +1328,19 @@ public class BulkTagJobTest { void twitterDatasetTest() throws Exception { final String sourcePath = getClass() .getResource( - "/eu/dnetlib/dhp/eosctag/twitter/dataset") + "/eu/dnetlib/dhp/eosctag/twitter/") .getPath(); SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Dataset", - "-outputPath", workingDir.toString() + "/dataset", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -1373,19 +1368,19 @@ public class BulkTagJobTest { void twitterOtherTest() throws Exception { final String sourcePath = getClass() .getResource( - "/eu/dnetlib/dhp/eosctag/twitter/otherresearchproduct") + "/eu/dnetlib/dhp/eosctag/twitter/") .getPath(); SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.OtherResearchProduct", - "-outputPath", workingDir.toString() + "/otherresearchproduct", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -1418,19 +1413,19 @@ public class BulkTagJobTest { void twitterSoftwareTest() throws Exception { final String sourcePath = getClass() .getResource( - "/eu/dnetlib/dhp/eosctag/twitter/software") + "/eu/dnetlib/dhp/eosctag/twitter/") .getPath(); SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Software", - "-outputPath", workingDir.toString() + "/software", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -1455,19 +1450,19 @@ public class BulkTagJobTest { void EoscContextTagTest() throws Exception { final String sourcePath = getClass() .getResource( - "/eu/dnetlib/dhp/bulktag/eosc/dataset/dataset_10.json") + "/eu/dnetlib/dhp/bulktag/eosc/dataset/") .getPath(); SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", sourcePath, "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Dataset", - "-outputPath", workingDir.toString() + "/dataset", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); @@ -1533,16 +1528,16 @@ public class BulkTagJobTest { SparkBulkTagJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", getClass() - .getResource("/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints") + .getResource("/eu/dnetlib/dhp/bulktag/sample/dataset/update_datasourcewithconstraints/") .getPath(), "-taggingConf", taggingConf, - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Dataset", - "-outputPath", workingDir.toString() + "/dataset", - "-isLookUpUrl", MOCK_IS_LOOK_UP_URL, + + "-outputPath", workingDir.toString() + "/", + "-pathMap", pathMap }); final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); @@ -1568,4 +1563,42 @@ public class BulkTagJobTest { } + @Test + void newConfTest() throws Exception { + final String pathMap = BulkTagJobTest.pathMap; + SparkBulkTagJob + .main( + new String[] { + + "-isSparkSessionManaged", Boolean.FALSE.toString(), + "-sourcePath", + getClass().getResource("/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates/").getPath(), + "-taggingConf", taggingConf, + + "-outputPath", workingDir.toString() + "/", + "-production", Boolean.TRUE.toString(), + "-pathMap", pathMap + }); + + final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .textFile(workingDir.toString() + "/dataset") + .map(item -> OBJECT_MAPPER.readValue(item, Dataset.class)); + + Assertions.assertEquals(10, tmp.count()); + org.apache.spark.sql.Dataset verificationDataset = spark + .createDataset(tmp.rdd(), Encoders.bean(Dataset.class)); + + verificationDataset.createOrReplaceTempView("dataset"); + + String query = "select id, MyT.id community " + + "from dataset " + + "lateral view explode(context) c as MyT " + + "lateral view explode(MyT.datainfo) d as MyD " + + "where MyD.inferenceprovenance = 'bulktagging'"; + + Assertions.assertEquals(0, spark.sql(query).count()); + } + } diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/CommunityConfigurationFactoryTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/CommunityConfigurationFactoryTest.java index c8fd62c8e..5f0b1d7f1 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/CommunityConfigurationFactoryTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/CommunityConfigurationFactoryTest.java @@ -47,7 +47,7 @@ class CommunityConfigurationFactoryTest { sc.setVerb("not_contains"); sc.setField("contributor"); sc.setValue("DARIAH"); - sc.setSelection(resolver.getSelectionCriteria(sc.getVerb(), sc.getValue())); + sc.setSelection(resolver);// .getSelectionCriteria(sc.getVerb(), sc.getValue())); String metadata = "This work has been partially supported by DARIAH-EU infrastructure"; Assertions.assertFalse(sc.verifyCriteria(metadata)); } From 34358afe756d2ea602a1aa7e58969b2adbb204cf Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 20 Oct 2023 15:48:27 +0200 Subject: [PATCH 075/148] modified resource file, workflow anf default-config. Add 3g of memory Overhead and specified the shuffle partition in the wf confiduration. Removed the multiple instantiation in the wf because of different implementation of the spark job --- .../dhp/bulktag/input_bulkTag_parameters.json | 28 ++--- .../dhp/bulktag/oozie_app/config-default.xml | 10 +- .../dhp/bulktag/oozie_app/workflow.xml | 108 ++---------------- 3 files changed, 25 insertions(+), 121 deletions(-) diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/input_bulkTag_parameters.json b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/input_bulkTag_parameters.json index a8be7c32e..dbe2d088f 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/input_bulkTag_parameters.json +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/input_bulkTag_parameters.json @@ -1,10 +1,5 @@ [ - { - "paramName":"is", - "paramLongName":"isLookUpUrl", - "paramDescription": "URL of the isLookUp Service", - "paramRequired": true - }, + { "paramName":"s", "paramLongName":"sourcePath", @@ -17,12 +12,7 @@ "paramDescription": "the json path associated to each selection field", "paramRequired": true }, - { - "paramName":"tn", - "paramLongName":"resultTableName", - "paramDescription": "the name of the result table we are currently working on", - "paramRequired": true - }, + { "paramName": "out", "paramLongName": "outputPath", @@ -35,17 +25,19 @@ "paramDescription": "true if the spark session is managed, false otherwise", "paramRequired": false }, - { - "paramName": "test", - "paramLongName": "isTest", - "paramDescription": "Parameter intended for testing purposes only. True if the reun is relatesd to a test and so the taggingConf parameter should be loaded", - "paramRequired": false - }, + { "paramName": "tg", "paramLongName": "taggingConf", "paramDescription": "this parameter is intended for testing purposes only. It is a possible tagging configuration obtained via the XQUERY. Intended to be removed", "paramRequired": false + }, + + { + "paramName": "p", + "paramLongName": "production", + "paramDescription": "this parameter is intended for testing purposes only. It is a possible tagging configuration obtained via the XQUERY. Intended to be removed", + "paramRequired": true } ] \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/config-default.xml b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/config-default.xml index fe82ae194..c92f559f9 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/config-default.xml +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/config-default.xml @@ -45,10 +45,14 @@ sparkExecutorMemory - 6G + 5G - sparkExecutorCores - 1 + memoryOverhead + 3g + + + partitions + 3284 \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml index b868e4c72..4b81c58e4 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml @@ -4,10 +4,6 @@ sourcePath the source path - - isLookUpUrl - the isLookup service endpoint - pathMap the json path associated to each selection field @@ -102,16 +98,9 @@ - + - - - - - - - - + yarn-cluster cluster @@ -122,104 +111,23 @@ --num-executors=${sparkExecutorNumber} --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} + --conf spark.executor.memoryOverhead=${memeoryOverhead} + --conf spark.sql.shuffle.partitions=${partitions} --driver-memory=${sparkDriverMemory} --conf spark.extraListeners=${spark2ExtraListeners} --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --sourcePath${sourcePath}/publication - --resultTableNameeu.dnetlib.dhp.schema.oaf.Publication - --outputPath${outputPath}/publication + --sourcePath${sourcePath}/ + --outputPath${outputPath}/ --pathMap${pathMap} - --isLookUpUrl${isLookUpUrl} + --production${production} - + - - - yarn-cluster - cluster - bulkTagging-dataset - eu.dnetlib.dhp.bulktag.SparkBulkTagJob - dhp-enrichment-${projectVersion}.jar - - --num-executors=${sparkExecutorNumber} - --executor-memory=${sparkExecutorMemory} - --executor-cores=${sparkExecutorCores} - --driver-memory=${sparkDriverMemory} - --conf spark.extraListeners=${spark2ExtraListeners} - --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} - --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} - --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - - --sourcePath${sourcePath}/dataset - --resultTableNameeu.dnetlib.dhp.schema.oaf.Dataset - --outputPath${outputPath}/dataset - --pathMap${pathMap} - --isLookUpUrl${isLookUpUrl} - - - - - - - - yarn-cluster - cluster - bulkTagging-orp - eu.dnetlib.dhp.bulktag.SparkBulkTagJob - dhp-enrichment-${projectVersion}.jar - - --num-executors=${sparkExecutorNumber} - --executor-memory=${sparkExecutorMemory} - --executor-cores=${sparkExecutorCores} - --driver-memory=${sparkDriverMemory} - --conf spark.extraListeners=${spark2ExtraListeners} - --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} - --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} - --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - - --sourcePath${sourcePath}/otherresearchproduct - --resultTableNameeu.dnetlib.dhp.schema.oaf.OtherResearchProduct - --outputPath${outputPath}/otherresearchproduct - --pathMap${pathMap} - --isLookUpUrl${isLookUpUrl} - - - - - - - - yarn-cluster - cluster - bulkTagging-software - eu.dnetlib.dhp.bulktag.SparkBulkTagJob - dhp-enrichment-${projectVersion}.jar - - --num-executors=${sparkExecutorNumber} - --executor-memory=${sparkExecutorMemory} - --executor-cores=${sparkExecutorCores} - --driver-memory=${sparkDriverMemory} - --conf spark.extraListeners=${spark2ExtraListeners} - --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} - --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} - --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - - --sourcePath${sourcePath}/software - --resultTableNameeu.dnetlib.dhp.schema.oaf.Software - --outputPath${outputPath}/software - --pathMap${pathMap} - --isLookUpUrl${isLookUpUrl} - - - - - - From f206ff42d6ebec290ce3cf9c3222c10a02313ce6 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 20 Oct 2023 15:49:41 +0200 Subject: [PATCH 076/148] modified code to use the the API. Removing not needed parameters. Rewritten the code to exploit the parallel stream on the entity types --- .../dnetlib/dhp/bulktag/SparkBulkTagJob.java | 83 +++---- .../dhp/bulktag/community/Community.java | 9 +- .../community/CommunityConfiguration.java | 8 +- .../CommunityConfigurationFactory.java | 12 +- .../dhp/bulktag/community/Constraint.java | 17 +- .../dhp/bulktag/community/ResultTagger.java | 26 +- .../community/SelectionConstraints.java | 2 + .../PrepareResultCommunity2.java | 225 ++++++++++++++++++ 8 files changed, 301 insertions(+), 81 deletions(-) create mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunity2.java diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java index 3186ed5c0..68c740dd5 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java @@ -9,7 +9,6 @@ import java.util.*; import org.apache.commons.io.IOUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.function.FilterFunction; -import org.apache.spark.api.java.function.ForeachFunction; import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoders; @@ -21,8 +20,11 @@ import org.slf4j.LoggerFactory; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; +import eu.dnetlib.dhp.api.Utils; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.bulktag.community.*; +import eu.dnetlib.dhp.schema.common.EntityType; +import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.oaf.Datasource; import eu.dnetlib.dhp.schema.oaf.Result; @@ -53,50 +55,38 @@ public class SparkBulkTagJob { .orElse(Boolean.TRUE); log.info("isSparkSessionManaged: {}", isSparkSessionManaged); - Boolean isTest = Optional - .ofNullable(parser.get("isTest")) - .map(Boolean::valueOf) - .orElse(Boolean.FALSE); - log.info("isTest: {} ", isTest); - final String inputPath = parser.get("sourcePath"); log.info("inputPath: {}", inputPath); final String outputPath = parser.get("outputPath"); log.info("outputPath: {}", outputPath); + final boolean production = Boolean.valueOf(parser.get("production")); + log.info("production: {}", production); + ProtoMap protoMappingParams = new Gson().fromJson(parser.get("pathMap"), ProtoMap.class); log.info("pathMap: {}", new Gson().toJson(protoMappingParams)); - final String resultClassName = parser.get("resultTableName"); - log.info("resultTableName: {}", resultClassName); - - final Boolean saveGraph = Optional - .ofNullable(parser.get("saveGraph")) - .map(Boolean::valueOf) - .orElse(Boolean.TRUE); - log.info("saveGraph: {}", saveGraph); - - Class resultClazz = (Class) Class.forName(resultClassName); - SparkConf conf = new SparkConf(); CommunityConfiguration cc; - String taggingConf = parser.get("taggingConf"); + String taggingConf = Optional + .ofNullable(parser.get("taggingConf")) + .map(String::valueOf) + .orElse(null); - if (isTest) { + if (taggingConf != null) { cc = CommunityConfigurationFactory.newInstance(taggingConf); } else { - cc = QueryInformationSystem.getCommunityConfiguration(parser.get("isLookUpUrl")); + cc = Utils.getCommunityConfiguration(production); } runWithSparkSession( conf, isSparkSessionManaged, spark -> { - removeOutputDir(spark, outputPath); extendCommunityConfigurationForEOSC(spark, inputPath, cc); - execBulkTag(spark, inputPath, outputPath, protoMappingParams, resultClazz, cc); + execBulkTag(spark, inputPath, outputPath, protoMappingParams, cc); }); } @@ -105,10 +95,7 @@ public class SparkBulkTagJob { Dataset datasources = readPath( spark, inputPath - .substring( - 0, - inputPath.lastIndexOf("/")) - + "/datasource", + + "datasource", Datasource.class) .filter((FilterFunction) ds -> isOKDatasource(ds)) .map((MapFunction) ds -> ds.getId(), Encoders.STRING()); @@ -116,10 +103,10 @@ public class SparkBulkTagJob { Map>> dsm = cc.getEoscDatasourceMap(); for (String ds : datasources.collectAsList()) { - final String dsId = ds.substring(3); - if (!dsm.containsKey(dsId)) { + // final String dsId = ds.substring(3); + if (!dsm.containsKey(ds)) { ArrayList> eoscList = new ArrayList<>(); - dsm.put(dsId, eoscList); + dsm.put(ds, eoscList); } } @@ -141,22 +128,30 @@ public class SparkBulkTagJob { String inputPath, String outputPath, ProtoMap protoMappingParams, - Class resultClazz, CommunityConfiguration communityConfiguration) { - ResultTagger resultTagger = new ResultTagger(); - readPath(spark, inputPath, resultClazz) - .map(patchResult(), Encoders.bean(resultClazz)) - .filter(Objects::nonNull) - .map( - (MapFunction) value -> resultTagger - .enrichContextCriteria( - value, communityConfiguration, protoMappingParams), - Encoders.bean(resultClazz)) - .write() - .mode(SaveMode.Overwrite) - .option("compression", "gzip") - .json(outputPath); + ModelSupport.entityTypes + .keySet() + .parallelStream() + .filter(e -> ModelSupport.isResult(e)) + .forEach(e -> { + removeOutputDir(spark, outputPath + e.name()); + ResultTagger resultTagger = new ResultTagger(); + Class resultClazz = ModelSupport.entityTypes.get(e); + readPath(spark, inputPath + e.name(), resultClazz) + .map(patchResult(), Encoders.bean(resultClazz)) + .filter(Objects::nonNull) + .map( + (MapFunction) value -> resultTagger + .enrichContextCriteria( + value, communityConfiguration, protoMappingParams), + Encoders.bean(resultClazz)) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(outputPath + e.name()); + }); + } public static Dataset readPath( diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Community.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Community.java index b44376e22..9cd3a8f82 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Community.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Community.java @@ -4,6 +4,7 @@ package eu.dnetlib.dhp.bulktag.community; import java.io.Serializable; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import com.google.gson.Gson; @@ -13,7 +14,7 @@ public class Community implements Serializable { private String id; private List subjects = new ArrayList<>(); private List providers = new ArrayList<>(); - private List zenodoCommunities = new ArrayList<>(); + private List zenodoCommunities = new ArrayList<>(); private SelectionConstraints constraints = new SelectionConstraints(); private SelectionConstraints removeConstraints = new SelectionConstraints(); @@ -26,7 +27,7 @@ public class Community implements Serializable { return !getSubjects().isEmpty() || !getProviders().isEmpty() || !getZenodoCommunities().isEmpty() - || getConstraints().getCriteria() != null; + || (Optional.ofNullable(getConstraints()).isPresent() && getConstraints().getCriteria() != null); } public String getId() { @@ -53,11 +54,11 @@ public class Community implements Serializable { this.providers = providers; } - public List getZenodoCommunities() { + public List getZenodoCommunities() { return zenodoCommunities; } - public void setZenodoCommunities(List zenodoCommunities) { + public void setZenodoCommunities(List zenodoCommunities) { this.zenodoCommunities = zenodoCommunities; } diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfiguration.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfiguration.java index e061ccd5e..a658c7ff5 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfiguration.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfiguration.java @@ -81,7 +81,7 @@ public class CommunityConfiguration implements Serializable { this.removeConstraintsMap = removeConstraintsMap; } - CommunityConfiguration(final Map communities) { + public CommunityConfiguration(final Map communities) { this.communities = communities; init(); } @@ -117,10 +117,10 @@ public class CommunityConfiguration implements Serializable { add(d.getOpenaireId(), new Pair<>(id, d.getSelectionConstraints()), datasourceMap); } // get zenodo communities - for (ZenodoCommunity zc : c.getZenodoCommunities()) { + for (String zc : c.getZenodoCommunities()) { add( - zc.getZenodoCommunityId(), - new Pair<>(id, zc.getSelCriteria()), + zc, + new Pair<>(id, null), zenodocommunityMap); } selectionConstraintsMap.put(id, c.getConstraints()); diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfigurationFactory.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfigurationFactory.java index 7b9e03ef6..955ca3856 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfigurationFactory.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/CommunityConfigurationFactory.java @@ -143,16 +143,16 @@ public class CommunityConfigurationFactory { return providerList; } - private static List parseZenodoCommunities(final Node node) { + private static List parseZenodoCommunities(final Node node) { final List list = node.selectNodes("./zenodocommunities/zenodocommunity"); - final List zenodoCommunityList = new ArrayList<>(); + final List zenodoCommunityList = new ArrayList<>(); for (Node n : list) { - ZenodoCommunity zc = new ZenodoCommunity(); - zc.setZenodoCommunityId(n.selectSingleNode("./zenodoid").getText()); - zc.setSelCriteria(n.selectSingleNode("./selcriteria")); +// ZenodoCommunity zc = new ZenodoCommunity(); +// zc.setZenodoCommunityId(n.selectSingleNode("./zenodoid").getText()); +// zc.setSelCriteria(n.selectSingleNode("./selcriteria")); - zenodoCommunityList.add(zc); + zenodoCommunityList.add(n.selectSingleNode("./zenodoid").getText()); } log.info("size of the zenodo community list " + zenodoCommunityList.size()); diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Constraint.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Constraint.java index ed58cc14d..48d9be7cd 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Constraint.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Constraint.java @@ -4,6 +4,8 @@ package eu.dnetlib.dhp.bulktag.community; import java.io.Serializable; import java.lang.reflect.InvocationTargetException; +import org.apache.htrace.fasterxml.jackson.annotation.JsonIgnore; + import eu.dnetlib.dhp.bulktag.criteria.Selection; import eu.dnetlib.dhp.bulktag.criteria.VerbResolver; @@ -11,7 +13,8 @@ public class Constraint implements Serializable { private String verb; private String field; private String value; -// private String element; + + @JsonIgnore private Selection selection; public String getVerb() { @@ -38,10 +41,8 @@ public class Constraint implements Serializable { this.value = value; } - public void setSelection(Selection sel) { - selection = sel; - } + @JsonIgnore public void setSelection(VerbResolver resolver) throws InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { @@ -52,11 +53,5 @@ public class Constraint implements Serializable { return selection.apply(metadata); } -// public String getElement() { -// return element; -// } -// -// public void setElement(String element) { -// this.element = element; -// } + } diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/ResultTagger.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/ResultTagger.java index 5f62c10f4..3b231a52d 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/ResultTagger.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/ResultTagger.java @@ -85,16 +85,18 @@ public class ResultTagger implements Serializable { conf .getRemoveConstraintsMap() .keySet() - .forEach(communityId -> { - if (conf.getRemoveConstraintsMap().get(communityId).getCriteria() != null && - conf - .getRemoveConstraintsMap() - .get(communityId) - .getCriteria() - .stream() - .anyMatch(crit -> crit.verifyCriteria(param))) - removeCommunities.add(communityId); - }); + .forEach( + communityId -> { + if (conf.getRemoveConstraintsMap().keySet().contains(communityId) && + conf.getRemoveConstraintsMap().get(communityId).getCriteria() != null && + conf + .getRemoveConstraintsMap() + .get(communityId) + .getCriteria() + .stream() + .anyMatch(crit -> crit.verifyCriteria(param))) + removeCommunities.add(communityId); + }); // communities contains all the communities to be added as context for the result final Set communities = new HashSet<>(); @@ -124,10 +126,10 @@ public class ResultTagger implements Serializable { if (Objects.nonNull(result.getInstance())) { for (Instance i : result.getInstance()) { if (Objects.nonNull(i.getCollectedfrom()) && Objects.nonNull(i.getCollectedfrom().getKey())) { - collfrom.add(StringUtils.substringAfter(i.getCollectedfrom().getKey(), "|")); + collfrom.add(i.getCollectedfrom().getKey()); } if (Objects.nonNull(i.getHostedby()) && Objects.nonNull(i.getHostedby().getKey())) { - hostdby.add(StringUtils.substringAfter(i.getHostedby().getKey(), "|")); + hostdby.add(i.getHostedby().getKey()); } } diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/SelectionConstraints.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/SelectionConstraints.java index c7dcce812..57cc658fc 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/SelectionConstraints.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/SelectionConstraints.java @@ -7,11 +7,13 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import eu.dnetlib.dhp.bulktag.criteria.VerbResolver; +@JsonAutoDetect public class SelectionConstraints implements Serializable { private List criteria; diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunity2.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunity2.java new file mode 100644 index 000000000..96523c502 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunity2.java @@ -0,0 +1,225 @@ + +package eu.dnetlib.dhp.resulttocommunityfromorganization; + +/** + * @author miriam.baglioni + * @Date 16/10/23 + */ +/** + * @author miriam.baglioni + * @Date 16/10/23 + */ +import static eu.dnetlib.dhp.PropagationConstant.*; +import static eu.dnetlib.dhp.PropagationConstant.OBJECT_MAPPER; +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession; +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.util.*; +import java.util.stream.Collectors; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.io.ObjectWritable; +import org.apache.hadoop.io.compress.GzipCodec; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.*; +import org.apache.spark.api.java.function.FilterFunction; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.sql.*; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.Gson; + +import eu.dnetlib.dhp.api.Utils; +import eu.dnetlib.dhp.api.model.CommunityEntityMap; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.schema.common.ModelConstants; +import eu.dnetlib.dhp.schema.oaf.Relation; +import scala.Tuple2; + +public class PrepareResultCommunity2 { + + private static final Logger log = LoggerFactory + .getLogger(eu.dnetlib.dhp.resulttocommunityfromorganization.PrepareResultCommunitySet.class); + + public static void main(String[] args) throws Exception { + String jsonConfiguration = IOUtils + .toString( + eu.dnetlib.dhp.resulttocommunityfromorganization.PrepareResultCommunitySet.class + .getResourceAsStream( + "/eu/dnetlib/dhp/resulttocommunityfromorganization/input_preparecommunitytoresult_parameters.json")); + + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + parser.parseArgument(args); + + Boolean isSparkSessionManaged = isSparkSessionManaged(parser); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + String inputPath = parser.get("sourcePath"); + log.info("inputPath: {}", inputPath); + + final String outputPath = parser.get("outputPath"); + log.info("outputPath: {}", outputPath); + + final boolean production = Boolean.valueOf(parser.get("production")); + log.info("production: {}", production); + + final CommunityEntityMap organizationMap = Utils.getCommunityOrganization(production); + log.info("organizationMap: {}", new Gson().toJson(organizationMap)); + + SparkConf conf = new SparkConf(); + + runWithSparkSession( + conf, + isSparkSessionManaged, + spark -> { + removeOutputDir(spark, outputPath); + prepareInfo(spark, inputPath, outputPath, organizationMap); + }); + } + + private static void prepareInfo( + SparkSession spark, + String inputPath, + String outputPath, + CommunityEntityMap organizationMap) { + + final StructType structureSchema = new StructType() + .add( + "dataInfo", new StructType() + .add("deletedbyinference", DataTypes.BooleanType) + .add("invisible", DataTypes.BooleanType)) + .add("source", DataTypes.StringType) + .add("target", DataTypes.StringType) + .add("relClass", DataTypes.StringType); + + readPath(spark, inputPath, Relation.class) + .foreach((ForeachFunction) r -> System.out.println(new ObjectMapper().writeValueAsString(r))); + + Dataset resultOrganization = spark + .read() + .schema(Encoders.bean(Relation.class).schema()) + .json(inputPath) + .filter( + "dataInfo.deletedbyinference != true " + + "and relClass == '" + ModelConstants.HAS_AUTHOR_INSTITUTION + "'") + .select( + new Column("source").as("resultId"), + new Column("target").as("organizationId")); + + resultOrganization + .foreach((ForeachFunction) r -> System.out.println(new ObjectMapper().writeValueAsString(r))); + + Dataset organizationOrganization = spark + .read() + .schema(structureSchema) + .json(inputPath) + .filter( + "dataInfo.deletedbyinference != true " + + "and relClass == '" + ModelConstants.MERGES + "'") + .select( + new Column("source").as("masterOrganization"), + new Column("target").as("duplicateOrganization")); + + resultOrganization + .joinWith( + organizationOrganization, resultOrganization + .col("organizationId") + .equalTo(organizationOrganization.col("masterOrganization")), + "left") + .groupByKey( + (MapFunction, String>) t2 -> (String) t2._1().getAs("resultId"), Encoders.STRING()) + .mapGroups((MapGroupsFunction, ResultCommunityList>) (k, v) -> { + ResultCommunityList rcl = new ResultCommunityList(); + rcl.setResultId(k); + ArrayList cl = new ArrayList<>(); + Tuple2 first = v.next(); + cl.addAll(organizationMap.get(first._1().getAs("organizationId"))); + if (Optional.ofNullable(first._2()).isPresent()) { + cl.addAll(organizationMap.get(first._2().getAs(("duplicateOrganization")))); + } + v.forEachRemaining(o -> cl.addAll(organizationMap.get(o._2().getAs("duplicateOrganization")))); + if (cl.size() == 0) + return null; + rcl.setCommunityList(new ArrayList<>(cl.stream().distinct().collect(Collectors.toList()))); + return rcl; + }, Encoders.bean(ResultCommunityList.class)) + .filter(Objects::nonNull) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(outputPath); + +// Dataset relation = readPath(spark, inputPath, Relation.class) +// .filter((FilterFunction) r -> !r.getDataInfo().getDeletedbyinference() ); +// +// relation.createOrReplaceTempView("relation"); +// +// String query = "SELECT result_organization.source resultId, result_organization.target orgId, org_set merges " +// + "FROM (SELECT source, target " +// + " FROM relation " +// + " AND lower(relClass) = '" +// + ModelConstants.HAS_AUTHOR_INSTITUTION.toLowerCase() +// + "') result_organization " +// + "LEFT JOIN (SELECT source, collect_set(target) org_set " +// + " FROM relation " +// + " AND lower(relClass) = '" +// + ModelConstants.MERGES.toLowerCase() +// + "' " +// + " GROUP BY source) organization_organization " +// + "ON result_organization.target = organization_organization.source "; +// +// Dataset result_organizationset = spark +// .sql(query) +// .as(Encoders.bean(ResultOrganizations.class)); +// +// result_organizationset +// .map(mapResultCommunityFn(organizationMap), Encoders.bean(ResultCommunityList.class)) +// .filter(Objects::nonNull) +// .toJavaRDD() +// .mapToPair(value -> new Tuple2<>(value.getResultId(), value)) +// .reduceByKey((a, b) -> { +// ArrayList cl = a.getCommunityList(); +// b.getCommunityList().stream().forEach(s -> { +// if (!cl.contains(s)) { +// cl.add(s); +// } +// }); +// a.setCommunityList(cl); +// return a; +// }) +// .map(value -> OBJECT_MAPPER.writeValueAsString(value._2())) +// .saveAsTextFile(outputPath, GzipCodec.class); + } + + private static MapFunction mapResultCommunityFn( + CommunityEntityMap organizationMap) { + return value -> { + String rId = value.getResultId(); + Optional> orgs = Optional.ofNullable(value.getMerges()); + String oTarget = value.getOrgId(); + Set communitySet = new HashSet<>(); + if (organizationMap.containsKey(oTarget)) { + communitySet.addAll(organizationMap.get(oTarget)); + } + if (orgs.isPresent()) + for (String oId : orgs.get()) { + if (organizationMap.containsKey(oId)) { + communitySet.addAll(organizationMap.get(oId)); + } + } + if (!communitySet.isEmpty()) { + ResultCommunityList rcl = new ResultCommunityList(); + rcl.setResultId(rId); + ArrayList communityList = new ArrayList<>(); + communityList.addAll(communitySet); + rcl.setCommunityList(communityList); + return rcl; + } + return null; + }; + } +} From 70b78a40c7b6fb1a9ec38c4bab5f13e1cc74850e Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 20 Oct 2023 15:50:49 +0200 Subject: [PATCH 077/148] removed file from different propagation --- .../PrepareResultCommunity2.java | 225 ------------------ 1 file changed, 225 deletions(-) delete mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunity2.java diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunity2.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunity2.java deleted file mode 100644 index 96523c502..000000000 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunity2.java +++ /dev/null @@ -1,225 +0,0 @@ - -package eu.dnetlib.dhp.resulttocommunityfromorganization; - -/** - * @author miriam.baglioni - * @Date 16/10/23 - */ -/** - * @author miriam.baglioni - * @Date 16/10/23 - */ -import static eu.dnetlib.dhp.PropagationConstant.*; -import static eu.dnetlib.dhp.PropagationConstant.OBJECT_MAPPER; -import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession; -import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; - -import java.util.*; -import java.util.stream.Collectors; - -import org.apache.commons.io.IOUtils; -import org.apache.hadoop.io.ObjectWritable; -import org.apache.hadoop.io.compress.GzipCodec; -import org.apache.spark.SparkConf; -import org.apache.spark.api.java.function.*; -import org.apache.spark.api.java.function.FilterFunction; -import org.apache.spark.api.java.function.MapFunction; -import org.apache.spark.sql.*; -import org.apache.spark.sql.types.DataTypes; -import org.apache.spark.sql.types.StructType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.gson.Gson; - -import eu.dnetlib.dhp.api.Utils; -import eu.dnetlib.dhp.api.model.CommunityEntityMap; -import eu.dnetlib.dhp.application.ArgumentApplicationParser; -import eu.dnetlib.dhp.schema.common.ModelConstants; -import eu.dnetlib.dhp.schema.oaf.Relation; -import scala.Tuple2; - -public class PrepareResultCommunity2 { - - private static final Logger log = LoggerFactory - .getLogger(eu.dnetlib.dhp.resulttocommunityfromorganization.PrepareResultCommunitySet.class); - - public static void main(String[] args) throws Exception { - String jsonConfiguration = IOUtils - .toString( - eu.dnetlib.dhp.resulttocommunityfromorganization.PrepareResultCommunitySet.class - .getResourceAsStream( - "/eu/dnetlib/dhp/resulttocommunityfromorganization/input_preparecommunitytoresult_parameters.json")); - - final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); - parser.parseArgument(args); - - Boolean isSparkSessionManaged = isSparkSessionManaged(parser); - log.info("isSparkSessionManaged: {}", isSparkSessionManaged); - - String inputPath = parser.get("sourcePath"); - log.info("inputPath: {}", inputPath); - - final String outputPath = parser.get("outputPath"); - log.info("outputPath: {}", outputPath); - - final boolean production = Boolean.valueOf(parser.get("production")); - log.info("production: {}", production); - - final CommunityEntityMap organizationMap = Utils.getCommunityOrganization(production); - log.info("organizationMap: {}", new Gson().toJson(organizationMap)); - - SparkConf conf = new SparkConf(); - - runWithSparkSession( - conf, - isSparkSessionManaged, - spark -> { - removeOutputDir(spark, outputPath); - prepareInfo(spark, inputPath, outputPath, organizationMap); - }); - } - - private static void prepareInfo( - SparkSession spark, - String inputPath, - String outputPath, - CommunityEntityMap organizationMap) { - - final StructType structureSchema = new StructType() - .add( - "dataInfo", new StructType() - .add("deletedbyinference", DataTypes.BooleanType) - .add("invisible", DataTypes.BooleanType)) - .add("source", DataTypes.StringType) - .add("target", DataTypes.StringType) - .add("relClass", DataTypes.StringType); - - readPath(spark, inputPath, Relation.class) - .foreach((ForeachFunction) r -> System.out.println(new ObjectMapper().writeValueAsString(r))); - - Dataset resultOrganization = spark - .read() - .schema(Encoders.bean(Relation.class).schema()) - .json(inputPath) - .filter( - "dataInfo.deletedbyinference != true " + - "and relClass == '" + ModelConstants.HAS_AUTHOR_INSTITUTION + "'") - .select( - new Column("source").as("resultId"), - new Column("target").as("organizationId")); - - resultOrganization - .foreach((ForeachFunction) r -> System.out.println(new ObjectMapper().writeValueAsString(r))); - - Dataset organizationOrganization = spark - .read() - .schema(structureSchema) - .json(inputPath) - .filter( - "dataInfo.deletedbyinference != true " + - "and relClass == '" + ModelConstants.MERGES + "'") - .select( - new Column("source").as("masterOrganization"), - new Column("target").as("duplicateOrganization")); - - resultOrganization - .joinWith( - organizationOrganization, resultOrganization - .col("organizationId") - .equalTo(organizationOrganization.col("masterOrganization")), - "left") - .groupByKey( - (MapFunction, String>) t2 -> (String) t2._1().getAs("resultId"), Encoders.STRING()) - .mapGroups((MapGroupsFunction, ResultCommunityList>) (k, v) -> { - ResultCommunityList rcl = new ResultCommunityList(); - rcl.setResultId(k); - ArrayList cl = new ArrayList<>(); - Tuple2 first = v.next(); - cl.addAll(organizationMap.get(first._1().getAs("organizationId"))); - if (Optional.ofNullable(first._2()).isPresent()) { - cl.addAll(organizationMap.get(first._2().getAs(("duplicateOrganization")))); - } - v.forEachRemaining(o -> cl.addAll(organizationMap.get(o._2().getAs("duplicateOrganization")))); - if (cl.size() == 0) - return null; - rcl.setCommunityList(new ArrayList<>(cl.stream().distinct().collect(Collectors.toList()))); - return rcl; - }, Encoders.bean(ResultCommunityList.class)) - .filter(Objects::nonNull) - .write() - .mode(SaveMode.Overwrite) - .option("compression", "gzip") - .json(outputPath); - -// Dataset relation = readPath(spark, inputPath, Relation.class) -// .filter((FilterFunction) r -> !r.getDataInfo().getDeletedbyinference() ); -// -// relation.createOrReplaceTempView("relation"); -// -// String query = "SELECT result_organization.source resultId, result_organization.target orgId, org_set merges " -// + "FROM (SELECT source, target " -// + " FROM relation " -// + " AND lower(relClass) = '" -// + ModelConstants.HAS_AUTHOR_INSTITUTION.toLowerCase() -// + "') result_organization " -// + "LEFT JOIN (SELECT source, collect_set(target) org_set " -// + " FROM relation " -// + " AND lower(relClass) = '" -// + ModelConstants.MERGES.toLowerCase() -// + "' " -// + " GROUP BY source) organization_organization " -// + "ON result_organization.target = organization_organization.source "; -// -// Dataset result_organizationset = spark -// .sql(query) -// .as(Encoders.bean(ResultOrganizations.class)); -// -// result_organizationset -// .map(mapResultCommunityFn(organizationMap), Encoders.bean(ResultCommunityList.class)) -// .filter(Objects::nonNull) -// .toJavaRDD() -// .mapToPair(value -> new Tuple2<>(value.getResultId(), value)) -// .reduceByKey((a, b) -> { -// ArrayList cl = a.getCommunityList(); -// b.getCommunityList().stream().forEach(s -> { -// if (!cl.contains(s)) { -// cl.add(s); -// } -// }); -// a.setCommunityList(cl); -// return a; -// }) -// .map(value -> OBJECT_MAPPER.writeValueAsString(value._2())) -// .saveAsTextFile(outputPath, GzipCodec.class); - } - - private static MapFunction mapResultCommunityFn( - CommunityEntityMap organizationMap) { - return value -> { - String rId = value.getResultId(); - Optional> orgs = Optional.ofNullable(value.getMerges()); - String oTarget = value.getOrgId(); - Set communitySet = new HashSet<>(); - if (organizationMap.containsKey(oTarget)) { - communitySet.addAll(organizationMap.get(oTarget)); - } - if (orgs.isPresent()) - for (String oId : orgs.get()) { - if (organizationMap.containsKey(oId)) { - communitySet.addAll(organizationMap.get(oId)); - } - } - if (!communitySet.isEmpty()) { - ResultCommunityList rcl = new ResultCommunityList(); - rcl.setResultId(rId); - ArrayList communityList = new ArrayList<>(); - communityList.addAll(communitySet); - rcl.setCommunityList(communityList); - return rcl; - } - return null; - }; - } -} From 7fc621cdecaa23f57aa7744f51b3f15c46366dd3 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 20 Oct 2023 22:28:12 +0200 Subject: [PATCH 078/148] added defaults to the graph resolution workflow config-default.xml --- .../resolution/oozie_app/config-default.xml | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/resolution/oozie_app/config-default.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/resolution/oozie_app/config-default.xml index 6fb2a1253..86847ed46 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/resolution/oozie_app/config-default.xml +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/resolution/oozie_app/config-default.xml @@ -1,4 +1,12 @@ + + jobTracker + yarnRM + + + nameNode + hdfs://nameservice1 + oozie.use.system.libpath true @@ -7,4 +15,28 @@ oozie.action.sharelib.for.spark spark2 + + hive_metastore_uris + thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083 + + + spark2YarnHistoryServerAddress + http://iis-cdh5-test-gw.ocean.icm.edu.pl:18089 + + + spark2ExtraListeners + com.cloudera.spark.lineage.NavigatorAppListener + + + spark2SqlQueryExecutionListeners + com.cloudera.spark.lineage.NavigatorQueryListener + + + sparkExecutorNumber + 4 + + + spark2EventLogDir + /user/spark/spark2ApplicationHistory + \ No newline at end of file From a870aa2b093929e190ae48cbb15cf98d732e2926 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 20 Oct 2023 22:28:39 +0200 Subject: [PATCH 079/148] depending on dhp-schemas:3.17.2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9cd82a343..f361a266c 100644 --- a/pom.xml +++ b/pom.xml @@ -888,7 +888,7 @@ 3.3.3 3.4.2 [2.12,3.0) - [3.17.1] + [3.17.2] [4.0.3] [6.0.5] [3.1.6] From 5c5a195e97b8a7711cce04854e834e3f63073e3e Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Mon, 23 Oct 2023 11:26:17 +0200 Subject: [PATCH 080/148] refactoring and fixing issue on property name --- .../main/java/eu/dnetlib/dhp/bulktag/community/Constraint.java | 2 -- .../resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml | 2 +- .../src/test/java/eu/dnetlib/dhp/api/QueryCommunityAPITest.java | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Constraint.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Constraint.java index 48d9be7cd..13d29b940 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Constraint.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/community/Constraint.java @@ -41,7 +41,6 @@ public class Constraint implements Serializable { this.value = value; } - @JsonIgnore public void setSelection(VerbResolver resolver) throws InvocationTargetException, NoSuchMethodException, InstantiationException, @@ -53,5 +52,4 @@ public class Constraint implements Serializable { return selection.apply(metadata); } - } diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml index 4b81c58e4..e3ca72493 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml @@ -111,7 +111,7 @@ --num-executors=${sparkExecutorNumber} --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} - --conf spark.executor.memoryOverhead=${memeoryOverhead} + --conf spark.executor.memoryOverhead=${memoryOverhead} --conf spark.sql.shuffle.partitions=${partitions} --driver-memory=${sparkDriverMemory} --conf spark.extraListeners=${spark2ExtraListeners} diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/api/QueryCommunityAPITest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/api/QueryCommunityAPITest.java index 0cdf0f50b..e3c2a5f45 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/api/QueryCommunityAPITest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/api/QueryCommunityAPITest.java @@ -1,7 +1,6 @@ package eu.dnetlib.dhp.api; - import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; From 89c4dfbaf455d9621e76d0eebf2ca6ea7c718686 Mon Sep 17 00:00:00 2001 From: dimitrispie Date: Tue, 24 Oct 2023 09:48:23 +0300 Subject: [PATCH 081/148] StatsDB workflow to export actionsets about OA routes, diamond, and publicly-funded A new oozie workflow capable to read from the stats db to produce a new actionSet for updating results with: - green_oa ={true, false} - openAccesColor = {gold, hybrid, bronze} - in_diamond_journal={true, false} - publicly_funded={true, false} Inputs: - outputPath - statsDB --- dhp-workflows/dhp-stats-actionsets/pom.xml | 114 +++++++ .../dnetlib/dhp/actionmanager/Constants.java | 102 +++++++ .../StatsAtomicActionsJob.java | 286 ++++++++++++++++++ .../stats_actionsets/StatsDiamondOAModel.java | 29 ++ .../stats_actionsets/StatsGreenOAModel.java | 29 ++ .../stats_actionsets/StatsOAColourModel.java | 47 +++ .../StatsPubliclyFundedModel.java | 29 ++ .../input_actionset_parameter.json | 32 ++ .../oozie_app/config-default.xml | 30 ++ .../stats_actionsets/oozie_app/workflow.xml | 125 ++++++++ .../src/main/resources/log4j.properties | 12 + 11 files changed, 835 insertions(+) create mode 100644 dhp-workflows/dhp-stats-actionsets/pom.xml create mode 100644 dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/Constants.java create mode 100644 dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsAtomicActionsJob.java create mode 100644 dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsDiamondOAModel.java create mode 100644 dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsGreenOAModel.java create mode 100644 dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsOAColourModel.java create mode 100644 dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsPubliclyFundedModel.java create mode 100644 dhp-workflows/dhp-stats-actionsets/src/main/resources/eu/dnetlib/dhp/actionmanager/stats_actionsets/input_actionset_parameter.json create mode 100644 dhp-workflows/dhp-stats-actionsets/src/main/resources/eu/dnetlib/dhp/actionmanager/stats_actionsets/oozie_app/config-default.xml create mode 100644 dhp-workflows/dhp-stats-actionsets/src/main/resources/eu/dnetlib/dhp/actionmanager/stats_actionsets/oozie_app/workflow.xml create mode 100644 dhp-workflows/dhp-stats-actionsets/src/main/resources/log4j.properties diff --git a/dhp-workflows/dhp-stats-actionsets/pom.xml b/dhp-workflows/dhp-stats-actionsets/pom.xml new file mode 100644 index 000000000..670fd4a5e --- /dev/null +++ b/dhp-workflows/dhp-stats-actionsets/pom.xml @@ -0,0 +1,114 @@ + + + 4.0.0 + + eu.dnetlib.dhp + dhp-workflows + 1.2.5-SNAPSHOT + + dhp-stats-actionsets + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + org.apache.httpcomponents + httpclient + + + + org.apache.spark + spark-core_2.11 + + + org.apache.spark + spark-sql_2.11 + + + + eu.dnetlib.dhp + dhp-common + ${project.version} + + + + net.sf.saxon + Saxon-HE + + + dom4j + dom4j + + + + xml-apis + xml-apis + + + + jaxen + jaxen + + + + org.json + json + + + + + org.apache.poi + poi-ooxml + + + + + org.apache.commons + commons-compress + + + + org.mongodb + mongo-java-driver + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/Constants.java b/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/Constants.java new file mode 100644 index 000000000..0df34364d --- /dev/null +++ b/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/Constants.java @@ -0,0 +1,102 @@ + +package eu.dnetlib.dhp.actionmanager; + +import java.util.Optional; + +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SparkSession; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.schema.common.ModelConstants; +import eu.dnetlib.dhp.schema.oaf.StructuredProperty; +import eu.dnetlib.dhp.schema.oaf.Subject; +import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils; + +public class Constants { + + public static final String DOI = "doi"; + public static final String DOI_CLASSNAME = "Digital Object Identifier"; + + public static final String DEFAULT_DELIMITER = ","; + public static final String DEFAULT_FOS_DELIMITER = "\t"; + + public static final String UPDATE_DATA_INFO_TYPE = "update"; +// public static final String UPDATE_SUBJECT_FOS_CLASS_ID = "subject:fos"; + public static final String UPDATE_CLASS_NAME = "Inferred by OpenAIRE"; +// public static final String UPDATE_MEASURE_BIP_CLASS_ID = "measure:bip"; +// public static final String UPDATE_SUBJECT_SDG_CLASS_ID = "subject:sdg"; +// public static final String UPDATE_MEASURE_USAGE_COUNTS_CLASS_ID = "measure:usage_counts"; +// public static final String UPDATE_KEY_USAGE_COUNTS = "count"; + + public static final String UPDATE_MEASURE_STATS_MODEL_CLASS_ID = "measure:stats_model"; + public static final String UPDATE_KEY_STATS_MODEL = "stats_model"; + +// public static final String UPDATE_MEASURE_PUBLICLY_FUNDED_CLASS_ID = "measure:publicly_funded"; +// public static final String UPDATE_KEY_PUBLICLY_FUNDED = "publicly_funded"; + +// public static final String FOS_CLASS_ID = "FOS"; +// public static final String FOS_CLASS_NAME = "Fields of Science and Technology classification"; +// +// public static final String SDG_CLASS_ID = "SDG"; +// public static final String SDG_CLASS_NAME = "Sustainable Development Goals"; + + public static final String NULL = "NULL"; + + public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private Constants() { + } + + public static Boolean isSparkSessionManaged(ArgumentApplicationParser parser) { + return Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + } + + public static Dataset readPath( + SparkSession spark, String inputPath, Class clazz) { + return spark + .read() + .textFile(inputPath) + .map((MapFunction) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz)); + } + + public static Subject getSubject(String sbj, String classid, String classname, + String diqualifierclassid) { + if (sbj == null || sbj.equals(NULL)) + return null; + Subject s = new Subject(); + s.setValue(sbj); + s + .setQualifier( + OafMapperUtils + .qualifier( + classid, + classname, + ModelConstants.DNET_SUBJECT_TYPOLOGIES, + ModelConstants.DNET_SUBJECT_TYPOLOGIES)); + s + .setDataInfo( + OafMapperUtils + .dataInfo( + false, + UPDATE_DATA_INFO_TYPE, + true, + false, + OafMapperUtils + .qualifier( + diqualifierclassid, + UPDATE_CLASS_NAME, + ModelConstants.DNET_PROVENANCE_ACTIONS, + ModelConstants.DNET_PROVENANCE_ACTIONS), + "")); + + return s; + + } +} diff --git a/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsAtomicActionsJob.java b/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsAtomicActionsJob.java new file mode 100644 index 000000000..f6e2cfe01 --- /dev/null +++ b/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsAtomicActionsJob.java @@ -0,0 +1,286 @@ + +package eu.dnetlib.dhp.actionmanager.stats_actionsets; + +import static eu.dnetlib.dhp.actionmanager.Constants.*; +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.SequenceFileOutputFormat; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.api.java.function.MapGroupsFunction; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SaveMode; +import org.apache.spark.sql.SparkSession; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.common.HdfsSupport; +import eu.dnetlib.dhp.schema.action.AtomicAction; +import eu.dnetlib.dhp.schema.common.ModelConstants; +import eu.dnetlib.dhp.schema.oaf.*; +import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils; +import scala.Tuple2; + +/** + * created the Atomic Action for each type of results + */ +public class StatsAtomicActionsJob implements Serializable { + + private static final Logger log = LoggerFactory.getLogger(StatsAtomicActionsJob.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + public static void main(String[] args) throws Exception { + + String jsonConfiguration = IOUtils + .toString( + StatsAtomicActionsJob.class + .getResourceAsStream( + "/eu/dnetlib/dhp/actionmanager/stats_actionsets/input_actionset_parameter.json")); + + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + + parser.parseArgument(args); + + Boolean isSparkSessionManaged = Optional + .ofNullable(parser.get("isSparkSessionManaged")) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + final String outputPath = parser.get("outputPath"); + log.info("outputPath {}: ", outputPath); + + SparkConf conf = new SparkConf(); + conf.set("hive.metastore.uris", parser.get("hive_metastore_uris")); + + final String dbname = parser.get("statsDB"); + + final String workingPath = parser.get("workingPath"); + + runWithSparkHiveSession( + conf, + isSparkSessionManaged, + spark -> { + removeOutputDir(spark, outputPath); + prepareGreenData(dbname, spark, workingPath + "/greenOADB", "indi_pub_green_oa", "id"); + prepareDiamondData(dbname, spark, workingPath + "/diamondOADΒ", "indi_pub_diamond", "id"); + preparePubliclyFundedData( + dbname, spark, workingPath + "/publiclyFundedDΒ", "indi_funded_result_with_fundref", "id"); + prepareOAColourData(dbname, spark, workingPath + "/oacolourDB", "", "id"); + writeActionSet(spark, workingPath, outputPath); + }); + } + + private static void prepareGreenData(String dbname, SparkSession spark, String workingPath, String tableName, + String resultAttributeName) { + spark + .sql( + String + .format( + "select %s as id, green_oa as green_oa " + + "from %s.%s", + resultAttributeName, dbname, tableName)) + .as(Encoders.bean(StatsGreenOAModel.class)) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(workingPath); + } + + private static void prepareDiamondData(String dbname, SparkSession spark, String workingPath, String tableName, + String resultAttributeName) { + spark + .sql( + String + .format( + "select %s as id, in_diamond_journal as in_diamond_journal " + + "from %s.%s", + resultAttributeName, dbname, tableName)) + .as(Encoders.bean(StatsDiamondOAModel.class)) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(workingPath); + } + + private static void preparePubliclyFundedData(String dbname, SparkSession spark, String workingPath, + String tableName, + String resultAttributeName) { + spark + .sql( + String + .format( + "select %s as id, fundref as publicly_funded " + + "from %s.%s", + resultAttributeName, dbname, tableName)) + .as(Encoders.bean(StatsPubliclyFundedModel.class)) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(workingPath); + } + + private static void prepareOAColourData(String dbname, SparkSession spark, String workingPath, String tableName, + String resultAttributeName) { + spark + .sql( + String + .format( + "select b.%s as id, is_gold, is_bronze_oa, is_hybrid from %s.indi_pub_bronze_oa b " + + "left outer join %s.indi_pub_gold_oa g on g.id=b.id " + + "left outer join %s.indi_pub_hybrid h on b.id=h.id", + resultAttributeName, dbname, dbname, dbname)) + .as(Encoders.bean(StatsOAColourModel.class)) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(workingPath); + } + + public static void writeActionSet(SparkSession spark, String inputPath, String outputPath) { + + getFinalIndicatorsGreenResult(spark, inputPath + "/greenOADB") + .toJavaRDD() + .map(p -> new AtomicAction(p.getClass(), p)) + .union( + getFinalIndicatorsDiamondResult(spark, inputPath + "/diamondOADΒ") + .toJavaRDD() + .map(p -> new AtomicAction(p.getClass(), p))) + .union( + getFinalIndicatorsPubliclyFundedResult(spark, inputPath + "/publiclyFundedDΒ") + .toJavaRDD() + .map(p -> new AtomicAction(p.getClass(), p))) + .union( + getFinalIndicatorsOAColourResult(spark, inputPath + "/oacolourDB") + .toJavaRDD() + .map(p -> new AtomicAction(p.getClass(), p))) + .mapToPair( + aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()), + new Text(OBJECT_MAPPER.writeValueAsString(aa)))) + .saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class); + } + + public static Measure newMeasureInstance(String id) { + Measure m = new Measure(); + m.setId(id); + m.setUnit(new ArrayList<>()); + return m; + } + + private static Dataset getFinalIndicatorsGreenResult(SparkSession spark, String inputPath) { + + return readPath(spark, inputPath, StatsGreenOAModel.class) + .map((MapFunction) usm -> { + Result r = new Result(); + r.setId("50|" + usm.getId()); + r.setMeasures(getMeasure(usm.isGreen_oa(), "green_oa")); + return r; + }, Encoders.bean(Result.class)); + } + + private static Dataset getFinalIndicatorsDiamondResult(SparkSession spark, String inputPath) { + + return readPath(spark, inputPath, StatsDiamondOAModel.class) + .map((MapFunction) usm -> { + Result r = new Result(); + r.setId("50|" + usm.getId()); + r.setMeasures(getMeasure(usm.isIn_diamond_journal(), "in_diamond_journal")); + return r; + }, Encoders.bean(Result.class)); + } + + private static Dataset getFinalIndicatorsPubliclyFundedResult(SparkSession spark, String inputPath) { + + return readPath(spark, inputPath, StatsPubliclyFundedModel.class) + .map((MapFunction) usm -> { + Result r = new Result(); + r.setId("50|" + usm.getId()); + r.setMeasures(getMeasure(usm.isPublicly_funded(), "publicly_funded")); + return r; + }, Encoders.bean(Result.class)); + } + + private static Dataset getFinalIndicatorsOAColourResult(SparkSession spark, String inputPath) { + + return readPath(spark, inputPath, StatsOAColourModel.class) + .map((MapFunction) usm -> { + Result r = new Result(); + r.setId("50|" + usm.getId()); + r.setMeasures(getMeasureOAColour(usm.isIs_gold(), usm.isIs_bronze_oa(), usm.isIs_hybrid())); + return r; + }, Encoders.bean(Result.class)); + } + + private static List getMeasure(Boolean is_model_oa, String model_type) { + DataInfo dataInfo = OafMapperUtils + .dataInfo( + false, + UPDATE_DATA_INFO_TYPE, + true, + false, + OafMapperUtils + .qualifier( + UPDATE_MEASURE_STATS_MODEL_CLASS_ID, + UPDATE_CLASS_NAME, + ModelConstants.DNET_PROVENANCE_ACTIONS, + ModelConstants.DNET_PROVENANCE_ACTIONS), + ""); + + return Arrays + .asList( + OafMapperUtils + .newMeasureInstance(model_type, String.valueOf(is_model_oa), UPDATE_KEY_STATS_MODEL, dataInfo)); + } + + private static List getMeasureOAColour(Boolean is_gold, Boolean is_bronze_oa, Boolean is_hybrid) { + DataInfo dataInfo = OafMapperUtils + .dataInfo( + false, + UPDATE_DATA_INFO_TYPE, + true, + false, + OafMapperUtils + .qualifier( + UPDATE_MEASURE_STATS_MODEL_CLASS_ID, + UPDATE_CLASS_NAME, + ModelConstants.DNET_PROVENANCE_ACTIONS, + ModelConstants.DNET_PROVENANCE_ACTIONS), + ""); + + return Arrays + .asList( + OafMapperUtils + .newMeasureInstance("is_gold", String.valueOf(is_gold), UPDATE_KEY_STATS_MODEL, dataInfo), + OafMapperUtils + .newMeasureInstance("is_bronze_oa", String.valueOf(is_bronze_oa), UPDATE_KEY_STATS_MODEL, dataInfo), + OafMapperUtils + .newMeasureInstance("is_hybrid", String.valueOf(is_hybrid), UPDATE_KEY_STATS_MODEL, dataInfo)); + + } + + private static void removeOutputDir(SparkSession spark, String path) { + HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration()); + } + + public static Dataset readPath( + SparkSession spark, String inputPath, Class clazz) { + return spark + .read() + .textFile(inputPath) + .map((MapFunction) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz)); + } + +} diff --git a/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsDiamondOAModel.java b/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsDiamondOAModel.java new file mode 100644 index 000000000..340790fa8 --- /dev/null +++ b/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsDiamondOAModel.java @@ -0,0 +1,29 @@ + +package eu.dnetlib.dhp.actionmanager.stats_actionsets; + +import java.io.Serializable; + +/** + * @author dimitris.pierrakos + * @Date 30/10/23 + */ +public class StatsDiamondOAModel implements Serializable { + private String id; + private boolean in_diamond_journal; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public boolean isIn_diamond_journal() { + return in_diamond_journal; + } + + public void setIn_diamond_journal(boolean in_diamond_journal) { + this.in_diamond_journal = in_diamond_journal; + } +} diff --git a/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsGreenOAModel.java b/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsGreenOAModel.java new file mode 100644 index 000000000..3d59bdf95 --- /dev/null +++ b/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsGreenOAModel.java @@ -0,0 +1,29 @@ + +package eu.dnetlib.dhp.actionmanager.stats_actionsets; + +import java.io.Serializable; + +/** + * @author dimitris.pierrakos + * @Date 30/10/23 + */ +public class StatsGreenOAModel implements Serializable { + private String id; + private boolean green_oa; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public boolean isGreen_oa() { + return green_oa; + } + + public void setGreen_oa(boolean green_oa) { + this.green_oa = green_oa; + } +} diff --git a/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsOAColourModel.java b/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsOAColourModel.java new file mode 100644 index 000000000..a83eb4f60 --- /dev/null +++ b/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsOAColourModel.java @@ -0,0 +1,47 @@ + +package eu.dnetlib.dhp.actionmanager.stats_actionsets; + +import java.io.Serializable; + +/** + * @author dimitris.pierrakos + * @Date 30/10/23 + */ +public class StatsOAColourModel implements Serializable { + private String id; + private boolean is_gold; + private boolean is_bronze_oa; + private boolean is_hybrid; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public boolean isIs_gold() { + return is_gold; + } + + public void setIs_gold(boolean is_gold) { + this.is_gold = is_gold; + } + + public boolean isIs_bronze_oa() { + return is_bronze_oa; + } + + public void setIs_bronze_oa(boolean is_bronze_oa) { + this.is_bronze_oa = is_bronze_oa; + } + + public boolean isIs_hybrid() { + return is_hybrid; + } + + public void setIs_hybrid(boolean is_hybrid) { + this.is_hybrid = is_hybrid; + } +} diff --git a/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsPubliclyFundedModel.java b/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsPubliclyFundedModel.java new file mode 100644 index 000000000..6bdc78220 --- /dev/null +++ b/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsPubliclyFundedModel.java @@ -0,0 +1,29 @@ + +package eu.dnetlib.dhp.actionmanager.stats_actionsets; + +import java.io.Serializable; + +/** + * @author dimitris.pierrakos + * @Date 30/10/23 + */ +public class StatsPubliclyFundedModel implements Serializable { + private String id; + private boolean publicly_funded; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public boolean isPublicly_funded() { + return publicly_funded; + } + + public void setPublicly_funded(boolean publicly_funded) { + this.publicly_funded = publicly_funded; + } +} diff --git a/dhp-workflows/dhp-stats-actionsets/src/main/resources/eu/dnetlib/dhp/actionmanager/stats_actionsets/input_actionset_parameter.json b/dhp-workflows/dhp-stats-actionsets/src/main/resources/eu/dnetlib/dhp/actionmanager/stats_actionsets/input_actionset_parameter.json new file mode 100644 index 000000000..bb24d4c99 --- /dev/null +++ b/dhp-workflows/dhp-stats-actionsets/src/main/resources/eu/dnetlib/dhp/actionmanager/stats_actionsets/input_actionset_parameter.json @@ -0,0 +1,32 @@ +[ + { + "paramName": "issm", + "paramLongName": "isSparkSessionManaged", + "paramDescription": "when true will stop SparkSession after job execution", + "paramRequired": false + }, + { + "paramName": "hmu", + "paramLongName": "hive_metastore_uris", + "paramDescription": "the URI for the hive metastore", + "paramRequired": true + }, + { + "paramName": "o", + "paramLongName": "outputPath", + "paramDescription": "the path of the new ActionSet", + "paramRequired": true + }, + { + "paramName": "sdb", + "paramLongName": "statsDB", + "paramDescription": "the name of the stats db to be used", + "paramRequired": true + }, + { + "paramName": "wp", + "paramLongName": "workingPath", + "paramDescription": "the workingPath where to save the content of the usage_stats table", + "paramRequired": true + } +] \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-actionsets/src/main/resources/eu/dnetlib/dhp/actionmanager/stats_actionsets/oozie_app/config-default.xml b/dhp-workflows/dhp-stats-actionsets/src/main/resources/eu/dnetlib/dhp/actionmanager/stats_actionsets/oozie_app/config-default.xml new file mode 100644 index 000000000..d262cb6e0 --- /dev/null +++ b/dhp-workflows/dhp-stats-actionsets/src/main/resources/eu/dnetlib/dhp/actionmanager/stats_actionsets/oozie_app/config-default.xml @@ -0,0 +1,30 @@ + + + jobTracker + yarnRM + + + nameNode + hdfs://nameservice1 + + + oozie.use.system.libpath + true + + + hiveMetastoreUris + thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083 + + + hiveJdbcUrl + jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000 + + + hiveDbName + openaire + + + oozie.launcher.mapreduce.user.classpath.first + true + + diff --git a/dhp-workflows/dhp-stats-actionsets/src/main/resources/eu/dnetlib/dhp/actionmanager/stats_actionsets/oozie_app/workflow.xml b/dhp-workflows/dhp-stats-actionsets/src/main/resources/eu/dnetlib/dhp/actionmanager/stats_actionsets/oozie_app/workflow.xml new file mode 100644 index 000000000..1aa4e1050 --- /dev/null +++ b/dhp-workflows/dhp-stats-actionsets/src/main/resources/eu/dnetlib/dhp/actionmanager/stats_actionsets/oozie_app/workflow.xml @@ -0,0 +1,125 @@ + + + + outputPath + the path where to store the actionset + + + statsDB + the name of the stats db to be used + + + sparkDriverMemory + memory for driver process + + + sparkExecutorMemory + memory for individual executor + + + sparkExecutorCores + number of cores used by single executor + + + oozieActionShareLibForSpark2 + oozie action sharelib for spark 2.* + + + spark2ExtraListeners + com.cloudera.spark.lineage.NavigatorAppListener + spark 2.* extra listeners classname + + + spark2SqlQueryExecutionListeners + com.cloudera.spark.lineage.NavigatorQueryListener + spark 2.* sql query execution listeners classname + + + spark2YarnHistoryServerAddress + spark 2.* yarn history server address + + + spark2EventLogDir + spark 2.* event log dir location + + + + + ${jobTracker} + ${nameNode} + + + mapreduce.job.queuename + ${queueName} + + + oozie.launcher.mapred.job.queue.name + ${oozieLauncherQueueName} + + + oozie.action.sharelib.for.spark + ${oozieActionShareLibForSpark2} + + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + yarn + cluster + Produces the atomic action with the usage stats count for results + + eu.dnetlib.dhp.actionmanager.stats_actionsets.SparkAtomicActionGreenOAJob + dhp-stats-actionsets-${projectVersion}.jar + + --executor-memory=${sparkExecutorMemory} + --executor-cores=${sparkExecutorCores} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} + + --hive_metastore_uris${hiveMetastoreUris} + --outputPath${outputPath} + --statsDB${statsDB} + --workingPath${workingDir} + + + + + + + + yarn + cluster + Produces the atomic action with the stats green_oa for results + + eu.dnetlib.dhp.actionmanager.stats_actionsets.StatsAtomicActionsJob + dhp-stats-actionsets-${projectVersion}.jar + + --executor-memory=${sparkExecutorMemory} + --executor-cores=${sparkExecutorCores} + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} + + --hive_metastore_uris${hiveMetastoreUris} + --outputPath${outputPath} + --statsDB${statsDB} + --workingPath${workingDir} + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-actionsets/src/main/resources/log4j.properties b/dhp-workflows/dhp-stats-actionsets/src/main/resources/log4j.properties new file mode 100644 index 000000000..81458d1f7 --- /dev/null +++ b/dhp-workflows/dhp-stats-actionsets/src/main/resources/log4j.properties @@ -0,0 +1,12 @@ +# Set root logger level to DEBUG and its only appender to A1. +log4j.rootLogger=INFO, A1 + +# A1 is set to be a ConsoleAppender. +log4j.appender.A1=org.apache.log4j.ConsoleAppender + +# A1 uses PatternLayout. +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n + +log4j.logger.org.apache.spark=FATAL +log4j.logger.org.spark_project=FATAL From a82aaf57b219faa880817cf18213b22944cb730b Mon Sep 17 00:00:00 2001 From: Serafeim Chatzopoulos Date: Wed, 25 Oct 2023 12:05:02 -0700 Subject: [PATCH 082/148] Renaming input param for crossref input path --- .../bipaffiliations/PrepareAffiliationRelations.java | 8 ++++---- .../bipaffiliations/input_actionset_parameter.json | 4 ++-- .../dhp/actionmanager/bipaffiliations/job.properties | 4 ++-- .../actionmanager/bipaffiliations/oozie_app/workflow.xml | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java index 18d98be54..1bdb06ecc 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java @@ -58,8 +58,8 @@ public class PrepareAffiliationRelations implements Serializable { Boolean isSparkSessionManaged = Constants.isSparkSessionManaged(parser); log.info("isSparkSessionManaged: {}", isSparkSessionManaged); - final String inputPath = parser.get("inputPath"); - log.info("inputPath: {}", inputPath); + final String crossrefInputPath = parser.get("crossrefInputPath"); + log.info("crossrefInputPath: {}", crossrefInputPath); final String pubmedInputPath = parser.get("pubmedInputPath"); log.info("pubmedInputPath: {}", pubmedInputPath); @@ -78,12 +78,12 @@ public class PrepareAffiliationRelations implements Serializable { List collectedFromCrossref = OafMapperUtils .listKeyValues(ModelConstants.CROSSREF_ID, "Crossref"); JavaPairRDD crossrefRelations = prepareAffiliationRelations( - spark, inputPath, collectedFromCrossref); + spark, crossrefInputPath, collectedFromCrossref); List collectedFromPubmed = OafMapperUtils .listKeyValues(ModelConstants.PUBMED_CENTRAL_ID, "Pubmed"); JavaPairRDD pubmedRelations = prepareAffiliationRelations( - spark, inputPath, collectedFromPubmed); + spark, pubmedInputPath, collectedFromPubmed); crossrefRelations .union(pubmedRelations) diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/input_actionset_parameter.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/input_actionset_parameter.json index 96dcc3b32..c6f905199 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/input_actionset_parameter.json +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/input_actionset_parameter.json @@ -6,8 +6,8 @@ "paramRequired": false }, { - "paramName": "ip", - "paramLongName": "inputPath", + "paramName": "cip", + "paramLongName": "crossrefInputPath", "paramDescription": "the path to get the input data from Crossref", "paramRequired": true }, diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/job.properties b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/job.properties index fe3cbb633..a3d55ff0c 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/job.properties +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/job.properties @@ -31,6 +31,6 @@ spark2SqlQueryExecutionListeners=com.cloudera.spark.lineage.NavigatorQueryListen # The following is needed as a property of a workflow oozie.wf.application.path=${oozieTopWfApplicationPath} -inputPath=/data/bip-affiliations/data.json -pubmedInputPath=/data/bip-affiiations/pubmed-data.json +crossrefInputPath=/data/bip-affiliations/data.json +pubmedInputPath=/data/bip-affiliations/pubmed-data.json outputPath=/tmp/crossref-affiliations-output-v5 diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/oozie_app/workflow.xml index e3fdddfd6..c5ac6f884 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/oozie_app/workflow.xml @@ -2,7 +2,7 @@ - inputPath + crossrefInputPath the path where to find the inferred affiliation relations from Crossref @@ -100,7 +100,7 @@ --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} - --inputPath${inputPath} + --crossrefInputPath${crossrefInputPath} --pubmedInputPath${pubmedInputPath} --outputPath${outputPath} From 0097f4e64bd79e67781c4d681dd8d91d70ab8938 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 26 Oct 2023 09:38:09 +0200 Subject: [PATCH 083/148] Removed Query community testing. Removed package from common related to the interaction with Zenodo since it was moved to the dump-project --- .../common/api/InputStreamRequestBody.java | 53 --- .../api/MissingConceptDoiException.java | 8 - .../dhp/common/api/ZenodoAPIClient.java | 365 ------------------ .../dhp/common/api/zenodo/Community.java | 14 - .../dhp/common/api/zenodo/Creator.java | 47 --- .../dnetlib/dhp/common/api/zenodo/File.java | 44 --- .../dnetlib/dhp/common/api/zenodo/Grant.java | 23 -- .../dnetlib/dhp/common/api/zenodo/Links.java | 92 ----- .../dhp/common/api/zenodo/Metadata.java | 153 -------- .../dhp/common/api/zenodo/PrereserveDoi.java | 25 -- .../common/api/zenodo/RelatedIdentifier.java | 43 --- .../dhp/common/api/zenodo/ZenodoModel.java | 118 ------ .../common/api/zenodo/ZenodoModelList.java | 7 - .../dhp/common/api/ZenodoAPIClientTest.java | 128 ------ .../eu/dnetlib/pace/util/DiffPatchMatch.java | 1 - .../main/java/eu/dnetlib/dhp/api/Utils.java | 3 - .../dhp/api/QueryCommunityAPITest.java | 115 ------ 17 files changed, 1239 deletions(-) delete mode 100644 dhp-common/src/main/java/eu/dnetlib/dhp/common/api/InputStreamRequestBody.java delete mode 100644 dhp-common/src/main/java/eu/dnetlib/dhp/common/api/MissingConceptDoiException.java delete mode 100644 dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java delete mode 100644 dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Community.java delete mode 100644 dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Creator.java delete mode 100644 dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/File.java delete mode 100644 dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Grant.java delete mode 100644 dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Links.java delete mode 100644 dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Metadata.java delete mode 100644 dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/PrereserveDoi.java delete mode 100644 dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/RelatedIdentifier.java delete mode 100644 dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/ZenodoModel.java delete mode 100644 dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/ZenodoModelList.java delete mode 100644 dhp-common/src/test/java/eu/dnetlib/dhp/common/api/ZenodoAPIClientTest.java delete mode 100644 dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/api/QueryCommunityAPITest.java diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/InputStreamRequestBody.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/InputStreamRequestBody.java deleted file mode 100644 index c127783e5..000000000 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/InputStreamRequestBody.java +++ /dev/null @@ -1,53 +0,0 @@ - -package eu.dnetlib.dhp.common.api; - -import java.io.IOException; -import java.io.InputStream; - -import okhttp3.MediaType; -import okhttp3.RequestBody; -import okhttp3.internal.Util; -import okio.BufferedSink; -import okio.Okio; -import okio.Source; - -public class InputStreamRequestBody extends RequestBody { - - private final InputStream inputStream; - private final MediaType mediaType; - private final long lenght; - - public static RequestBody create(final MediaType mediaType, final InputStream inputStream, final long len) { - - return new InputStreamRequestBody(inputStream, mediaType, len); - } - - private InputStreamRequestBody(InputStream inputStream, MediaType mediaType, long len) { - this.inputStream = inputStream; - this.mediaType = mediaType; - this.lenght = len; - } - - @Override - public MediaType contentType() { - return mediaType; - } - - @Override - public long contentLength() { - - return lenght; - - } - - @Override - public void writeTo(BufferedSink sink) throws IOException { - Source source = null; - try { - source = Okio.source(inputStream); - sink.writeAll(source); - } finally { - Util.closeQuietly(source); - } - } -} diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/MissingConceptDoiException.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/MissingConceptDoiException.java deleted file mode 100644 index b75872eb4..000000000 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/MissingConceptDoiException.java +++ /dev/null @@ -1,8 +0,0 @@ - -package eu.dnetlib.dhp.common.api; - -public class MissingConceptDoiException extends Throwable { - public MissingConceptDoiException(String message) { - super(message); - } -} diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java deleted file mode 100644 index 544da78f5..000000000 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/ZenodoAPIClient.java +++ /dev/null @@ -1,365 +0,0 @@ - -package eu.dnetlib.dhp.common.api; - -import java.io.*; -import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.URL; -import java.util.concurrent.TimeUnit; - -import org.apache.http.HttpHeaders; -import org.apache.http.entity.ContentType; -import org.jetbrains.annotations.NotNull; - -import com.google.gson.Gson; - -import eu.dnetlib.dhp.common.api.zenodo.ZenodoModel; -import eu.dnetlib.dhp.common.api.zenodo.ZenodoModelList; -import okhttp3.*; - -public class ZenodoAPIClient implements Serializable { - - String urlString; - String bucket; - - String deposition_id; - String access_token; - - public static final MediaType MEDIA_TYPE_JSON = MediaType.parse("application/json; charset=utf-8"); - - private static final MediaType MEDIA_TYPE_ZIP = MediaType.parse("application/zip"); - - public String getUrlString() { - return urlString; - } - - public void setUrlString(String urlString) { - this.urlString = urlString; - } - - public String getBucket() { - return bucket; - } - - public void setBucket(String bucket) { - this.bucket = bucket; - } - - public void setDeposition_id(String deposition_id) { - this.deposition_id = deposition_id; - } - - public ZenodoAPIClient(String urlString, String access_token) { - - this.urlString = urlString; - this.access_token = access_token; - } - - /** - * Brand new deposition in Zenodo. It sets the deposition_id and the bucket where to store the files to upload - * - * @return response code - * @throws IOException - */ - public int newDeposition() throws IOException { - String json = "{}"; - - URL url = new URL(urlString); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - conn.setRequestProperty(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()); - conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token); - conn.setRequestMethod("POST"); - conn.setDoOutput(true); - try (OutputStream os = conn.getOutputStream()) { - byte[] input = json.getBytes("utf-8"); - os.write(input, 0, input.length); - } - - String body = getBody(conn); - - int responseCode = conn.getResponseCode(); - conn.disconnect(); - - if (!checkOKStatus(responseCode)) - throw new IOException("Unexpected code " + responseCode + body); - - ZenodoModel newSubmission = new Gson().fromJson(body, ZenodoModel.class); - this.bucket = newSubmission.getLinks().getBucket(); - this.deposition_id = newSubmission.getId(); - - return responseCode; - } - - /** - * Upload files in Zenodo. - * - * @param is the inputStream for the file to upload - * @param file_name the name of the file as it will appear on Zenodo - * @return the response code - */ - public int uploadIS(InputStream is, String file_name) throws IOException { - - URL url = new URL(bucket + "/" + file_name); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - conn.setRequestProperty(HttpHeaders.CONTENT_TYPE, "application/zip"); - conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token); - conn.setDoOutput(true); - conn.setRequestMethod("PUT"); - - byte[] buf = new byte[8192]; - int length; - try (OutputStream os = conn.getOutputStream()) { - while ((length = is.read(buf)) != -1) { - os.write(buf, 0, length); - } - - } - int responseCode = conn.getResponseCode(); - if (!checkOKStatus(responseCode)) { - throw new IOException("Unexpected code " + responseCode + getBody(conn)); - } - - return responseCode; - } - - @NotNull - private String getBody(HttpURLConnection conn) throws IOException { - String body = "{}"; - try (BufferedReader br = new BufferedReader( - new InputStreamReader(conn.getInputStream(), "utf-8"))) { - StringBuilder response = new StringBuilder(); - String responseLine = null; - while ((responseLine = br.readLine()) != null) { - response.append(responseLine.trim()); - } - - body = response.toString(); - - } - return body; - } - - /** - * Associates metadata information to the current deposition - * - * @param metadata the metadata - * @return response code - * @throws IOException - */ - public int sendMretadata(String metadata) throws IOException { - - URL url = new URL(urlString + "/" + deposition_id); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - conn.setRequestProperty(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()); - conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token); - conn.setDoOutput(true); - conn.setRequestMethod("PUT"); - - try (OutputStream os = conn.getOutputStream()) { - byte[] input = metadata.getBytes("utf-8"); - os.write(input, 0, input.length); - - } - - final int responseCode = conn.getResponseCode(); - conn.disconnect(); - if (!checkOKStatus(responseCode)) - throw new IOException("Unexpected code " + responseCode + getBody(conn)); - - return responseCode; - - } - - private boolean checkOKStatus(int responseCode) { - - if (HttpURLConnection.HTTP_OK != responseCode || - HttpURLConnection.HTTP_CREATED != responseCode) - return true; - return false; - } - - /** - * To publish the current deposition. It works for both new deposition or new version of an old deposition - * - * @return response code - * @throws IOException - */ - @Deprecated - public int publish() throws IOException { - - String json = "{}"; - - OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build(); - - RequestBody body = RequestBody.create(json, MEDIA_TYPE_JSON); - - Request request = new Request.Builder() - .url(urlString + "/" + deposition_id + "/actions/publish") - .addHeader("Authorization", "Bearer " + access_token) - .post(body) - .build(); - - try (Response response = httpClient.newCall(request).execute()) { - - if (!response.isSuccessful()) - throw new IOException("Unexpected code " + response + response.body().string()); - - return response.code(); - - } - } - - /** - * To create a new version of an already published deposition. It sets the deposition_id and the bucket to be used - * for the new version. - * - * @param concept_rec_id the concept record id of the deposition for which to create a new version. It is the last - * part of the url for the DOI Zenodo suggests to use to cite all versions: DOI: 10.xxx/zenodo.656930 - * concept_rec_id = 656930 - * @return response code - * @throws IOException - * @throws MissingConceptDoiException - */ - public int newVersion(String concept_rec_id) throws IOException, MissingConceptDoiException { - setDepositionId(concept_rec_id, 1); - String json = "{}"; - - URL url = new URL(urlString + "/" + deposition_id + "/actions/newversion"); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - - conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token); - conn.setDoOutput(true); - conn.setRequestMethod("POST"); - - try (OutputStream os = conn.getOutputStream()) { - byte[] input = json.getBytes("utf-8"); - os.write(input, 0, input.length); - - } - - String body = getBody(conn); - - int responseCode = conn.getResponseCode(); - - conn.disconnect(); - if (!checkOKStatus(responseCode)) - throw new IOException("Unexpected code " + responseCode + body); - - ZenodoModel zenodoModel = new Gson().fromJson(body, ZenodoModel.class); - String latest_draft = zenodoModel.getLinks().getLatest_draft(); - deposition_id = latest_draft.substring(latest_draft.lastIndexOf("/") + 1); - bucket = getBucket(latest_draft); - - return responseCode; - - } - - /** - * To finish uploading a version or new deposition not published - * It sets the deposition_id and the bucket to be used - * - * - * @param deposition_id the deposition id of the not yet published upload - * concept_rec_id = 656930 - * @return response code - * @throws IOException - * @throws MissingConceptDoiException - */ - public int uploadOpenDeposition(String deposition_id) throws IOException, MissingConceptDoiException { - - this.deposition_id = deposition_id; - - String json = "{}"; - - URL url = new URL(urlString + "/" + deposition_id); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - - conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token); - conn.setRequestMethod("POST"); - conn.setDoOutput(true); - try (OutputStream os = conn.getOutputStream()) { - byte[] input = json.getBytes("utf-8"); - os.write(input, 0, input.length); - } - - String body = getBody(conn); - - int responseCode = conn.getResponseCode(); - conn.disconnect(); - - if (!checkOKStatus(responseCode)) - throw new IOException("Unexpected code " + responseCode + body); - - ZenodoModel zenodoModel = new Gson().fromJson(body, ZenodoModel.class); - bucket = zenodoModel.getLinks().getBucket(); - - return responseCode; - - } - - private void setDepositionId(String concept_rec_id, Integer page) throws IOException, MissingConceptDoiException { - - ZenodoModelList zenodoModelList = new Gson() - .fromJson(getPrevDepositions(String.valueOf(page)), ZenodoModelList.class); - - for (ZenodoModel zm : zenodoModelList) { - if (zm.getConceptrecid().equals(concept_rec_id)) { - deposition_id = zm.getId(); - return; - } - } - if (zenodoModelList.size() == 0) - throw new MissingConceptDoiException( - "The concept record id specified was missing in the list of depositions"); - setDepositionId(concept_rec_id, page + 1); - - } - - private String getPrevDepositions(String page) throws IOException { - - HttpUrl.Builder urlBuilder = HttpUrl.parse(urlString).newBuilder(); - urlBuilder.addQueryParameter("page", page); - - URL url = new URL(urlBuilder.build().toString()); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - conn.setRequestProperty(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()); - conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token); - conn.setDoOutput(true); - conn.setRequestMethod("GET"); - - String body = getBody(conn); - - int responseCode = conn.getResponseCode(); - - conn.disconnect(); - if (!checkOKStatus(responseCode)) - throw new IOException("Unexpected code " + responseCode + body); - - return body; - - } - - private String getBucket(String inputUurl) throws IOException { - - URL url = new URL(inputUurl); - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - conn.setRequestProperty(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()); - conn.setRequestProperty(HttpHeaders.AUTHORIZATION, "Bearer " + access_token); - conn.setDoOutput(true); - conn.setRequestMethod("GET"); - - String body = getBody(conn); - - int responseCode = conn.getResponseCode(); - - conn.disconnect(); - if (!checkOKStatus(responseCode)) - throw new IOException("Unexpected code " + responseCode + body); - - ZenodoModel zenodoModel = new Gson().fromJson(body, ZenodoModel.class); - - return zenodoModel.getLinks().getBucket(); - - } - -} diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Community.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Community.java deleted file mode 100644 index a02224383..000000000 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Community.java +++ /dev/null @@ -1,14 +0,0 @@ - -package eu.dnetlib.dhp.common.api.zenodo; - -public class Community { - private String identifier; - - public String getIdentifier() { - return identifier; - } - - public void setIdentifier(String identifier) { - this.identifier = identifier; - } -} diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Creator.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Creator.java deleted file mode 100644 index c14af55b6..000000000 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Creator.java +++ /dev/null @@ -1,47 +0,0 @@ - -package eu.dnetlib.dhp.common.api.zenodo; - -public class Creator { - private String affiliation; - private String name; - private String orcid; - - public String getAffiliation() { - return affiliation; - } - - public void setAffiliation(String affiliation) { - this.affiliation = affiliation; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getOrcid() { - return orcid; - } - - public void setOrcid(String orcid) { - this.orcid = orcid; - } - - public static Creator newInstance(String name, String affiliation, String orcid) { - Creator c = new Creator(); - if (name != null) { - c.name = name; - } - if (affiliation != null) { - c.affiliation = affiliation; - } - if (orcid != null) { - c.orcid = orcid; - } - - return c; - } -} diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/File.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/File.java deleted file mode 100644 index 509f444b9..000000000 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/File.java +++ /dev/null @@ -1,44 +0,0 @@ - -package eu.dnetlib.dhp.common.api.zenodo; - -import java.io.Serializable; - -public class File implements Serializable { - private String checksum; - private String filename; - private long filesize; - private String id; - - public String getChecksum() { - return checksum; - } - - public void setChecksum(String checksum) { - this.checksum = checksum; - } - - public String getFilename() { - return filename; - } - - public void setFilename(String filename) { - this.filename = filename; - } - - public long getFilesize() { - return filesize; - } - - public void setFilesize(long filesize) { - this.filesize = filesize; - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - -} diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Grant.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Grant.java deleted file mode 100644 index 476f1d9d8..000000000 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Grant.java +++ /dev/null @@ -1,23 +0,0 @@ - -package eu.dnetlib.dhp.common.api.zenodo; - -import java.io.Serializable; - -public class Grant implements Serializable { - private String id; - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public static Grant newInstance(String id) { - Grant g = new Grant(); - g.id = id; - - return g; - } -} diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Links.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Links.java deleted file mode 100644 index bdf8e5d2c..000000000 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Links.java +++ /dev/null @@ -1,92 +0,0 @@ - -package eu.dnetlib.dhp.common.api.zenodo; - -import java.io.Serializable; - -public class Links implements Serializable { - - private String bucket; - - private String discard; - - private String edit; - private String files; - private String html; - private String latest_draft; - private String latest_draft_html; - private String publish; - - private String self; - - public String getBucket() { - return bucket; - } - - public void setBucket(String bucket) { - this.bucket = bucket; - } - - public String getDiscard() { - return discard; - } - - public void setDiscard(String discard) { - this.discard = discard; - } - - public String getEdit() { - return edit; - } - - public void setEdit(String edit) { - this.edit = edit; - } - - public String getFiles() { - return files; - } - - public void setFiles(String files) { - this.files = files; - } - - public String getHtml() { - return html; - } - - public void setHtml(String html) { - this.html = html; - } - - public String getLatest_draft() { - return latest_draft; - } - - public void setLatest_draft(String latest_draft) { - this.latest_draft = latest_draft; - } - - public String getLatest_draft_html() { - return latest_draft_html; - } - - public void setLatest_draft_html(String latest_draft_html) { - this.latest_draft_html = latest_draft_html; - } - - public String getPublish() { - return publish; - } - - public void setPublish(String publish) { - this.publish = publish; - } - - public String getSelf() { - return self; - } - - public void setSelf(String self) { - this.self = self; - } -} diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Metadata.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Metadata.java deleted file mode 100644 index b161adb9b..000000000 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/Metadata.java +++ /dev/null @@ -1,153 +0,0 @@ - -package eu.dnetlib.dhp.common.api.zenodo; - -import java.io.Serializable; -import java.util.List; - -public class Metadata implements Serializable { - - private String access_right; - private List communities; - private List creators; - private String description; - private String doi; - private List grants; - private List keywords; - private String language; - private String license; - private PrereserveDoi prereserve_doi; - private String publication_date; - private List references; - private List related_identifiers; - private String title; - private String upload_type; - private String version; - - public String getUpload_type() { - return upload_type; - } - - public void setUpload_type(String upload_type) { - this.upload_type = upload_type; - } - - public String getVersion() { - return version; - } - - public void setVersion(String version) { - this.version = version; - } - - public String getAccess_right() { - return access_right; - } - - public void setAccess_right(String access_right) { - this.access_right = access_right; - } - - public List getCommunities() { - return communities; - } - - public void setCommunities(List communities) { - this.communities = communities; - } - - public List getCreators() { - return creators; - } - - public void setCreators(List creators) { - this.creators = creators; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getDoi() { - return doi; - } - - public void setDoi(String doi) { - this.doi = doi; - } - - public List getGrants() { - return grants; - } - - public void setGrants(List grants) { - this.grants = grants; - } - - public List getKeywords() { - return keywords; - } - - public void setKeywords(List keywords) { - this.keywords = keywords; - } - - public String getLanguage() { - return language; - } - - public void setLanguage(String language) { - this.language = language; - } - - public String getLicense() { - return license; - } - - public void setLicense(String license) { - this.license = license; - } - - public PrereserveDoi getPrereserve_doi() { - return prereserve_doi; - } - - public void setPrereserve_doi(PrereserveDoi prereserve_doi) { - this.prereserve_doi = prereserve_doi; - } - - public String getPublication_date() { - return publication_date; - } - - public void setPublication_date(String publication_date) { - this.publication_date = publication_date; - } - - public List getReferences() { - return references; - } - - public void setReferences(List references) { - this.references = references; - } - - public List getRelated_identifiers() { - return related_identifiers; - } - - public void setRelated_identifiers(List related_identifiers) { - this.related_identifiers = related_identifiers; - } - - public String getTitle() { - return title; - } - - public void setTitle(String title) { - this.title = title; - } -} diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/PrereserveDoi.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/PrereserveDoi.java deleted file mode 100644 index aa088ef31..000000000 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/PrereserveDoi.java +++ /dev/null @@ -1,25 +0,0 @@ - -package eu.dnetlib.dhp.common.api.zenodo; - -import java.io.Serializable; - -public class PrereserveDoi implements Serializable { - private String doi; - private String recid; - - public String getDoi() { - return doi; - } - - public void setDoi(String doi) { - this.doi = doi; - } - - public String getRecid() { - return recid; - } - - public void setRecid(String recid) { - this.recid = recid; - } -} diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/RelatedIdentifier.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/RelatedIdentifier.java deleted file mode 100644 index 15a349636..000000000 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/RelatedIdentifier.java +++ /dev/null @@ -1,43 +0,0 @@ - -package eu.dnetlib.dhp.common.api.zenodo; - -import java.io.Serializable; - -public class RelatedIdentifier implements Serializable { - private String identifier; - private String relation; - private String resource_type; - private String scheme; - - public String getIdentifier() { - return identifier; - } - - public void setIdentifier(String identifier) { - this.identifier = identifier; - } - - public String getRelation() { - return relation; - } - - public void setRelation(String relation) { - this.relation = relation; - } - - public String getResource_type() { - return resource_type; - } - - public void setResource_type(String resource_type) { - this.resource_type = resource_type; - } - - public String getScheme() { - return scheme; - } - - public void setScheme(String scheme) { - this.scheme = scheme; - } -} diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/ZenodoModel.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/ZenodoModel.java deleted file mode 100644 index 9843ea0f9..000000000 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/ZenodoModel.java +++ /dev/null @@ -1,118 +0,0 @@ - -package eu.dnetlib.dhp.common.api.zenodo; - -import java.io.Serializable; -import java.util.List; - -public class ZenodoModel implements Serializable { - - private String conceptrecid; - private String created; - - private List files; - private String id; - private Links links; - private Metadata metadata; - private String modified; - private String owner; - private String record_id; - private String state; - private boolean submitted; - private String title; - - public String getConceptrecid() { - return conceptrecid; - } - - public void setConceptrecid(String conceptrecid) { - this.conceptrecid = conceptrecid; - } - - public String getCreated() { - return created; - } - - public void setCreated(String created) { - this.created = created; - } - - public List getFiles() { - return files; - } - - public void setFiles(List files) { - this.files = files; - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public Links getLinks() { - return links; - } - - public void setLinks(Links links) { - this.links = links; - } - - public Metadata getMetadata() { - return metadata; - } - - public void setMetadata(Metadata metadata) { - this.metadata = metadata; - } - - public String getModified() { - return modified; - } - - public void setModified(String modified) { - this.modified = modified; - } - - public String getOwner() { - return owner; - } - - public void setOwner(String owner) { - this.owner = owner; - } - - public String getRecord_id() { - return record_id; - } - - public void setRecord_id(String record_id) { - this.record_id = record_id; - } - - public String getState() { - return state; - } - - public void setState(String state) { - this.state = state; - } - - public boolean isSubmitted() { - return submitted; - } - - public void setSubmitted(boolean submitted) { - this.submitted = submitted; - } - - public String getTitle() { - return title; - } - - public void setTitle(String title) { - this.title = title; - } -} diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/ZenodoModelList.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/ZenodoModelList.java deleted file mode 100644 index b3b150714..000000000 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/api/zenodo/ZenodoModelList.java +++ /dev/null @@ -1,7 +0,0 @@ - -package eu.dnetlib.dhp.common.api.zenodo; - -import java.util.ArrayList; - -public class ZenodoModelList extends ArrayList { -} diff --git a/dhp-common/src/test/java/eu/dnetlib/dhp/common/api/ZenodoAPIClientTest.java b/dhp-common/src/test/java/eu/dnetlib/dhp/common/api/ZenodoAPIClientTest.java deleted file mode 100644 index 5dbe3b75b..000000000 --- a/dhp-common/src/test/java/eu/dnetlib/dhp/common/api/ZenodoAPIClientTest.java +++ /dev/null @@ -1,128 +0,0 @@ - -package eu.dnetlib.dhp.common.api; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; - -import org.apache.commons.io.IOUtils; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -@Disabled -class ZenodoAPIClientTest { - - private final String URL_STRING = "https://sandbox.zenodo.org/api/deposit/depositions"; - private final String ACCESS_TOKEN = "OzzOsyucEIHxCEfhlpsMo3myEiwpCza3trCRL7ddfGTAK9xXkIP2MbXd6Vg4"; - - private final String CONCEPT_REC_ID = "657113"; - - private final String depositionId = "674915"; - - @Test - void testUploadOldDeposition() throws IOException, MissingConceptDoiException { - ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING, - ACCESS_TOKEN); - Assertions.assertEquals(200, client.uploadOpenDeposition(depositionId)); - - File file = new File(getClass() - .getResource("/eu/dnetlib/dhp/common/api/COVID-19.json.gz") - .getPath()); - - InputStream is = new FileInputStream(file); - - Assertions.assertEquals(200, client.uploadIS(is, "COVID-19.json.gz")); - - String metadata = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/common/api/metadata.json")); - - Assertions.assertEquals(200, client.sendMretadata(metadata)); - - Assertions.assertEquals(202, client.publish()); - - } - - @Test - void testNewDeposition() throws IOException { - - ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING, - ACCESS_TOKEN); - Assertions.assertEquals(201, client.newDeposition()); - - File file = new File(getClass() - .getResource("/eu/dnetlib/dhp/common/api/newVersion") - .getPath()); - - InputStream is = new FileInputStream(file); - -// Assertions.assertEquals(200, client.uploadIS(is, "COVID-19.json.gz")); - - String metadata = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/common/api/metadata.json")); - - Assertions.assertEquals(200, client.sendMretadata(metadata)); - - // Assertions.assertEquals(202, client.publish()); - - } - - @Test - void testNewVersionNewName() throws IOException, MissingConceptDoiException { - - ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING, - ACCESS_TOKEN); - - Assertions.assertEquals(201, client.newVersion(CONCEPT_REC_ID)); - - File file = new File(getClass() - .getResource("/eu/dnetlib/dhp/common/api/newVersion") - .getPath()); - - InputStream is = new FileInputStream(file); - - Assertions.assertEquals(200, client.uploadIS(is, "newVersion_deposition")); - - Assertions.assertEquals(202, client.publish()); - - } - - @Test - void testNewVersionOldName() throws IOException, MissingConceptDoiException { - - ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING, - ACCESS_TOKEN); - - Assertions.assertEquals(201, client.newVersion(CONCEPT_REC_ID)); - - File file = new File(getClass() - .getResource("/eu/dnetlib/dhp/common/api/newVersion2") - .getPath()); - - InputStream is = new FileInputStream(file); - - Assertions.assertEquals(200, client.uploadIS(is, "newVersion_deposition")); - - Assertions.assertEquals(202, client.publish()); - - } - - @Test - void depositBigFile() throws MissingConceptDoiException, IOException { - ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING, - ACCESS_TOKEN); - - Assertions.assertEquals(201, client.newDeposition()); - - File file = new File("/Users/miriam.baglioni/Desktop/EOSC_DUMP/publication.tar"); -// File file = new File(getClass() -// .getResource("/eu/dnetlib/dhp/common/api/newVersion2") -// .getPath()); - - InputStream is = new FileInputStream(file); - - Assertions.assertEquals(200, client.uploadIS(is, "newVersion_deposition")); - - // Assertions.assertEquals(202, client.publish()); - } - -} diff --git a/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java b/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java index 12c96500e..cfd9acd70 100644 --- a/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java +++ b/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java @@ -18,7 +18,6 @@ package eu.dnetlib.pace.util; * See the License for the specific language governing permissions and * limitations under the License. */ - /* * Diff Match and Patch * Copyright 2018 The diff-match-patch Authors. diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java index 43d5e7e98..c1dda1ddc 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java @@ -9,8 +9,6 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; -import javax.management.Query; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,7 +22,6 @@ import eu.dnetlib.dhp.bulktag.community.CommunityConfiguration; import eu.dnetlib.dhp.bulktag.community.Provider; import eu.dnetlib.dhp.bulktag.criteria.VerbResolver; import eu.dnetlib.dhp.bulktag.criteria.VerbResolverFactory; -import eu.dnetlib.dhp.resulttocommunityfromorganization.SparkResultToCommunityFromOrganizationJob; /** * @author miriam.baglioni diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/api/QueryCommunityAPITest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/api/QueryCommunityAPITest.java deleted file mode 100644 index e3c2a5f45..000000000 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/api/QueryCommunityAPITest.java +++ /dev/null @@ -1,115 +0,0 @@ - -package eu.dnetlib.dhp.api; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; - -import eu.dnetlib.dhp.api.model.CommunityEntityMap; -import eu.dnetlib.dhp.api.model.CommunityModel; -import eu.dnetlib.dhp.api.model.CommunitySummary; -import eu.dnetlib.dhp.api.model.DatasourceList; -import eu.dnetlib.dhp.bulktag.community.Community; -import eu.dnetlib.dhp.bulktag.community.CommunityConfiguration; - -/** - * @author miriam.baglioni - * @Date 06/10/23 - */ -public class QueryCommunityAPITest { - - @Test - void communityList() throws Exception { - String body = QueryCommunityAPI.communities(true); - new ObjectMapper() - .readValue(body, CommunitySummary.class) - .forEach(p -> { - try { - System.out.println(new ObjectMapper().writeValueAsString(p)); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - }); - } - - @Test - void community() throws Exception { - String id = "dh-ch"; - String body = QueryCommunityAPI.community(id, true); - System.out - .println( - new ObjectMapper() - .writeValueAsString( - new ObjectMapper() - .readValue(body, CommunityModel.class))); - } - - @Test - void communityDatasource() throws Exception { - String id = "dh-ch"; - String body = QueryCommunityAPI.communityDatasource(id, true); - new ObjectMapper() - .readValue(body, DatasourceList.class) - .forEach(ds -> { - try { - System.out.println(new ObjectMapper().writeValueAsString(ds)); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - }); - ; - } - - @Test - void validCommunities() throws Exception { - CommunityConfiguration cc = Utils.getCommunityConfiguration(true); - System.out.println(cc.getCommunities().keySet()); - Community community = cc.getCommunities().get("aurora"); - Assertions.assertEquals(0, community.getSubjects().size()); - Assertions.assertEquals(null, community.getConstraints()); - Assertions.assertEquals(null, community.getRemoveConstraints()); - Assertions.assertEquals(2, community.getZenodoCommunities().size()); - Assertions - .assertTrue( - community.getZenodoCommunities().stream().anyMatch(c -> c.equals("aurora-universities-network"))); - Assertions - .assertTrue(community.getZenodoCommunities().stream().anyMatch(c -> c.equals("university-of-innsbruck"))); - Assertions.assertEquals(35, community.getProviders().size()); - Assertions - .assertEquals( - 35, community.getProviders().stream().filter(p -> p.getSelectionConstraints() == null).count()); - - } - - @Test - void eutopiaCommunityConfiguration() throws Exception { - CommunityConfiguration cc = Utils.getCommunityConfiguration(true); - System.out.println(cc.getCommunities().keySet()); - Community community = cc.getCommunities().get("eutopia"); - community.getProviders().forEach(p -> System.out.println(p.getOpenaireId())); - } - - @Test - void getCommunityProjects() throws Exception { - CommunityEntityMap projectMap = Utils.getCommunityProjects(true); - - Assertions - .assertTrue( - projectMap - .keySet() - .stream() - .allMatch(k -> k.startsWith("40|"))); - - System.out.println(projectMap); - } - - @Test - void getCommunityOrganizations() throws Exception { - CommunityEntityMap organizationMap = Utils.getCommunityOrganization(true); - Assertions.assertTrue(organizationMap.keySet().stream().allMatch(k -> k.startsWith("20|"))); - - } - -} From 2090003ea96dee43c3cd6c90f50f6d4a2b39a869 Mon Sep 17 00:00:00 2001 From: Serafeim Chatzopoulos Date: Thu, 26 Oct 2023 13:47:06 -0700 Subject: [PATCH 084/148] Adjust tests to new WF input params --- .../PrepareAffiliationRelationsTest.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelationsTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelationsTest.java index ed8e5fe0d..e2639996c 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelationsTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelationsTest.java @@ -74,17 +74,22 @@ public class PrepareAffiliationRelationsTest { @Test void testMatch() throws Exception { - String affiliationRelationsPath = getClass() + String crossrefAffiliationRelationPath = getClass() .getResource("/eu/dnetlib/dhp/actionmanager/bipaffiliations/doi_to_ror.json") .getPath(); + String pubmedAffiliationRelationsPath = getClass() + .getResource("/eu/dnetlib/dhp/actionmanager/bipaffiliations/doi_to_ror.json") + .getPath(); + String outputPath = workingDir.toString() + "/actionSet"; PrepareAffiliationRelations .main( new String[] { "-isSparkSessionManaged", Boolean.FALSE.toString(), - "-inputPath", affiliationRelationsPath, + "-crossrefInputPath", crossrefAffiliationRelationPath, + "-pubmedInputPath", pubmedAffiliationRelationsPath, "-outputPath", outputPath }); @@ -101,7 +106,7 @@ public class PrepareAffiliationRelationsTest { // ); // } // count the number of relations - assertEquals(20, tmp.count()); + assertEquals(40, tmp.count()); Dataset dataset = spark.createDataset(tmp.rdd(), Encoders.bean(Relation.class)); dataset.createOrReplaceTempView("result"); @@ -112,7 +117,7 @@ public class PrepareAffiliationRelationsTest { // verify that we have equal number of bi-directional relations Assertions .assertEquals( - 10, execVerification + 20, execVerification .filter( "relClass='" + ModelConstants.HAS_AUTHOR_INSTITUTION + "'") .collectAsList() @@ -120,7 +125,7 @@ public class PrepareAffiliationRelationsTest { Assertions .assertEquals( - 10, execVerification + 20, execVerification .filter( "relClass='" + ModelConstants.IS_AUTHOR_INSTITUTION_OF + "'") .collectAsList() From 262d7c581b56244cc3eab179bcb3ad82ba6174ae Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 31 Oct 2023 14:34:10 +0100 Subject: [PATCH 085/148] [graph cleaning] implemented further suggestions from https://support.openaire.eu/issues/8898 --- .../oaf/utils/GraphCleaningFunctions.java | 95 +++++++++++++++++-- .../graph/clean/CleanGraphSparkJobTest.java | 15 +++ .../clean/graph/publication/publication.json | 2 +- 3 files changed, 103 insertions(+), 9 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java index b4402a2fb..e8bada028 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java @@ -3,16 +3,20 @@ package eu.dnetlib.dhp.schema.oaf.utils; import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.getProvenance; +import java.net.MalformedURLException; +import java.net.URL; import java.time.LocalDate; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.util.*; import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; +import org.apache.cxf.common.util.UrlUtils; import com.github.sisyphsu.dateparser.DateParserUtils; import com.google.common.collect.Lists; @@ -23,6 +27,7 @@ import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.oaf.*; import me.xuender.unidecode.Unidecode; +import sun.awt.HKSCS; public class GraphCleaningFunctions extends CleaningFunctions { @@ -36,6 +41,13 @@ public class GraphCleaningFunctions extends CleaningFunctions { public static final int TITLE_FILTER_RESIDUAL_LENGTH = 5; private static final String NAME_CLEANING_REGEX = "[\\r\\n\\t\\s]+"; + + private static final Set INVALID_AUTHOR_NAMES = new HashSet<>(); + + private static final Set INVALID_URLS = new HashSet<>(); + + private static final Set INVALID_URL_HOSTS = new HashSet<>(); + private static final HashSet PEER_REVIEWED_TYPES = new HashSet<>(); static { @@ -48,6 +60,47 @@ public class GraphCleaningFunctions extends CleaningFunctions { PEER_REVIEWED_TYPES.add("Thesis"); PEER_REVIEWED_TYPES.add("Bachelor thesis"); PEER_REVIEWED_TYPES.add("Conference object"); + + INVALID_AUTHOR_NAMES.add("(:null)"); + INVALID_AUTHOR_NAMES.add("(:unap)"); + INVALID_AUTHOR_NAMES.add("(:tba)"); + INVALID_AUTHOR_NAMES.add("(:unas)"); + INVALID_AUTHOR_NAMES.add("(:unav)"); + INVALID_AUTHOR_NAMES.add("(:unkn)"); + INVALID_AUTHOR_NAMES.add("(:unkn) unknown"); + INVALID_AUTHOR_NAMES.add(":none"); + INVALID_AUTHOR_NAMES.add(":null"); + INVALID_AUTHOR_NAMES.add(":unas"); + INVALID_AUTHOR_NAMES.add(":unav"); + INVALID_AUTHOR_NAMES.add(":unkn"); + INVALID_AUTHOR_NAMES.add("[autor desconocido]"); + INVALID_AUTHOR_NAMES.add("[s. n.]"); + INVALID_AUTHOR_NAMES.add("[s.n]"); + INVALID_AUTHOR_NAMES.add("[unknown]"); + INVALID_AUTHOR_NAMES.add("anonymous"); + INVALID_AUTHOR_NAMES.add("n.n."); + INVALID_AUTHOR_NAMES.add("nn"); + INVALID_AUTHOR_NAMES.add("no name supplied"); + INVALID_AUTHOR_NAMES.add("none"); + INVALID_AUTHOR_NAMES.add("none available"); + INVALID_AUTHOR_NAMES.add("not available not available"); + INVALID_AUTHOR_NAMES.add("null &na;"); + INVALID_AUTHOR_NAMES.add("null anonymous"); + INVALID_AUTHOR_NAMES.add("unbekannt"); + INVALID_AUTHOR_NAMES.add("unknown"); + + INVALID_URL_HOSTS.add("creativecommons.org"); + INVALID_URL_HOSTS.add("www.academia.edu"); + INVALID_URL_HOSTS.add("academia.edu"); + INVALID_URL_HOSTS.add("researchgate.net"); + INVALID_URL_HOSTS.add("www.researchgate.net"); + + INVALID_URLS.add("http://repo.scoap3.org/api"); + INVALID_URLS.add("http://ora.ox.ac.uk/objects/uuid:"); + INVALID_URLS.add("http://ntur.lib.ntu.edu.tw/news/agent_contract.pdf"); + INVALID_URLS.add("https://media.springer.com/full/springer-instructions-for-authors-assets/pdf/SN_BPF_EN.pdf"); + INVALID_URLS.add("http://www.tobaccoinduceddiseases.org/dl/61aad426c96519bea4040a374c6a6110/"); + INVALID_URLS.add("https://www.bilboard.nl/verenigingsbladen/bestuurskundige-berichten"); } public static T cleanContext(T value, String contextId, String verifyParam) { @@ -558,6 +611,15 @@ public class GraphCleaningFunctions extends CleaningFunctions { ModelConstants.DATASET_RESULTTYPE_CLASSID.equals(r.getResulttype().getClassid()))) { i.setFulltext(null); } + if (Objects.nonNull(i.getUrl())) { + i + .setUrl( + i + .getUrl() + .stream() + .filter(GraphCleaningFunctions::urlFilter) + .collect(Collectors.toList())); + } } } if (Objects.isNull(r.getBestaccessright()) @@ -580,8 +642,7 @@ public class GraphCleaningFunctions extends CleaningFunctions { .getAuthor() .stream() .filter(Objects::nonNull) - .filter(a -> StringUtils.isNotBlank(a.getFullname())) - .filter(a -> StringUtils.isNotBlank(a.getFullname().replaceAll("[\\W]", ""))) + .filter(GraphCleaningFunctions::isValidAuthorName) .map(GraphCleaningFunctions::cleanupAuthor) .collect(Collectors.toList())); @@ -739,12 +800,30 @@ public class GraphCleaningFunctions extends CleaningFunctions { // HELPERS private static boolean isValidAuthorName(Author a) { - return !Stream - .of(a.getFullname(), a.getName(), a.getSurname()) - .filter(s -> s != null && !s.isEmpty()) - .collect(Collectors.joining("")) - .toLowerCase() - .matches(INVALID_AUTHOR_REGEX); + return StringUtils.isNotBlank(a.getFullname()) && + StringUtils.isNotBlank(a.getFullname().replaceAll("[\\W]", "")) && + !INVALID_AUTHOR_NAMES.contains(StringUtils.lowerCase(a.getFullname()).trim()) && + !Stream + .of(a.getFullname(), a.getName(), a.getSurname()) + .filter(StringUtils::isNotBlank) + .collect(Collectors.joining("")) + .toLowerCase() + .matches(INVALID_AUTHOR_REGEX); + } + + private static boolean urlFilter(String u) { + try { + final URL url = new URL(u); + if (StringUtils.isBlank(url.getPath()) || "/".equals(url.getPath())) { + return false; + } + if (INVALID_URL_HOSTS.contains(url.getHost())) { + return false; + } + return !INVALID_URLS.contains(url.toString()); + } catch (MalformedURLException ex) { + return false; + } } private static List processPidCleaning(List pids) { diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleanGraphSparkJobTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleanGraphSparkJobTest.java index 65182108e..a1a4c29cd 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleanGraphSparkJobTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleanGraphSparkJobTest.java @@ -251,9 +251,19 @@ public class CleanGraphSparkJobTest { .filter(String.format("id = '%s'", id)) .first(); + final Set invalidURLs = new HashSet<>(); + invalidURLs.add("http://academia.edu/abcd"); + invalidURLs.add("http://repo.scoap3.org/api"); + invalidURLs.add("http://hdl.handle.net/"); + assertNull(p_in.getBestaccessright()); assertTrue(p_in instanceof Result); assertTrue(p_in instanceof Publication); + assertNotNull(p_in.getAuthor()); + assertEquals(14, p_in.getAuthor().size()); + assertNotNull(p_in.getInstance()); + assertNotNull(p_in.getInstance().get(0)); + assertEquals(3, p_in.getInstance().get(0).getUrl().stream().filter(invalidURLs::contains).count()); new CleanGraphSparkJob( args( @@ -273,6 +283,9 @@ public class CleanGraphSparkJobTest { assertNull(p.getPublisher()); + assertNotNull(p.getAuthor()); + assertEquals(12, p.getAuthor().size()); + assertEquals("und", p.getLanguage().getClassid()); assertEquals("Undetermined", p.getLanguage().getClassname()); @@ -364,6 +377,8 @@ public class CleanGraphSparkJobTest { .stream() .anyMatch(s -> s.getValue().equals("10.1009/qwerty"))); + assertTrue(p.getInstance().get(0).getUrl().stream().noneMatch(invalidURLs::contains)); + assertNotNull(p.getSubject()); List fos_subjects = p diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/graph/publication/publication.json b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/graph/publication/publication.json index e5d7c88db..e2a2b9449 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/graph/publication/publication.json +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/graph/publication/publication.json @@ -1,4 +1,4 @@ -{"id":"50|CSC_________::2250a70c903c6ac6e4c01438259e9375","author":[{"affiliation":[],"fullname":"Brien, Tom","name":"Tom","pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"ORCID12","classname":"ORCID12","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"0000-0001-9613-6639"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"ORCID12","classname":"ORCID12","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"https://orcid.org/0000-0001-9613-6639"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"sysimport:crosswalk:entityregistry","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"orcid","classname":"ORCID12","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"0000-0001-9613-6639"}],"rank":1,"surname":"Brien"},{"affiliation":[],"fullname":"Ade, Peter","name":"Peter","pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"xyz","classname":"XYZ","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"qwerty"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"","schemename":""},"value":"asdasd"}],"rank":2,"surname":"Ade"},{"affiliation":[],"fullname":"Barry, Peter S.","name":"Peter S.","pid":null,"rank":3,"surname":"Barry"},{"affiliation":[],"fullname":"Dunscombe, Chris J.","name":"Chris J.","pid":[],"rank":4,"surname":"Dunscombe"},{"affiliation":[],"fullname":"Leadley, David R.","name":"David R.","pid":[],"rank":5,"surname":"Leadley"},{"affiliation":[],"fullname":"Morozov, Dmitry V.","name":"Dmitry V.","pid":[],"rank":6,"surname":"Morozov"},{"affiliation":[],"fullname":"Myronov, Maksym","name":"Maksym","pid":[],"rank":7,"surname":"Myronov"},{"affiliation":[],"fullname":"Parker, Evan","name":"Evan","pid":[],"rank":8,"surname":"Parker"},{"affiliation":[],"fullname":"Prest, Martin J.","name":"Martin J.","pid":[],"rank":9,"surname":"Prest"},{"affiliation":[],"fullname":"Prunnila, Mika","name":"Mika","pid":[],"rank":10,"surname":"Prunnila"},{"affiliation":[],"fullname":"Sudiwala, Rashmi V.","name":"Rashmi V.","pid":[],"rank":11,"surname":"Sudiwala"},{"affiliation":[],"fullname":"Whall, Terry E.","name":"Terry E.","pid":[],"rank":12,"surname":"Whall"},{"affiliation":[],"fullname":"Mauskopf","name":"","pid":[],"rank":13,"surname":""},{"affiliation":[],"fullname":" P. D. ","name":"","pid":[],"rank":14,"surname":""}],"bestaccessright":null,"publisher":{"value":null},"collectedfrom":[{"key":"10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747","value":"VIRTA"}],"context":[],"contributor":[],"country":[{"classid":"DE","classname":"DE","schemeid":"dnet:countries","schemename":"dnet:countries"}],"coverage":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"7 oct 1970"},"dateofcollection":"","dateoftransformation":"2020-04-22T12:34:08.009Z","description":[],"externalReference":[],"extraInfo":[],"format":[],"fulltext":[],"instance":[{"pid":[{"dataInfo":null,"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1007/s109090161569x"},{"dataInfo":null,"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1008/abcd"}],"alternateIdentifier":[{"dataInfo":null,"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1007/s109090161569x"},{"dataInfo":null,"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1009/qwerty"}],"accessright":{"classid":"CLOSED","classname":"CLOSED","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":{"key":"10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747","value":"VIRTA"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2016-01-01"},"distributionlocation":"","hostedby":{"key":"10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747","value":"VIRTA"},"instancetype":{"classid":"Comment/debate","classname":"Comment/debate","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"url":["http://juuli.fi/Record/0275158616","http://dx.doi.org/10.1007/s109090161569x"]},{"pid":[{"dataInfo":null,"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1002/s21010127267xy"},{"dataInfo":null,"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1008/abcd"}],"alternateIdentifier":[{"dataInfo":null,"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1007/s109090161569x"},{"dataInfo":null,"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1009/qwerty"}],"accessright":{"classid":"CLOSED","classname":"CLOSED","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":{"key":"10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747","value":"VIRTA"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2016-01-01"},"distributionlocation":"","hostedby":{"key":"10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747","value":"VIRTA"},"instancetype":{"classid":"Model","classname":"Model","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"url":["http://dx.doi.org/10.1002/s21010127267xy"]},{"pid":[{"dataInfo":null,"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1002/s21010127267xy"},{"dataInfo":null,"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1008/abcd"}],"alternateIdentifier":[{"dataInfo":null,"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1007/s109090161569x"},{"dataInfo":null,"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1009/qwerty"}],"accessright":{"classid":"CLOSED","classname":"CLOSED","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":{"key":"10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747","value":"VIRTA"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2016-01-01"},"distributionlocation":"","hostedby":{"key":"10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747","value":"VIRTA"},"instancetype":{"classid":"xyz","classname":"xyz","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"url":["http://dx.doi.org/10.1002/t32121238378t"]}],"journal":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"edition":"","ep":" 7","iss":"9 March","issnLinking":"","issnOnline":"","issnPrinted":"0022-2291","name":"Journal of Low Temperature Physics - Early Acces","sp":"1 ","vol":""},"language":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastupdatetimestamp":1591283286319,"oaiprovenance":{"originDescription":{"altered":true,"baseURL":"https%3A%2F%2Fvirta-jtp.csc.fi%2Fapi%2Fcerif","datestamp":"2019-07-30","harvestDate":"2020-04-22T11:04:38.685Z","identifier":"oai:virta-jtp.csc.fi:Publications/0275158616","metadataNamespace":""}},"originalId":["CSC_________::2250a70c903c6ac6e4c01438259e9375"],"pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1007/s109090161569x"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1007/s109090161569x"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":""}],"relevantdate":[],"resourcetype":{"classid":"0001","classname":"0001","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"source":[],"subject":[{"dataInfo":{"provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"deletedbyinference":false,"inferred":false,"inferenceprovenance":"","invisible":false,"trust":"0.9"},"qualifier":{"classid":"FOS","classname":"Fields of Science and Technology classification","schemeid":"dnet:result_subject","schemename":"dnet:result_subject"},"value":"In Situ Hybridization"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"value":"ta213"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"FOS: Mathematics"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"FOS: Computer and information sciences"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"0101 mathematics"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"subject:fos","classname":"subject:fos","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"0101 mathematics"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"value":"slot antennas"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"value":"strained silicon"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"value":"cold electron bolometers"},{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:actionset","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"FOS","classname":"Fields of Science and Technology classification","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Avicennia"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"value":"measure noise"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"value":"noise equivalent power"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"value":"optical characterisation"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"value":"optical response"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"value":"photon noise"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"value":"silicon absorbers"}],"title":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"Optical response of strained- and unstrained-silicon cold-electron bolometers test"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"test test 123 test"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"omic"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"｢マキャベリ的知性と心の理論の進化論｣ リチャード・バーン， アンドリュー・ホワイトゥン 編／藤田和生， 山下博志， 友永雅巳 監訳"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"-"}]} +{"id":"50|CSC_________::2250a70c903c6ac6e4c01438259e9375","author":[{"affiliation":[],"fullname":"Brien, Tom","name":"Tom","pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"ORCID12","classname":"ORCID12","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"0000-0001-9613-6639"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"ORCID12","classname":"ORCID12","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"https://orcid.org/0000-0001-9613-6639"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:entityregistry","classname":"sysimport:crosswalk:entityregistry","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"orcid","classname":"ORCID12","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"0000-0001-9613-6639"}],"rank":1,"surname":"Brien"},{"affiliation":[],"fullname":"Ade, Peter","name":"Peter","pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"xyz","classname":"XYZ","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"qwerty"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"","schemename":""},"value":"asdasd"}],"rank":2,"surname":"Ade"},{"affiliation":[],"fullname":"Barry, Peter S.","name":"Peter S.","pid":null,"rank":3,"surname":"Barry"},{"affiliation":[],"fullname":"Dunscombe, Chris J.","name":"Chris J.","pid":[],"rank":4,"surname":"Dunscombe"},{"affiliation":[],"fullname":"Leadley, David R.","name":"David R.","pid":[],"rank":5,"surname":"Leadley"},{"affiliation":[],"fullname":"Morozov, Dmitry V.","name":"Dmitry V.","pid":[],"rank":6,"surname":"Morozov"},{"affiliation":[],"fullname":"Myronov, Maksym","name":"Maksym","pid":[],"rank":7,"surname":"Myronov"},{"affiliation":[],"fullname":"Parker, Evan","name":"Evan","pid":[],"rank":8,"surname":"Parker"},{"affiliation":[],"fullname":"Prest, Martin J.","name":"Martin J.","pid":[],"rank":9,"surname":"Prest"},{"affiliation":[],"fullname":"Prunnila, Mika","name":"Mika","pid":[],"rank":10,"surname":"Prunnila"},{"affiliation":[],"fullname":"Sudiwala, Rashmi V.","name":"Rashmi V.","pid":[],"rank":11,"surname":"Sudiwala"},{"affiliation":[],"fullname":"Whall, Terry E.","name":"Terry E.","pid":[],"rank":12,"surname":"Whall"},{"affiliation":[],"fullname":" - ","name":"","pid":[],"rank":13,"surname":""},{"affiliation":[],"fullname":" :none","name":"","pid":[],"rank":14,"surname":""}],"bestaccessright":null,"publisher":{"value":null},"collectedfrom":[{"key":"10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747","value":"VIRTA"}],"context":[],"contributor":[],"country":[{"classid":"DE","classname":"DE","schemeid":"dnet:countries","schemename":"dnet:countries"}],"coverage":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"7 oct 1970"},"dateofcollection":"","dateoftransformation":"2020-04-22T12:34:08.009Z","description":[],"externalReference":[],"extraInfo":[],"format":[],"fulltext":[],"instance":[{"pid":[{"dataInfo":null,"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1007/s109090161569x"},{"dataInfo":null,"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1008/abcd"}],"alternateIdentifier":[{"dataInfo":null,"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1007/s109090161569x"},{"dataInfo":null,"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1009/qwerty"}],"accessright":{"classid":"CLOSED","classname":"CLOSED","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":{"key":"10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747","value":"VIRTA"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2016-01-01"},"distributionlocation":"","hostedby":{"key":"10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747","value":"VIRTA"},"instancetype":{"classid":"Comment/debate","classname":"Comment/debate","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"url":["http://juuli.fi/Record/0275158616","http://dx.doi.org/10.1007/s109090161569x","http://academia.edu/abcd","http://repo.scoap3.org/api","http://hdl.handle.net/"]},{"pid":[{"dataInfo":null,"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1002/s21010127267xy"},{"dataInfo":null,"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1008/abcd"}],"alternateIdentifier":[{"dataInfo":null,"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1007/s109090161569x"},{"dataInfo":null,"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1009/qwerty"}],"accessright":{"classid":"CLOSED","classname":"CLOSED","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":{"key":"10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747","value":"VIRTA"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2016-01-01"},"distributionlocation":"","hostedby":{"key":"10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747","value":"VIRTA"},"instancetype":{"classid":"Model","classname":"Model","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"url":["http://dx.doi.org/10.1002/s21010127267xy"]},{"pid":[{"dataInfo":null,"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1002/s21010127267xy"},{"dataInfo":null,"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1008/abcd"}],"alternateIdentifier":[{"dataInfo":null,"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1007/s109090161569x"},{"dataInfo":null,"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1009/qwerty"}],"accessright":{"classid":"CLOSED","classname":"CLOSED","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":{"key":"10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747","value":"VIRTA"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2016-01-01"},"distributionlocation":"","hostedby":{"key":"10|CSC_________::a2b9ce8435390bcbfc05f3cae3948747","value":"VIRTA"},"instancetype":{"classid":"xyz","classname":"xyz","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"url":["http://dx.doi.org/10.1002/t32121238378t"]}],"journal":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"edition":"","ep":" 7","iss":"9 March","issnLinking":"","issnOnline":"","issnPrinted":"0022-2291","name":"Journal of Low Temperature Physics - Early Acces","sp":"1 ","vol":""},"language":{"classid":"UNKNOWN","classname":"UNKNOWN","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastupdatetimestamp":1591283286319,"oaiprovenance":{"originDescription":{"altered":true,"baseURL":"https%3A%2F%2Fvirta-jtp.csc.fi%2Fapi%2Fcerif","datestamp":"2019-07-30","harvestDate":"2020-04-22T11:04:38.685Z","identifier":"oai:virta-jtp.csc.fi:Publications/0275158616","metadataNamespace":""}},"originalId":["CSC_________::2250a70c903c6ac6e4c01438259e9375"],"pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1007/s109090161569x"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.1007/s109090161569x"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":""}],"relevantdate":[],"resourcetype":{"classid":"0001","classname":"0001","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"source":[],"subject":[{"dataInfo":{"provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"deletedbyinference":false,"inferred":false,"inferenceprovenance":"","invisible":false,"trust":"0.9"},"qualifier":{"classid":"FOS","classname":"Fields of Science and Technology classification","schemeid":"dnet:result_subject","schemename":"dnet:result_subject"},"value":"In Situ Hybridization"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"value":"ta213"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"FOS: Mathematics"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"FOS: Computer and information sciences"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"0101 mathematics"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"subject:fos","classname":"subject:fos","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"0101 mathematics"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"value":"slot antennas"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"value":"strained silicon"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"value":"cold electron bolometers"},{"dataInfo":{"deletedbyinference":false,"inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:actionset","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"FOS","classname":"Fields of Science and Technology classification","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Avicennia"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"value":"measure noise"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"value":"noise equivalent power"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"value":"optical characterisation"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"value":"optical response"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"value":"photon noise"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"value":"silicon absorbers"}],"title":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"Optical response of strained- and unstrained-silicon cold-electron bolometers test"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"test test 123 test"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"omic"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"｢マキャベリ的知性と心の理論の進化論｣ リチャード・バーン， アンドリュー・ホワイトゥン 編／藤田和生， 山下博志， 友永雅巳 監訳"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"-"}]} {"id":"50|doi_________::b0baa0eb88a5788f0b8815560d2a32f2","context": [], "dataInfo": {"invisible": false, "trust": "0.9", "provenanceaction": {"classid": "sysimport:actionset", "classname": "sysimport:actionset", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "inferred": false, "deletedbyinference": false}, "resourcetype": {"classid": "0001", "classname": "Article", "schemeid": "dnet:publication_resource", "schemename": "dnet:publication_resource"}, "pid": [{"qualifier": {"classid": "doi", "classname": "doi", "schemeid": "dnet:pid_types", "schemename": "dnet:pid_types"}, "value": "10.1097/00132586-197308000-00003"}], "contributor": [], "bestaccessright": {"classid": "UNKNOWN", "classname": "not available", "schemeid": "dnet:access_modes", "schemename": "dnet:access_modes"}, "relevantdate": [{"qualifier": {"classid": "created", "classname": "created", "schemeid": "dnet:dataCite_date", "schemename": "dnet:dataCite_date"}, "value": "2006-11-06T11:36:37Z"}], "collectedfrom": [{"key": "10|openaire____::081b82f96300b6a6e3d282bad31cb6e2", "value": "Crossref"}], "subject": [], "lastupdatetimestamp": 1620353302565, "author": [{"fullname": "N. S. AGRUSS", "surname": "AGRUSS", "name": "N. S.", "rank": 1}, {"fullname": "E. Y. ROSIN", "surname": "ROSIN", "name": "E. Y.", "rank": 2}, {"fullname": "R. J. ADOLPH", "surname": "ADOLPH", "name": "R. J.", "rank": 3}, {"fullname": "N. O. FOWLER", "surname": "FOWLER", "name": "N. O.", "rank": 4}], "instance": [{"hostedby": {"key": "10|issn___print::b8cee613d4f898f8c03956d57ea69be2", "value": "Survey of Anesthesiology"}, "url": ["https://doi.org/10.1097/00132586-197308000-00003"], "pid": [{"qualifier": {"classid": "doi", "classname": "doi", "schemeid": "dnet:pid_types", "schemename": "dnet:pid_types"}, "value": "10.1097/00132586-197308000-00003"}], "dateofacceptance": {"value": "2006-11-06T11:36:37Z"}, "collectedfrom": {"key": "10|openaire____::081b82f96300b6a6e3d282bad31cb6e2", "value": "Crossref"}, "accessright": {"classid": "UNKNOWN", "classname": "not available", "schemeid": "dnet:access_modes", "schemename": "dnet:access_modes"}, "instancetype": {"classid": "0001", "classname": "Article", "schemeid": "dnet:publication_resource", "schemename": "dnet:publication_resource"}}], "dateofcollection": "2021-05-07T02:08:22Z", "fulltext": [], "description": [], "format": [], "journal": {"issnPrinted": "0039-6206", "vol": "17", "sp": "304", "name": "Survey of Anesthesiology"}, "measures": [], "coverage": [], "externalReference": [], "publisher": {"value": "Ovid Technologies (Wolters Kluwer Health)"}, "resulttype": {"classid": "publication", "classname": "publication", "schemeid": "dnet:result_typologies", "schemename": "dnet:result_typologies"}, "country": [], "extraInfo": [], "originalId": ["10.1097/00132586-197308000-00003", "50|doiboost____::b0baa0eb88a5788f0b8815560d2a32f2"], "source": [{"value": "Crossref"}], "dateofacceptance": {"value": "2006-11-06T11:36:37Z"}, "title": [{"qualifier": {"classid": "main title", "classname": "main title", "schemeid": "dnet:dataCite_title", "schemename": "dnet:dataCite_title"}, "value": "SIGNIFICANCE OF CHRONIC SINUS BRADYCARDIA IN ELDERLY PEOPLE"}]} {"id":"50|doi_________::4972b0ca81b96b225aed8038bb965656","context": [], "dataInfo": {"invisible": false, "trust": "0.9", "provenanceaction": {"classid": "sysimport:actionset", "classname": "sysimport:actionset", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "inferred": false, "deletedbyinference": false}, "resourcetype": {"classid": "0001", "classname": "Article", "schemeid": "dnet:publication_resource", "schemename": "dnet:publication_resource"}, "pid": [{"qualifier": {"classid": "doi", "classname": "doi", "schemeid": "dnet:pid_types", "schemename": "dnet:pid_types"}, "value": "10.2143/tvg.62.1.5002364"}], "contributor": [], "bestaccessright": {"classid": "UNKNOWN", "classname": "not available", "schemeid": "dnet:access_modes", "schemename": "dnet:access_modes"}, "relevantdate": [{"qualifier": {"classid": "created", "classname": "created", "schemeid": "dnet:dataCite_date", "schemename": "dnet:dataCite_date"}, "value": "2007-08-20T08:35:04Z"}, {"qualifier": {"classid": "published-online", "classname": "published-online", "schemeid": "dnet:dataCite_date", "schemename": "dnet:dataCite_date"}, "value": "2006-01-01"}], "collectedfrom": [{"key": "10|openaire____::081b82f96300b6a6e3d282bad31cb6e2", "value": "Crossref"}], "subject": [{"qualifier": {"classid": "keywords", "classname": "keywords", "schemeid": "dnet:subject_classification_typologies", "schemename": "dnet:subject_classification_typologies"}, "value": "General Medicine"}], "lastupdatetimestamp": 1620381522840, "author": [{"fullname": "null VERHAMME P", "surname": "VERHAMME P", "rank": 1}], "instance": [{"hostedby": {"key": "10|issn__online::7ec728ad1ac65c60cd563a5137111125", "value": "Tijdschrift voor Geneeskunde"}, "url": ["https://doi.org/10.2143/tvg.62.1.5002364"], "pid": [{"qualifier": {"classid": "doi", "classname": "doi", "schemeid": "dnet:pid_types", "schemename": "dnet:pid_types"}, "value": "10.2143/tvg.62.1.5002364"}], "dateofacceptance": {"value": "2006-01-01"}, "collectedfrom": {"key": "10|openaire____::081b82f96300b6a6e3d282bad31cb6e2", "value": "Crossref"}, "accessright": {"classid": "UNKNOWN", "classname": "not available", "schemeid": "dnet:access_modes", "schemename": "dnet:access_modes"}, "instancetype": {"classid": "0001", "classname": "Article", "schemeid": "dnet:publication_resource", "schemename": "dnet:publication_resource"}}], "dateofcollection": "2021-05-07T09:58:42Z", "fulltext": [], "description": [], "format": [], "journal": {"vol": "62", "sp": "55", "issnOnline": "0371-683X", "ep": "61", "name": "Tijdschrift voor Geneeskunde"}, "measures": [], "coverage": [], "externalReference": [], "publisher": {"value": "Peeters Publishers"}, "resulttype": {"classid": "publication", "classname": "publication", "schemeid": "dnet:result_typologies", "schemename": "dnet:result_typologies"}, "country": [], "extraInfo": [], "originalId": ["10.2143/tvg.62.1.5002364", "50|doiboost____::4972b0ca81b96b225aed8038bb965656"], "source": [{"value": "Crossref"}], "dateofacceptance": {"value": "2006-01-01"}, "title": [{"qualifier": {"classid": "main title", "classname": "main title", "schemeid": "dnet:dataCite_title", "schemename": "dnet:dataCite_title"}, "value": "Antitrombotica: nieuwe moleculen"}]} {"id":"50|DansKnawCris::0224aae28af558f21768dbc6439a_ctx","author":[],"bestaccessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":[{"dataInfo":null,"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"}],"context":[{"dataInfo":[{"deletedbyinference":false,"inferenceprovenance":"bulktagging","inferred":true,"invisible":false,"provenanceaction":{"classid":"community:subject","classname":"Bulktagging for Community - Subject","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":null}],"id":"sobigdata::projects::2"}],"contributor":[],"country":[],"coverage":[],"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2007-01-01"},"dateofcollection":"","dateoftransformation":"2020-05-25T16:14:18.452Z","description":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Lit.opg., bijl."}],"embargoenddate":null,"externalReference":[],"extraInfo":[],"format":[],"fulltext":[],"instance":[{"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":{"dataInfo":null,"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"},"dateofacceptance":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"2007-01-01"},"distributionlocation":"","hostedby":{"dataInfo":null,"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"},"instancetype":{"classid":"0017","classname":"Report","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"license":null,"processingchargeamount":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":true,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"8250"},"processingchargecurrency":{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":true,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"EUR"},"refereed":null,"url":null}],"language":{"classid":"nl","classname":"nl","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastupdatetimestamp":1591282676557,"oaiprovenance":{"originDescription":{"altered":true,"baseURL":"http%3A%2F%2Fservices.nod.dans.knaw.nl%2Foa-cerif","datestamp":"2019-12-01T07:51:24Z","harvestDate":"2020-05-25T11:33:13.427Z","identifier":"oai:services.nod.dans.knaw.nl:Publications/rce:document:550013110","metadataNamespace":""}},"originalId":["DansKnawCris::0224aae28af558f21768dbc6439c7a95"],"pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"urn","classname":"urn","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"http://cultureelerfgoed.adlibsoft.com/dispatcher.aspx?action=search&database=ChoiceRapporten&search=priref=550013110"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"urn","classname":"urn","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"http://cultureelerfgoed.adlibsoft.com/dispatcher.aspx?action=search&database=ChoiceRapporten&search=priref=550013110"}],"publisher":null,"relevantdate":[],"resourcetype":{"classid":"0017","classname":"0017","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"source":[],"subject":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"value":"archeologie"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"value":"prospectie"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"value":"Archaeology"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"value":"Archaeology"}],"title":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"Gcube veldonderzoek d.m.v. boringen (karterende fase) : Raadhuisstraat te Dirkshorn, gemeente Harenkarspel"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"sysimport:crosswalk:datasetarchive","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"Synthegra Archeologie Rapportenreeks P0502381"}],"journal":null} From dde2fec0357eccbbc8b0fad1391ad070eb36cad9 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 31 Oct 2023 14:35:33 +0100 Subject: [PATCH 086/148] [graph cleaning] cleanup --- .../dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java index e8bada028..7a8acbd36 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java @@ -11,12 +11,10 @@ import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.util.*; import java.util.function.Function; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; -import org.apache.cxf.common.util.UrlUtils; import com.github.sisyphsu.dateparser.DateParserUtils; import com.google.common.collect.Lists; @@ -27,7 +25,6 @@ import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.oaf.*; import me.xuender.unidecode.Unidecode; -import sun.awt.HKSCS; public class GraphCleaningFunctions extends CleaningFunctions { From c80b768af062166e4ae94abcf529af0aac994b7b Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Tue, 31 Oct 2023 15:49:42 +0100 Subject: [PATCH 087/148] test for project propagation --- .../PrepareResultCommunitySet.java | 2 +- .../PrepareAssocTest.java | 14 +- .../ResultToCommunityJobTest.java | 257 ++---------------- 3 files changed, 36 insertions(+), 237 deletions(-) diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java index c47075272..9a2f25fa6 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java @@ -56,7 +56,7 @@ public class PrepareResultCommunitySet { log.info("production: {}", production); final CommunityEntityMap projectsMap = Utils.getCommunityProjects(production); - log.info("projectsMap: {}", new Gson().toJson(projectsMap)); + //log.info("projectsMap: {}", new Gson().toJson(projectsMap)); SparkConf conf = new SparkConf(); diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareAssocTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareAssocTest.java index 0e10b3edf..172873af3 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareAssocTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareAssocTest.java @@ -10,9 +10,7 @@ import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SparkSession; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,6 +59,7 @@ public class PrepareAssocTest { spark.stop(); } + @Disabled @Test void test1() throws Exception { @@ -72,8 +71,8 @@ public class PrepareAssocTest { "-sourcePath", getClass().getResource("/eu/dnetlib/dhp/resulttocommunityfromproject/relation/").getPath(), "-outputPath", workingDir.toString() + "/prepared", - "-production", Boolean.TRUE.toString(), - "-hive_metastore_uris", "" + "-production", Boolean.TRUE.toString() + }); final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); @@ -82,7 +81,10 @@ public class PrepareAssocTest { .textFile(workingDir.toString() + "/prepared") .map(item -> new ObjectMapper().readValue(item, ResultProjectList.class)); - tmp.foreach(r -> System.out.println(new ObjectMapper().writeValueAsString(r))); + Assertions.assertEquals(4, tmp.count()); + Assertions.assertEquals(2, tmp.filter(rpl -> rpl.getCommunityList().contains("aurora")).count()); + Assertions.assertEquals(1, tmp.filter(rpl -> rpl.getCommunityList().contains("sdsn-gr")).count()); + Assertions.assertEquals(1, tmp.filter(rpl -> rpl.getCommunityList().contains("netherlands")).count()); } } diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultToCommunityJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultToCommunityJobTest.java index 6a5726cbe..b705af8ac 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultToCommunityJobTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultToCommunityJobTest.java @@ -6,7 +6,9 @@ import static org.apache.spark.sql.functions.desc; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; +import eu.dnetlib.dhp.schema.oaf.Context; import org.apache.commons.io.FileUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; @@ -89,235 +91,30 @@ public class ResultToCommunityJobTest { .textFile(workingDir.toString() + "/dataset") .map(item -> OBJECT_MAPPER.readValue(item, Dataset.class)); - tmp.foreach(d -> System.out.println(new ObjectMapper().writeValueAsString(d))); -// Assertions.assertEquals(10, tmp.count()); -// org.apache.spark.sql.Dataset verificationDataset = spark -// .createDataset(tmp.rdd(), Encoders.bean(Dataset.class)); -// -// verificationDataset.createOrReplaceTempView("dataset"); -// -// String query = "select id, MyT.id community " -// + "from dataset " -// + "lateral view explode(context) c as MyT " -// + "lateral view explode(MyT.datainfo) d as MyD " -// + "where MyD.inferenceprovenance = 'propagation'"; -// -// org.apache.spark.sql.Dataset resultExplodedProvenance = spark.sql(query); -// Assertions.assertEquals(5, resultExplodedProvenance.count()); -// Assertions -// .assertEquals( -// 0, -// resultExplodedProvenance -// .filter("id = '50|dedup_wf_001::afaf128022d29872c4dad402b2db04fe'") -// .count()); -// Assertions -// .assertEquals( -// 1, -// resultExplodedProvenance -// .filter("id = '50|dedup_wf_001::3f62cfc27024d564ea86760c494ba93b'") -// .count()); -// Assertions -// .assertEquals( -// "beopen", -// resultExplodedProvenance -// .select("community") -// .where( -// resultExplodedProvenance -// .col("id") -// .equalTo( -// "50|dedup_wf_001::3f62cfc27024d564ea86760c494ba93b")) -// .collectAsList() -// .get(0) -// .getString(0)); -// -// Assertions -// .assertEquals( -// 2, -// resultExplodedProvenance -// .filter("id = '50|od________18::8887b1df8b563c4ea851eb9c882c9d7b'") -// .count()); -// Assertions -// .assertEquals( -// "mes", -// resultExplodedProvenance -// .select("community") -// .where( -// resultExplodedProvenance -// .col("id") -// .equalTo( -// "50|od________18::8887b1df8b563c4ea851eb9c882c9d7b")) -// .sort(desc("community")) -// .collectAsList() -// .get(0) -// .getString(0)); -// Assertions -// .assertEquals( -// "euromarine", -// resultExplodedProvenance -// .select("community") -// .where( -// resultExplodedProvenance -// .col("id") -// .equalTo( -// "50|od________18::8887b1df8b563c4ea851eb9c882c9d7b")) -// .sort(desc("community")) -// .collectAsList() -// .get(1) -// .getString(0)); -// -// Assertions -// .assertEquals( -// 1, -// resultExplodedProvenance -// .filter("id = '50|doajarticles::8d817039a63710fcf97e30f14662c6c8'") -// .count()); -// Assertions -// .assertEquals( -// "mes", -// resultExplodedProvenance -// .select("community") -// .where( -// resultExplodedProvenance -// .col("id") -// .equalTo( -// "50|doajarticles::8d817039a63710fcf97e30f14662c6c8")) -// .sort(desc("community")) -// .collectAsList() -// .get(0) -// .getString(0)); -// -// Assertions -// .assertEquals( -// 1, -// resultExplodedProvenance -// .filter("id = '50|doajarticles::3c98f0632f1875b4979e552ba3aa01e6'") -// .count()); -// Assertions -// .assertEquals( -// "mes", -// resultExplodedProvenance -// .select("community") -// .where( -// resultExplodedProvenance -// .col("id") -// .equalTo( -// "50|doajarticles::3c98f0632f1875b4979e552ba3aa01e6")) -// .sort(desc("community")) -// .collectAsList() -// .get(0) -// .getString(0)); -// -// query = "select id, MyT.id community " -// + "from dataset " -// + "lateral view explode(context) c as MyT " -// + "lateral view explode(MyT.datainfo) d as MyD "; -// -// org.apache.spark.sql.Dataset resultCommunityId = spark.sql(query); -// -// Assertions.assertEquals(10, resultCommunityId.count()); -// -// Assertions -// .assertEquals( -// 1, -// resultCommunityId -// .filter("id = '50|dedup_wf_001::afaf128022d29872c4dad402b2db04fe'") -// .count()); -// Assertions -// .assertEquals( -// "beopen", -// resultCommunityId -// .select("community") -// .where( -// resultCommunityId -// .col("id") -// .equalTo( -// "50|dedup_wf_001::afaf128022d29872c4dad402b2db04fe")) -// .collectAsList() -// .get(0) -// .getString(0)); -// -// Assertions -// .assertEquals( -// 1, -// resultCommunityId -// .filter("id = '50|dedup_wf_001::3f62cfc27024d564ea86760c494ba93b'") -// .count()); -// -// Assertions -// .assertEquals( -// 3, -// resultCommunityId -// .filter("id = '50|od________18::8887b1df8b563c4ea851eb9c882c9d7b'") -// .count()); -// Assertions -// .assertEquals( -// "beopen", -// resultCommunityId -// .select("community") -// .where( -// resultCommunityId -// .col("id") -// .equalTo( -// "50|od________18::8887b1df8b563c4ea851eb9c882c9d7b")) -// .sort(desc("community")) -// .collectAsList() -// .get(2) -// .getString(0)); -// -// Assertions -// .assertEquals( -// 2, -// resultCommunityId -// .filter("id = '50|doajarticles::8d817039a63710fcf97e30f14662c6c8'") -// .count()); -// Assertions -// .assertEquals( -// "euromarine", -// resultCommunityId -// .select("community") -// .where( -// resultCommunityId -// .col("id") -// .equalTo( -// "50|doajarticles::8d817039a63710fcf97e30f14662c6c8")) -// .sort(desc("community")) -// .collectAsList() -// .get(1) -// .getString(0)); -// -// Assertions -// .assertEquals( -// 3, -// resultCommunityId -// .filter("id = '50|doajarticles::3c98f0632f1875b4979e552ba3aa01e6'") -// .count()); -// Assertions -// .assertEquals( -// "euromarine", -// resultCommunityId -// .select("community") -// .where( -// resultCommunityId -// .col("id") -// .equalTo( -// "50|doajarticles::3c98f0632f1875b4979e552ba3aa01e6")) -// .sort(desc("community")) -// .collectAsList() -// .get(2) -// .getString(0)); -// Assertions -// .assertEquals( -// "ni", -// resultCommunityId -// .select("community") -// .where( -// resultCommunityId -// .col("id") -// .equalTo( -// "50|doajarticles::3c98f0632f1875b4979e552ba3aa01e6")) -// .sort(desc("community")) -// .collectAsList() -// .get(0) -// .getString(0)); + Assertions.assertEquals(10, tmp.count()); + /** + * {"resultId":"50|57a035e5b1ae::d5be548ca7ae489d762f893be67af52f","communityList":["aurora"]} + * {"resultId":"50|57a035e5b1ae::a77232ffca9115fcad51c3503dbc7e3e","communityList":["aurora"]} + * {"resultId":"50|57a035e5b1ae::803aaad4decab7e27cd4b52a1931b3a1","communityList":["sdsn-gr"]} + * {"resultId":"50|57a035e5b1ae::a02e9e4087bca50687731ae5c765b5e1","communityList":["netherlands"]} + */ + List context = tmp.filter(r -> r.getId().equals("50|57a035e5b1ae::d5be548ca7ae489d762f893be67af52f")) + .first().getContext(); + Assertions.assertTrue(context.stream().anyMatch(c -> containsResultCommunityProject(c))); + + context = tmp.filter(r -> r.getId().equals("50|57a035e5b1ae::a77232ffca9115fcad51c3503dbc7e3e")) + .first().getContext(); + Assertions.assertTrue(context.stream().anyMatch(c -> containsResultCommunityProject(c))); + + Assertions.assertEquals(0, tmp.filter(r -> r.getId().equals("50|57a035e5b1ae::803aaad4decab7e27cd4b52a1931b3a1")).count()); + + Assertions.assertEquals(0, tmp.filter(r -> r.getId().equals("50|57a035e5b1ae::a02e9e4087bca50687731ae5c765b5e1")).count()); + + Assertions.assertEquals(2, tmp.filter(r -> r.getContext().stream().anyMatch(c -> c.getId().equals("aurora"))).count()); + + } + + private static boolean containsResultCommunityProject(Context c) { + return c.getDataInfo().stream().anyMatch(di -> di.getProvenanceaction().getClassid().equals("result:community:project")); } } From a737dd47b6268111a18550a3e94241ec8358447c Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Tue, 31 Oct 2023 15:54:49 +0100 Subject: [PATCH 088/148] removed not needed test class --- .../dnetlib/dhp/bulktag/BulkTagJobTest.java | 2 +- .../PrepareAssocTest.java | 90 ------------------- 2 files changed, 1 insertion(+), 91 deletions(-) delete mode 100644 dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareAssocTest.java diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java index 7cbbcaafb..5e106ef45 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java @@ -6,7 +6,7 @@ import static eu.dnetlib.dhp.bulktag.community.TaggingConstants.ZENODO_COMMUNITY import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.HashMap; + import java.util.List; import org.apache.commons.io.FileUtils; diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareAssocTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareAssocTest.java deleted file mode 100644 index 172873af3..000000000 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareAssocTest.java +++ /dev/null @@ -1,90 +0,0 @@ - -package eu.dnetlib.dhp.resulttocommunityfromproject; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; - -import org.apache.commons.io.FileUtils; -import org.apache.spark.SparkConf; -import org.apache.spark.api.java.JavaRDD; -import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.sql.SparkSession; -import org.junit.jupiter.api.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.databind.ObjectMapper; - -import eu.dnetlib.dhp.bulktag.BulkTagJobTest; -import eu.dnetlib.dhp.resulttocommunityfromorganization.ResultCommunityList; - -/** - * @author miriam.baglioni - * @Date 13/10/23 - */ -public class PrepareAssocTest { - - private static SparkSession spark; - - private static Path workingDir; - - private static final Logger log = LoggerFactory.getLogger(PrepareAssocTest.class); - - @BeforeAll - public static void beforeAll() throws IOException { - workingDir = Files.createTempDirectory(BulkTagJobTest.class.getSimpleName()); - log.info("using work dir {}", workingDir); - - SparkConf conf = new SparkConf(); - conf.setAppName(BulkTagJobTest.class.getSimpleName()); - - conf.setMaster("local[*]"); - conf.set("spark.driver.host", "localhost"); - conf.set("hive.metastore.local", "true"); - conf.set("spark.ui.enabled", "false"); - conf.set("spark.sql.warehouse.dir", workingDir.toString()); - conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString()); - - spark = SparkSession - .builder() - .appName(PrepareAssocTest.class.getSimpleName()) - .config(conf) - .getOrCreate(); - } - - @AfterAll - public static void afterAll() throws IOException { - FileUtils.deleteDirectory(workingDir.toFile()); - spark.stop(); - } - - @Disabled - @Test - void test1() throws Exception { - - PrepareResultCommunitySet - .main( - new String[] { - - "-isSparkSessionManaged", Boolean.FALSE.toString(), - "-sourcePath", - getClass().getResource("/eu/dnetlib/dhp/resulttocommunityfromproject/relation/").getPath(), - "-outputPath", workingDir.toString() + "/prepared", - "-production", Boolean.TRUE.toString() - - }); - - final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); - - JavaRDD tmp = sc - .textFile(workingDir.toString() + "/prepared") - .map(item -> new ObjectMapper().readValue(item, ResultProjectList.class)); - - Assertions.assertEquals(4, tmp.count()); - Assertions.assertEquals(2, tmp.filter(rpl -> rpl.getCommunityList().contains("aurora")).count()); - Assertions.assertEquals(1, tmp.filter(rpl -> rpl.getCommunityList().contains("sdsn-gr")).count()); - Assertions.assertEquals(1, tmp.filter(rpl -> rpl.getCommunityList().contains("netherlands")).count()); - } - -} From 937ff6a7c77fa6422463471d8f3aff9c22ff775b Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Tue, 31 Oct 2023 15:56:08 +0100 Subject: [PATCH 089/148] - --- .../dhp/api/QueryCommunityAPITest.java | 119 ------------------ .../dnetlib/dhp/bulktag/BulkTagJobTest.java | 2 - 2 files changed, 121 deletions(-) delete mode 100644 dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/api/QueryCommunityAPITest.java diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/api/QueryCommunityAPITest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/api/QueryCommunityAPITest.java deleted file mode 100644 index 6ee01a6f0..000000000 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/api/QueryCommunityAPITest.java +++ /dev/null @@ -1,119 +0,0 @@ - -package eu.dnetlib.dhp.api; - -import java.util.List; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; - -import eu.dnetlib.dhp.api.QueryCommunityAPI; -import eu.dnetlib.dhp.api.Utils; -import eu.dnetlib.dhp.api.model.CommunityEntityMap; -import eu.dnetlib.dhp.api.model.CommunityModel; -import eu.dnetlib.dhp.api.model.CommunitySummary; -import eu.dnetlib.dhp.api.model.DatasourceList; -import eu.dnetlib.dhp.bulktag.community.Community; -import eu.dnetlib.dhp.bulktag.community.CommunityConfiguration; - -/** - * @author miriam.baglioni - * @Date 06/10/23 - */ -public class QueryCommunityAPITest { - - @Test - void communityList() throws Exception { - String body = QueryCommunityAPI.communities(true); - new ObjectMapper() - .readValue(body, CommunitySummary.class) - .forEach(p -> { - try { - System.out.println(new ObjectMapper().writeValueAsString(p)); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - }); - } - - @Test - void community() throws Exception { - String id = "dh-ch"; - String body = QueryCommunityAPI.community(id, true); - System.out - .println( - new ObjectMapper() - .writeValueAsString( - new ObjectMapper() - .readValue(body, CommunityModel.class))); - } - - @Test - void communityDatasource() throws Exception { - String id = "dh-ch"; - String body = QueryCommunityAPI.communityDatasource(id, true); - new ObjectMapper() - .readValue(body, DatasourceList.class) - .forEach(ds -> { - try { - System.out.println(new ObjectMapper().writeValueAsString(ds)); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - }); - ; - } - - @Test - void validCommunities() throws Exception { - CommunityConfiguration cc = Utils.getCommunityConfiguration(true); - System.out.println(cc.getCommunities().keySet()); - Community community = cc.getCommunities().get("aurora"); - Assertions.assertEquals(0, community.getSubjects().size()); - Assertions.assertEquals(null, community.getConstraints()); - Assertions.assertEquals(null, community.getRemoveConstraints()); - Assertions.assertEquals(2, community.getZenodoCommunities().size()); - Assertions - .assertTrue( - community.getZenodoCommunities().stream().anyMatch(c -> c.equals("aurora-universities-network"))); - Assertions - .assertTrue(community.getZenodoCommunities().stream().anyMatch(c -> c.equals("university-of-innsbruck"))); - Assertions.assertEquals(35, community.getProviders().size()); - Assertions - .assertEquals( - 35, community.getProviders().stream().filter(p -> p.getSelectionConstraints() == null).count()); - - } - - @Test - void eutopiaCommunityConfiguration() throws Exception { - CommunityConfiguration cc = Utils.getCommunityConfiguration(true); - System.out.println(cc.getCommunities().keySet()); - Community community = cc.getCommunities().get("eutopia"); - community.getProviders().forEach(p -> System.out.println(p.getOpenaireId())); - } - - @Test - void getCommunityProjects() throws Exception { - CommunityEntityMap projectMap = Utils.getCommunityProjects(true); - - Assertions - .assertTrue( - projectMap - .keySet() - .stream() - .allMatch(k -> k.startsWith("40|"))); - - System.out.println(projectMap); - } - - @Test - void getCommunityOrganizations() throws Exception { - CommunityEntityMap organizationMap = Utils.getCommunityOrganization(true); - Assertions.assertTrue(organizationMap.keySet().stream().allMatch(k -> k.startsWith("20|"))); - - } - -} diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java index 5e106ef45..d02faedec 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java @@ -32,8 +32,6 @@ public class BulkTagJobTest { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - public static final String MOCK_IS_LOOK_UP_URL = "BASEURL:8280/is/services/isLookUp"; - public static final String pathMap = "{ \"author\" : \"$['author'][*]['fullname']\"," + " \"title\" : \"$['title'][*]['value']\"," + " \"orcid\" : \"$['author'][*]['pid'][*][?(@['key']=='ORCID')]['value']\"," From 94089878fd08d518e11b4df06c5d8ecf9689ea2f Mon Sep 17 00:00:00 2001 From: Serafeim Chatzopoulos Date: Thu, 26 Oct 2023 13:47:06 -0700 Subject: [PATCH 090/148] Adjust tests to new WF input params --- .../PrepareAffiliationRelationsTest.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelationsTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelationsTest.java index ed8e5fe0d..e2639996c 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelationsTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelationsTest.java @@ -74,17 +74,22 @@ public class PrepareAffiliationRelationsTest { @Test void testMatch() throws Exception { - String affiliationRelationsPath = getClass() + String crossrefAffiliationRelationPath = getClass() .getResource("/eu/dnetlib/dhp/actionmanager/bipaffiliations/doi_to_ror.json") .getPath(); + String pubmedAffiliationRelationsPath = getClass() + .getResource("/eu/dnetlib/dhp/actionmanager/bipaffiliations/doi_to_ror.json") + .getPath(); + String outputPath = workingDir.toString() + "/actionSet"; PrepareAffiliationRelations .main( new String[] { "-isSparkSessionManaged", Boolean.FALSE.toString(), - "-inputPath", affiliationRelationsPath, + "-crossrefInputPath", crossrefAffiliationRelationPath, + "-pubmedInputPath", pubmedAffiliationRelationsPath, "-outputPath", outputPath }); @@ -101,7 +106,7 @@ public class PrepareAffiliationRelationsTest { // ); // } // count the number of relations - assertEquals(20, tmp.count()); + assertEquals(40, tmp.count()); Dataset dataset = spark.createDataset(tmp.rdd(), Encoders.bean(Relation.class)); dataset.createOrReplaceTempView("result"); @@ -112,7 +117,7 @@ public class PrepareAffiliationRelationsTest { // verify that we have equal number of bi-directional relations Assertions .assertEquals( - 10, execVerification + 20, execVerification .filter( "relClass='" + ModelConstants.HAS_AUTHOR_INSTITUTION + "'") .collectAsList() @@ -120,7 +125,7 @@ public class PrepareAffiliationRelationsTest { Assertions .assertEquals( - 10, execVerification + 20, execVerification .filter( "relClass='" + ModelConstants.IS_AUTHOR_INSTITUTION_OF + "'") .collectAsList() From 6ce9b600c1ec3a68328f0f814f895aee4edd421c Mon Sep 17 00:00:00 2001 From: Serafeim Chatzopoulos Date: Thu, 19 Oct 2023 19:58:25 +0300 Subject: [PATCH 091/148] Add actionset creation for pubmed affiliations --- .../PrepareAffiliationRelations.java | 32 ++++++++++++------- .../input_actionset_parameter.json | 8 ++++- .../bipaffiliations/job.properties | 1 + .../bipaffiliations/oozie_app/workflow.xml | 7 +++- 4 files changed, 35 insertions(+), 13 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java index 603ad6339..cbfba30c5 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java @@ -12,6 +12,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.io.compress.GzipCodec; import org.apache.hadoop.mapred.SequenceFileOutputFormat; import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.sql.*; import org.apache.spark.sql.Dataset; @@ -58,10 +59,13 @@ public class PrepareAffiliationRelations implements Serializable { log.info("isSparkSessionManaged: {}", isSparkSessionManaged); final String inputPath = parser.get("inputPath"); - log.info("inputPath {}: ", inputPath); + log.info("inputPath: {}", inputPath); + + final String pubmedInputPath = parser.get("pubmedInputPath"); + log.info("pubmedInputPath: {}", pubmedInputPath); final String outputPath = parser.get("outputPath"); - log.info("outputPath {}: ", outputPath); + log.info("outputPath: {}", outputPath); SparkConf conf = new SparkConf(); @@ -70,12 +74,22 @@ public class PrepareAffiliationRelations implements Serializable { isSparkSessionManaged, spark -> { Constants.removeOutputDir(spark, outputPath); - prepareAffiliationRelations(spark, inputPath, outputPath); + + List collectedFromCrossref = OafMapperUtils.listKeyValues(ModelConstants.CROSSREF_ID, "Crossref"); + JavaPairRDD crossrefRelations = prepareAffiliationRelations(spark, inputPath, collectedFromCrossref); + + List collectedFromPubmed = OafMapperUtils.listKeyValues(ModelConstants.PUBMED_CENTRAL_ID, "Pubmed"); + JavaPairRDD pubmedRelations = prepareAffiliationRelations(spark, inputPath, collectedFromPubmed); + + crossrefRelations + .union(pubmedRelations) + .saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, GzipCodec.class); + }); } - private static void prepareAffiliationRelations(SparkSession spark, String inputPath, - String outputPath) { + private static JavaPairRDD prepareAffiliationRelations(SparkSession spark, String inputPath, + List collectedfrom) { // load and parse affiliation relations from HDFS Dataset df = spark @@ -92,7 +106,7 @@ public class PrepareAffiliationRelations implements Serializable { new Column("matching.Confidence").as("confidence")); // prepare action sets for affiliation relations - df + return df .toJavaRDD() .flatMap((FlatMapFunction) row -> { @@ -120,8 +134,6 @@ public class PrepareAffiliationRelations implements Serializable { qualifier, Double.toString(row.getAs("confidence"))); - List collectedfrom = OafMapperUtils.listKeyValues(ModelConstants.CROSSREF_ID, "Crossref"); - // return bi-directional relations return getAffiliationRelationPair(paperId, affId, collectedfrom, dataInfo).iterator(); @@ -129,9 +141,7 @@ public class PrepareAffiliationRelations implements Serializable { .map(p -> new AtomicAction(Relation.class, p)) .mapToPair( aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()), - new Text(OBJECT_MAPPER.writeValueAsString(aa)))) - .saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, GzipCodec.class); - + new Text(OBJECT_MAPPER.writeValueAsString(aa)))); } private static List getAffiliationRelationPair(String paperId, String affId, List collectedfrom, diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/input_actionset_parameter.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/input_actionset_parameter.json index 7663a454b..96dcc3b32 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/input_actionset_parameter.json +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/input_actionset_parameter.json @@ -8,7 +8,13 @@ { "paramName": "ip", "paramLongName": "inputPath", - "paramDescription": "the URL from where to get the programme file", + "paramDescription": "the path to get the input data from Crossref", + "paramRequired": true + }, + { + "paramName": "pip", + "paramLongName": "pubmedInputPath", + "paramDescription": "the path to get the input data from Pubmed", "paramRequired": true }, { diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/job.properties b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/job.properties index d942e6772..fe3cbb633 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/job.properties +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/job.properties @@ -32,4 +32,5 @@ spark2SqlQueryExecutionListeners=com.cloudera.spark.lineage.NavigatorQueryListen oozie.wf.application.path=${oozieTopWfApplicationPath} inputPath=/data/bip-affiliations/data.json +pubmedInputPath=/data/bip-affiiations/pubmed-data.json outputPath=/tmp/crossref-affiliations-output-v5 diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/oozie_app/workflow.xml index 9930cfe17..c0a6bfc52 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/oozie_app/workflow.xml @@ -3,7 +3,11 @@ inputPath - the path where to find the inferred affiliation relations + the path where to find the inferred affiliation relations from Crossref + + + pubmedInputPath + the path where to find the inferred affiliation relations from Pubmed outputPath @@ -97,6 +101,7 @@ --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} --inputPath${inputPath} + --pubmedInputPath${pubmedInputPath} --outputPath${outputPath} From 24c3f92d87e084c1b0b85f9f30707b7dc28f3c07 Mon Sep 17 00:00:00 2001 From: Serafeim Chatzopoulos Date: Fri, 20 Oct 2023 12:48:21 +0300 Subject: [PATCH 092/148] Change the description of the workflow --- .../PrepareAffiliationRelations.java | 22 ++++++++++++------- .../bipaffiliations/oozie_app/workflow.xml | 2 +- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java index cbfba30c5..18d98be54 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java @@ -75,21 +75,27 @@ public class PrepareAffiliationRelations implements Serializable { spark -> { Constants.removeOutputDir(spark, outputPath); - List collectedFromCrossref = OafMapperUtils.listKeyValues(ModelConstants.CROSSREF_ID, "Crossref"); - JavaPairRDD crossrefRelations = prepareAffiliationRelations(spark, inputPath, collectedFromCrossref); + List collectedFromCrossref = OafMapperUtils + .listKeyValues(ModelConstants.CROSSREF_ID, "Crossref"); + JavaPairRDD crossrefRelations = prepareAffiliationRelations( + spark, inputPath, collectedFromCrossref); - List collectedFromPubmed = OafMapperUtils.listKeyValues(ModelConstants.PUBMED_CENTRAL_ID, "Pubmed"); - JavaPairRDD pubmedRelations = prepareAffiliationRelations(spark, inputPath, collectedFromPubmed); + List collectedFromPubmed = OafMapperUtils + .listKeyValues(ModelConstants.PUBMED_CENTRAL_ID, "Pubmed"); + JavaPairRDD pubmedRelations = prepareAffiliationRelations( + spark, inputPath, collectedFromPubmed); crossrefRelations - .union(pubmedRelations) - .saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, GzipCodec.class); + .union(pubmedRelations) + .saveAsHadoopFile( + outputPath, Text.class, Text.class, SequenceFileOutputFormat.class, GzipCodec.class); }); } - private static JavaPairRDD prepareAffiliationRelations(SparkSession spark, String inputPath, - List collectedfrom) { + private static JavaPairRDD prepareAffiliationRelations(SparkSession spark, + String inputPath, + List collectedfrom) { // load and parse affiliation relations from HDFS Dataset df = spark diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/oozie_app/workflow.xml index c0a6bfc52..e3fdddfd6 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/oozie_app/workflow.xml @@ -87,7 +87,7 @@ yarn cluster - Produces the atomic action with the inferred by BIP! affiliation relations from Crossref + Produces the atomic action with the inferred by BIP! affiliation relations (from Crossref and Pubmed) eu.dnetlib.dhp.actionmanager.bipaffiliations.PrepareAffiliationRelations dhp-aggregation-${projectVersion}.jar From 7e34dde774eaa326ee8a7cbfaad75f42b5bd2eb9 Mon Sep 17 00:00:00 2001 From: Serafeim Chatzopoulos Date: Wed, 25 Oct 2023 12:05:02 -0700 Subject: [PATCH 093/148] Renaming input param for crossref input path --- .../bipaffiliations/PrepareAffiliationRelations.java | 8 ++++---- .../bipaffiliations/input_actionset_parameter.json | 4 ++-- .../dhp/actionmanager/bipaffiliations/job.properties | 4 ++-- .../actionmanager/bipaffiliations/oozie_app/workflow.xml | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java index 18d98be54..1bdb06ecc 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelations.java @@ -58,8 +58,8 @@ public class PrepareAffiliationRelations implements Serializable { Boolean isSparkSessionManaged = Constants.isSparkSessionManaged(parser); log.info("isSparkSessionManaged: {}", isSparkSessionManaged); - final String inputPath = parser.get("inputPath"); - log.info("inputPath: {}", inputPath); + final String crossrefInputPath = parser.get("crossrefInputPath"); + log.info("crossrefInputPath: {}", crossrefInputPath); final String pubmedInputPath = parser.get("pubmedInputPath"); log.info("pubmedInputPath: {}", pubmedInputPath); @@ -78,12 +78,12 @@ public class PrepareAffiliationRelations implements Serializable { List collectedFromCrossref = OafMapperUtils .listKeyValues(ModelConstants.CROSSREF_ID, "Crossref"); JavaPairRDD crossrefRelations = prepareAffiliationRelations( - spark, inputPath, collectedFromCrossref); + spark, crossrefInputPath, collectedFromCrossref); List collectedFromPubmed = OafMapperUtils .listKeyValues(ModelConstants.PUBMED_CENTRAL_ID, "Pubmed"); JavaPairRDD pubmedRelations = prepareAffiliationRelations( - spark, inputPath, collectedFromPubmed); + spark, pubmedInputPath, collectedFromPubmed); crossrefRelations .union(pubmedRelations) diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/input_actionset_parameter.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/input_actionset_parameter.json index 96dcc3b32..c6f905199 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/input_actionset_parameter.json +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/input_actionset_parameter.json @@ -6,8 +6,8 @@ "paramRequired": false }, { - "paramName": "ip", - "paramLongName": "inputPath", + "paramName": "cip", + "paramLongName": "crossrefInputPath", "paramDescription": "the path to get the input data from Crossref", "paramRequired": true }, diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/job.properties b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/job.properties index fe3cbb633..a3d55ff0c 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/job.properties +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/job.properties @@ -31,6 +31,6 @@ spark2SqlQueryExecutionListeners=com.cloudera.spark.lineage.NavigatorQueryListen # The following is needed as a property of a workflow oozie.wf.application.path=${oozieTopWfApplicationPath} -inputPath=/data/bip-affiliations/data.json -pubmedInputPath=/data/bip-affiiations/pubmed-data.json +crossrefInputPath=/data/bip-affiliations/data.json +pubmedInputPath=/data/bip-affiliations/pubmed-data.json outputPath=/tmp/crossref-affiliations-output-v5 diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/oozie_app/workflow.xml index e3fdddfd6..c5ac6f884 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/bipaffiliations/oozie_app/workflow.xml @@ -2,7 +2,7 @@ - inputPath + crossrefInputPath the path where to find the inferred affiliation relations from Crossref @@ -100,7 +100,7 @@ --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} - --inputPath${inputPath} + --crossrefInputPath${crossrefInputPath} --pubmedInputPath${pubmedInputPath} --outputPath${outputPath} From edcb17ca9827c39c8662b3083d2246dd91bd26ab Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 3 Nov 2023 13:01:14 +0100 Subject: [PATCH 094/148] refactoring and test --- .../PrepareResultCommunitySet.java | 2 +- .../SparkResultToCommunityFromProject.java | 18 +++++++++-- .../dnetlib/dhp/bulktag/BulkTagJobTest.java | 1 - .../ResultToCommunityJobTest.java | 31 +++++++++++++------ 4 files changed, 39 insertions(+), 13 deletions(-) diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java index 9a2f25fa6..7bee1ea0c 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java @@ -56,7 +56,7 @@ public class PrepareResultCommunitySet { log.info("production: {}", production); final CommunityEntityMap projectsMap = Utils.getCommunityProjects(production); - //log.info("projectsMap: {}", new Gson().toJson(projectsMap)); + // log.info("projectsMap: {}", new Gson().toJson(projectsMap)); SparkConf conf = new SparkConf(); diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java index 90c30f812..6e298cf94 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java @@ -112,7 +112,7 @@ public class SparkResultToCommunityFromProject implements Serializable { R ret = value._1(); Optional rcl = Optional.ofNullable(value._2()); if (rcl.isPresent()) { - ArrayList communitySet = rcl.get().getCommunityList(); + // ArrayList communitySet = rcl.get().getCommunityList(); List contextList = ret .getContext() .stream() @@ -124,7 +124,7 @@ public class SparkResultToCommunityFromProject implements Serializable { res.setId(ret.getId()); List propagatedContexts = new ArrayList<>(); - for (String cId : communitySet) { + for (String cId : rcl.get().getCommunityList()) { if (!contextList.contains(cId)) { Context newContext = new Context(); newContext.setId(cId); @@ -138,6 +138,20 @@ public class SparkResultToCommunityFromProject implements Serializable { PROPAGATION_RESULT_COMMUNITY_PROJECT_CLASS_NAME, ModelConstants.DNET_PROVENANCE_ACTIONS))); propagatedContexts.add(newContext); + } else { + ret + .getContext() + .stream() + .filter(c -> c.getId().equals(cId)) + .findFirst() + .get() + .getDataInfo() + .add( + getDataInfo( + PROPAGATION_DATA_INFO_TYPE, + PROPAGATION_RESULT_COMMUNITY_PROJECT_CLASS_ID, + PROPAGATION_RESULT_COMMUNITY_PROJECT_CLASS_NAME, + ModelConstants.DNET_PROVENANCE_ACTIONS)); } } res.setContext(propagatedContexts); diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java index d02faedec..efabcc6bc 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java @@ -6,7 +6,6 @@ import static eu.dnetlib.dhp.bulktag.community.TaggingConstants.ZENODO_COMMUNITY import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; - import java.util.List; import org.apache.commons.io.FileUtils; diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultToCommunityJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultToCommunityJobTest.java index b705af8ac..5642501b3 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultToCommunityJobTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultToCommunityJobTest.java @@ -8,7 +8,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; -import eu.dnetlib.dhp.schema.oaf.Context; import org.apache.commons.io.FileUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; @@ -27,6 +26,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import eu.dnetlib.dhp.orcidtoresultfromsemrel.OrcidPropagationJobTest; import eu.dnetlib.dhp.resulttocommunityfromorganization.SparkResultToCommunityFromOrganizationJob; +import eu.dnetlib.dhp.schema.oaf.Context; import eu.dnetlib.dhp.schema.oaf.Dataset; public class ResultToCommunityJobTest { @@ -98,23 +98,36 @@ public class ResultToCommunityJobTest { * {"resultId":"50|57a035e5b1ae::803aaad4decab7e27cd4b52a1931b3a1","communityList":["sdsn-gr"]} * {"resultId":"50|57a035e5b1ae::a02e9e4087bca50687731ae5c765b5e1","communityList":["netherlands"]} */ - List context = tmp.filter(r -> r.getId().equals("50|57a035e5b1ae::d5be548ca7ae489d762f893be67af52f")) - .first().getContext(); + List context = tmp + .filter(r -> r.getId().equals("50|57a035e5b1ae::d5be548ca7ae489d762f893be67af52f")) + .first() + .getContext(); Assertions.assertTrue(context.stream().anyMatch(c -> containsResultCommunityProject(c))); - context = tmp.filter(r -> r.getId().equals("50|57a035e5b1ae::a77232ffca9115fcad51c3503dbc7e3e")) - .first().getContext(); + context = tmp + .filter(r -> r.getId().equals("50|57a035e5b1ae::a77232ffca9115fcad51c3503dbc7e3e")) + .first() + .getContext(); Assertions.assertTrue(context.stream().anyMatch(c -> containsResultCommunityProject(c))); - Assertions.assertEquals(0, tmp.filter(r -> r.getId().equals("50|57a035e5b1ae::803aaad4decab7e27cd4b52a1931b3a1")).count()); + Assertions + .assertEquals( + 0, tmp.filter(r -> r.getId().equals("50|57a035e5b1ae::803aaad4decab7e27cd4b52a1931b3a1")).count()); - Assertions.assertEquals(0, tmp.filter(r -> r.getId().equals("50|57a035e5b1ae::a02e9e4087bca50687731ae5c765b5e1")).count()); + Assertions + .assertEquals( + 0, tmp.filter(r -> r.getId().equals("50|57a035e5b1ae::a02e9e4087bca50687731ae5c765b5e1")).count()); - Assertions.assertEquals(2, tmp.filter(r -> r.getContext().stream().anyMatch(c -> c.getId().equals("aurora"))).count()); + Assertions + .assertEquals( + 2, tmp.filter(r -> r.getContext().stream().anyMatch(c -> c.getId().equals("aurora"))).count()); } private static boolean containsResultCommunityProject(Context c) { - return c.getDataInfo().stream().anyMatch(di -> di.getProvenanceaction().getClassid().equals("result:community:project")); + return c + .getDataInfo() + .stream() + .anyMatch(di -> di.getProvenanceaction().getClassid().equals("result:community:project")); } } From 638ad9e74f09a7ea975e49cc7842746096608cbc Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 3 Nov 2023 15:06:50 +0100 Subject: [PATCH 095/148] changing test for new implementation --- ...kResultToCommunityFromOrganizationJob.java | 2 - .../dnetlib/dhp/bulktag/BulkTagJobTest.java | 4 +- .../PrepareAssocTest.java | 95 ------------------ .../ResultToCommunityJobTest.java | 8 +- .../sample/dataset | 10 ++ .../sample/dataset_10.json.gz | Bin 7128 -> 0 bytes 6 files changed, 16 insertions(+), 103 deletions(-) delete mode 100644 dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareAssocTest.java create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/sample/dataset delete mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/sample/dataset_10.json.gz diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/SparkResultToCommunityFromOrganizationJob.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/SparkResultToCommunityFromOrganizationJob.java index fe79f1be1..df8ca3805 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/SparkResultToCommunityFromOrganizationJob.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/SparkResultToCommunityFromOrganizationJob.java @@ -60,8 +60,6 @@ public class SparkResultToCommunityFromOrganizationJob { conf, isSparkSessionManaged, spark -> { - // removeOutputDir(spark, outputPath); - execPropagation(spark, inputPath, outputPath, possibleupdatespath); }); diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java index efabcc6bc..64eb86338 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java @@ -40,7 +40,9 @@ public class BulkTagJobTest { "\"fos\" : \"$['subject'][?(@['qualifier']['classid']=='FOS')].value\"," + "\"sdg\" : \"$['subject'][?(@['qualifier']['classid']=='SDG')].value\"," + "\"hostedby\" : \"$['instance'][*]['hostedby']['key']\" , " + - "\"collectedfrom\" : \"$['instance'][*]['collectedfrom']['key']\"} "; + "\"collectedfrom\" : \"$['instance'][*]['collectedfrom']['key']\"," + + "\"publisher\":\"$['publisher'].value\"," + + "\"publicationyear\":\"$['dateofacceptance'].value\"} "; private static SparkSession spark; diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareAssocTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareAssocTest.java deleted file mode 100644 index 6536ecc85..000000000 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareAssocTest.java +++ /dev/null @@ -1,95 +0,0 @@ - -package eu.dnetlib.dhp.resulttocommunityfromorganization; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.apache.spark.SparkConf; -import org.apache.spark.api.java.JavaRDD; -import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.sql.Encoders; -import org.apache.spark.sql.SparkSession; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.gson.Gson; - -import eu.dnetlib.dhp.api.Utils; -import eu.dnetlib.dhp.api.model.CommunityEntityMap; -import eu.dnetlib.dhp.bulktag.BulkTagJobTest; -import eu.dnetlib.dhp.bulktag.SparkBulkTagJob; -import eu.dnetlib.dhp.schema.oaf.Dataset; - -/** - * @author miriam.baglioni - * @Date 13/10/23 - */ -public class PrepareAssocTest { - - private static SparkSession spark; - - private static Path workingDir; - - private static final Logger log = LoggerFactory.getLogger(PrepareAssocTest.class); - - @BeforeAll - public static void beforeAll() throws IOException { - workingDir = Files.createTempDirectory(BulkTagJobTest.class.getSimpleName()); - log.info("using work dir {}", workingDir); - - SparkConf conf = new SparkConf(); - conf.setAppName(BulkTagJobTest.class.getSimpleName()); - - conf.setMaster("local[*]"); - conf.set("spark.driver.host", "localhost"); - conf.set("hive.metastore.local", "true"); - conf.set("spark.ui.enabled", "false"); - conf.set("spark.sql.warehouse.dir", workingDir.toString()); - conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString()); - - spark = SparkSession - .builder() - .appName(PrepareAssocTest.class.getSimpleName()) - .config(conf) - .getOrCreate(); - } - - @AfterAll - public static void afterAll() throws IOException { - FileUtils.deleteDirectory(workingDir.toFile()); - spark.stop(); - } - - @Test - void test1() throws Exception { - - PrepareResultCommunitySet - .main( - new String[] { - - "-isSparkSessionManaged", Boolean.FALSE.toString(), - "-sourcePath", - getClass().getResource("/eu/dnetlib/dhp/resulttocommunityfromorganization/relation/").getPath(), - "-outputPath", workingDir.toString() + "/prepared", - "-production", Boolean.TRUE.toString(), - "-hive_metastore_uris", "" - }); - - final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); - - JavaRDD tmp = sc - .textFile(workingDir.toString() + "/prepared") - .map(item -> new ObjectMapper().readValue(item, ResultCommunityList.class)); - - tmp.foreach(r -> System.out.println(new ObjectMapper().writeValueAsString(r))); - } - -} diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromorganization/ResultToCommunityJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromorganization/ResultToCommunityJobTest.java index da3be8b35..6440ded92 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromorganization/ResultToCommunityJobTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromorganization/ResultToCommunityJobTest.java @@ -72,15 +72,13 @@ public class ResultToCommunityJobTest { SparkResultToCommunityFromOrganizationJob .main( new String[] { - "-isTest", Boolean.TRUE.toString(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", getClass() - .getResource("/eu/dnetlib/dhp/resulttocommunityfromorganization/sample") + .getResource("/eu/dnetlib/dhp/resulttocommunityfromorganization/sample/") .getPath(), - "-hive_metastore_uris", "", - "-resultTableName", "eu.dnetlib.dhp.schema.oaf.Dataset", - "-outputPath", workingDir.toString() + "/dataset", + "-outputPath", workingDir.toString() + "/", "-preparedInfoPath", preparedInfoPath }); diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/sample/dataset b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/sample/dataset new file mode 100644 index 000000000..61a1da1ac --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/sample/dataset @@ -0,0 +1,10 @@ +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055868909,"id":"50|dedup_wf_001::afaf128022d29872c4dad402b2db04fe","originalId":["od______3989::02dd5d2c222191b0b9bd4f33c8e96529"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[{"value":"10.4185/RLCS-2018-1243","qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofcollection":"2020-03-03T13:05:26.091Z","dateoftransformation":"2020-03-03T13:06:53.161Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:26.091Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/562","datestamp":"2018-01-23T15:06:07Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"affiliation":[],"fullname":"Jung, Nicole","name":"Nicole","pid":[{"dataInfo":null,"qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":null,"schemename":null},"value":"0000-0001-9513-2468"}],"rank":1,"surname":"Jung"},{"affiliation":[],"fullname":"Gräßle, Simone","name":"Simone","pid":[],"rank":2,"surname":"Gräßle"}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Comunicación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"yihadismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"viralidad","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"vídeo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"propaganda","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"terrorismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"terrorism","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"jihadism","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"virality","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"video","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"5903 Ideologías Políticas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"6310.13 Terrorismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"El individualismo como estrategia","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Individualism as an improvised strategy","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Se analiza como el Yihadismo distribuye su mensaje propaganístico en You Tube.\nUniversidad Camilo José Cela","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"MediaLab - UCJC","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[{"dataInfo":[{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"propagation:community:organizationtoresult","classname":"Propagation of community result through organization to result association","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.85"}],"id":"beopen"}],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/562"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055868954,"id":"50|dedup_wf_001::3f62cfc27024d564ea86760c494ba93b","originalId":["od______3989::05d8c751462f9bb8d2b06956dfbc5c7b"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:26.079Z","dateoftransformation":"2020-03-03T13:06:53.239Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:26.079Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/227","datestamp":"2017-11-15T16:29:55Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"affiliation":[],"fullname":"Fischer, Malte","name":"Malte","pid":[{"dataInfo":null,"qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":null,"schemename":null},"value":"0000-0002-2806-1302"}],"rank":1,"surname":"Fischer"},{"affiliation":[],"fullname":"Barbul, Daniel","name":"Daniel","pid":[],"rank":2,"surname":"Barbul"},{"affiliation":[],"fullname":"Schmidtmann, Marc","name":"Marc","pid":[{"dataInfo":null,"qualifier":{"classid":"MAG Identifier","classname":"MAG Identifier","schemeid":null,"schemename":null},"value":"2024269002"}],"rank":3,"surname":"Schmidtmann"},{"affiliation":[],"fullname":"Beckhaus, Ruediger","name":"Ruediger","pid":[],"rank":4,"surname":"Beckhaus"}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Actividad Física y del Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Eventos deportivos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Retos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"I Congreso Nuevos Retos en los Eventos Deportivos","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"I Congreso Nuevos Retos en los Eventos Deportivos, celebrado en la Universidad Camilo José Cela.\n\n\n\nDerivado del proyecto de investigación \"La situación de los eventos deportivos en España a través de su estructuras y de sus herramientas\", tiene como finalidad conocer la situación de los eventos deportivos en España. Se realizó un estudio descriptivo holístico en el que se investigaron elementos culturales, el perfil de los recursos humanos, la estructura de los comités, entre otras.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2015-06-20","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/227"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2015-06-20","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055869097,"id":"50|od________18::8887b1df8b563c4ea851eb9c882c9d7b","originalId":["od______3989::0f89464c4ac4c398fe0c71433b175a62"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:26.069Z","dateoftransformation":"2020-03-03T13:06:53.54Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:26.069Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/802","datestamp":"2018-11-14T15:30:47Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Ciencias de la Actividad Física y del Deporte","name":"Ciencias La Actividad Física Y. Del Deporte","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Actividad Física y del Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Actividades deportivas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Impacto físico","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Proyectos de investigación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"2411.06 Fisiología del Ejercicio","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Investigaciones en Ciencias de la Actividad Física y del Deporte CCAFD - UCJC","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Blanca Romero, investigadora y docente de Ciencias de la Actividad Física y del Deporte en la Universidad Camilo José Cela, describe dos de las líneas de investigación que se han desarrollado en el departamento CCAFD de UCJC: el impacto físico de las diferentes actividades deportivas y los métodos de recuperación más eficaces tras el ejercicio físico.\nBlanca también destaca la importancia en el ámbito de la salud los estudios en Ciencias de la Actividad Física y del Deporte.\nCiencias de la Actividad Física y del Deporte en la Universidad Camilo José Cela","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2016-03-21","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Universidad Camilo José Cela, Ciencias de la Actividad Física y del Deporte","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[{"dataInfo":[{"deletedbyinference":false,"inferenceprovenance":"","inferred":true,"invisible":false,"provenanceaction":{"classid":"propagation:community:organizationtoresult","classname":"Propagation of community result through organization to result association","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.85"}],"id":"beopen"}],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/802"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2016-03-21","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055869215,"id":"50|doajarticles::8d817039a63710fcf97e30f14662c6c8","originalId":["od______3989::1729c3988199b95d1d566851af7d3c55"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:25.653Z","dateoftransformation":"2020-03-03T13:06:53.764Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:25.653Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/801","datestamp":"2018-11-14T15:19:38Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Ciencias de la Actividad Física y del Deporte","name":"Ciencias La Actividad Física Y. Del Deporte","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Actividad Física y del Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Prevención","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Hidratación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deportistas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Electrolitos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deportes de resistencia","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"2411.06 Fisiología del Ejercicio","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"La importancia de la hidratación en competiciones resistencia, por Juan del Coso","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Juan del Coso, investigador y profesor en el Grado en Ciencias de la Actividad Física y del Deporte UCJC, presenta el proyecto de investigación que ha realizado el departamento CCAFD de UCJC: la prevención de desequilibrios de agua y electrolitos que se dan en los deportes de resistencia para que los deportistas realicen su actividad de una manera más segura.\nMás información: http://www.ucjc.edu/2016/03/investiga...\nUniversidad Camilo José Cela, Ciencias de la Actividad Física y del Deporte","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2016-03-21","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Grado en Ciencias de la Actividad Física y del Deporte UCJC","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[{"dataInfo":[{"deletedbyinference":false,"inferenceprovenance":"","inferred":true,"invisible":false,"provenanceaction":{"classid":"propagation:community:organizationtoresult","classname":"Propagation of community result through organization to result association","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.85"}],"id":"euromarine"}],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/801"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2016-03-21","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055869487,"id":"50|doajarticles::3c98f0632f1875b4979e552ba3aa01e6","originalId":["od______3989::2e3f34ce90520fae350a7e1148d7dcea"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.912Z","dateoftransformation":"2020-03-03T13:06:54.218Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.912Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/800","datestamp":"2018-11-14T13:43:21Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Actividad Física y del Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Cafeína","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Efectos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Alto rendimiento","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Competiciones","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Beneficios","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"2411.06 Fisiología del Ejercicio","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Investigación - Efectos positivos y negativos del uso de la cafeína en el deporte","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Proyecto de investigación que se están desarrollando en la Universidad Camilo José Cela (UCJC) sobre la cafeína y sus efectos cuando se realiza deporte. https://www.ucjc.edu/2016/08/la-bbc-s...\n\nJuan Del Coso, director del Laboratorio de Fisiología del Ejercicio de la Universidad Camilo José Cela, nos describe cómo está siendo la investigación que desarrolla con deportistas de alto rendimiento y el consumo de cafeína.\n\n\"Nuestra investigación se enmarca dentro del campo de la fisiología del ejercicio y el objetivo que tenemos es investigar los efectos que tiene la cafeína en los deportistas, principalmente saber si es una ayuda ergogénica en la mayor parte de los deportes, pero también conocer qué perjuicios existen para aquellos deportistas que la utilizan para incrementar el rendimiento deportivo\", explica Del Coso.\n\nY es que los datos son muy llamativos: tres de cada cuatro deportistas de alto rendimiento utilizan la cafeína antes de la competición y muchas veces se utiliza sin saber los perjuicios que puede tener para el deportista. \"Lo que estamos investigando es cómo una sustancia, que te puede hacer rendir mejor, cómo puede afectar negativamente cuando termina una competición\".\n\nPero esta investigación no sólo puede extraer conclusiones interesantes y útiles para los deportistas profesionales, \"investigando el deporte de alto rendimiento podemos conocer los beneficios y perjuicios de la cafeína que también afectarían al deportista amateur y hacer un deporte en general más seguro\".\n\nPara el desarrollo de este proyecto de investigación están colaborando el Laboratorio de Fisiología del Ejercicio de la Universidad Camilo José Cela y la Agencia para la Protección de la Salud en el Deporte.\nPara el desarrollo de este proyecto de investigación están colaborando el Laboratorio de Fisiología del Ejercicio de la Universidad Camilo José Cela y la Agencia para la Protección de la Salud en el Deporte.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2017-10-06","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Universidad Camilo José Cela, Vicerrectorado de Innovación","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[{"dataInfo":[{"deletedbyinference":false,"inferenceprovenance":"","inferred":true,"invisible":false,"provenanceaction":{"classid":"propagation:community:organizationtoresult","classname":"Propagation of community result through organization to result association","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.85"}],"id":"euromarine"},{"dataInfo":[{"deletedbyinference":false,"inferenceprovenance":"","inferred":true,"invisible":false,"provenanceaction":{"classid":"propagation:community:organizationtoresult","classname":"Propagation of community result through organization to result association","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.85"}],"id":"ni"}],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/800"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2017-10-06","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055869498,"id":"50|od______3989::2f4f3c820c450bd08dac08d07cc82dcf","originalId":["od______3989::2f4f3c820c450bd08dac08d07cc82dcf"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.806Z","dateoftransformation":"2020-03-03T13:06:54.229Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.806Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/797","datestamp":"2018-11-14T12:12:23Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Cuesta Cano, Laura","name":"Laura","surname":"Cuesta Cano","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Comunicación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Marketing digital","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Medios sociales","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Marcas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Empresas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Emprendedores","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"II Liga de Debate CICAE - UCJC","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"6114.13 Marketing","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Laura Cuesta - Liga de Debate CICAE - UCJC","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Laura Cuesta, profesora en la Universidad Camilo José Cela (UCJC) y Especialista en Marketing Digital y Medios Sociales para marcas y emprendedores. \n\nEn esta ocasión, Laura Cuesta, nos explica el significado de redes sociales y nos habla sobre el uso que le dan los usuarios.\n\nII Liga de Debate CICAE - UCJC http://ligacicae.ucjc.edu/\n\nLa Universidad Camilo José Cela (UCJC) y la Asociación de Colegios Privados e Independientes –Círculo de Calidad Educativa (CICAE) – organizan la segunda edición de esta gran liga de debate académico. El formato es al estilo inglés, similar al de la Oxford Union, donde algunos jóvenes se reunían a debatir sobre temas de la más candente actualidad y polemizaban con la intención de formarse en tan valoradas habilidades.\n\nLa Liga de Debate CICAE-UCJC es una ocasión única para los alumnos de ponerse en contacto con herramientas tan útiles como la oratoria y la comunicación efectiva. Pero, sobre todo, una oportunidad de fomentar el pensamiento crítico y la socialización entre alumnos con espíritu emprendedor. La Liga de Debate CICAE - Universidad Camilo José Cela se diferencia del resto de torneos en que potenciamos que sea eminentemente formativa. Alumnos y profesores reciben formación en comunicación, oratoria y debate por parte de alguno de los expertos en debate españoles más reconocidos en la escena internacional.\n\nDatos de la III Liga de Debate CICAE - UCJC http://ligacicae.ucjc.edu/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-10-23","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Universidad Camilo José Cela, Vicerrectorado de Innovación","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/797"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-10-23","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055870582,"id":"50|od______3989::752fd0b2bbac1ea1cc50e52fd46eb663","originalId":["od______3989::752fd0b2bbac1ea1cc50e52fd46eb663"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.561Z","dateoftransformation":"2020-03-03T13:06:55.716Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.561Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/798","datestamp":"2018-11-14T12:57:26Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Psicología","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Alcohol","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Dependencia","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Proyectos de investigación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Alcoholismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Realidad virtual","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Psicología conductual","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Investigación - Intervención basada en realidad virtual sobre la dependencia del alcohol","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Proyecto de investigación que se están desarrollando en la Universidad Camilo José Cela (UCJC) con tecnología aplicada al abandono de la dependencia del alcohol. \nRosa Jurado, profesora e investigadora de UCJC https://www.ucjc.edu/, explica cómo se conjuga el alcoholismo con la realidad virtual. \"Mi investigación consiste en desarrollar una técnica de intervención o rehabilitación que tiene como propósito fomentar la capacidad de inhibición para que las personas que tienen dependencia del alcohol sean capaces de controlar su conducta de aproximación hacia aquello que tiene que ver con el consumo de alcohol\".\n\nEn esta investigación, liderada por la Universidad Camilo José Cela, están participando, el Instituto de Investigación Biomédica del Hospital 12 de Octubre y la Universidad Politécnica de Madrid, con el CeDint. \"En nuestro equipo de investigación pensamos que, además del deseo de consumo, existen una carencia clara de capacidad de control inhibitorio en este tipo de personas\", destaca Rosa Jurado.\n\nLa realidad virtual sirve, según explica Gabriel Rubio, jefe de Psiquiatría del Hospital 12 de Octubre, \"para dar un paso más, para ver cómo reacciona un sujeto en un ambiente que no es el real, pero se parece mucho al real\". El objetivo final de estas acciones es \"mejorar la capacidad del individuo para poner en marcha mecanismos de inhibición conductual, para que el sujeto sea capaz de decir que no\".\n\nY no acaba ahí el proceso. La fase final del tratamiento se enfoca en que el paciente \"sea capaz de reconstruir su vida\" a través de programas de entrenamiento de habilidades sociales, asertividad, familias... \"Juntando todas las áreas de intervención, la abstinencia se mantiene mucho más tiempo y las recaídas disminuyen\".","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2017-10-05","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Universidad Camilo José Cela, Vicerrectorado de Innovación","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/798"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2017-10-05","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055870750,"id":"50|od______3989::7fcbe3a03280663cddebfd3cb9203177","originalId":["od______3989::7fcbe3a03280663cddebfd3cb9203177"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:25.652Z","dateoftransformation":"2020-03-03T13:06:55.95Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:25.652Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/796","datestamp":"2018-11-05T13:36:54Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Comunicación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Redes sociales","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Comunicación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Información","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Desinformación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"III Liga de Debate CICAE","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Instantaneidad","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Información periodística","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Fake news","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"5910.02 Medios de Comunicación de Masas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Jorge Gallardo - Liga de Debate CICAE","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Jorge Gallardo, profesor de Derecho en la Universidad Camilo José Cela (UCJC), Doctor en Comunicación Audiovisual y MBA en Empresas Audiovisuales. Subdirector de Espejo Público en Antena 3 Noticias. \n\nEn esta ocasión, Jorge Gallardo habla sobre las redes sociales, cómo han transformado la manera en la que nos comunicamos y nos informamos. \n\nIII Liga de Debate CICAE - UCJC http://ligacicae.ucjc.edu/\n\nLa Universidad Camilo José Cela (UCJC) y la Asociación de Colegios Privados e Independientes –Círculo de Calidad Educativa (CICAE) – organizan la tercera edición de esta gran liga de debate académico. El formato es al estilo inglés, similar al de la Oxford Union, donde algunos jóvenes se reunían a debatir sobre temas de la más candente actualidad y polemizaban con la intención de formarse en tan valoradas habilidades.\n\nLa Liga de Debate CICAE-UCJC es una ocasión única para los alumnos de ponerse en contacto con herramientas tan útiles como la oratoria y la comunicación efectiva. Pero, sobre todo, una oportunidad de fomentar el pensamiento crítico y la socialización entre alumnos con espíritu emprendedor. La Liga de Debate CICAE - Universidad Camilo José Cela se diferencia del resto de torneos en que potenciamos que sea eminentemente formativa. Alumnos y profesores reciben formación en comunicación, oratoria y debate por parte de alguno de los expertos en debate españoles más reconocidos en la escena internacional.\n\nDatos de la III Liga de Debate CICAE - UCJC http://ligacicae.ucjc.edu/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-10-23","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/796"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-10-23","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055872001,"id":"50|od______3989::d791339867bec6d3eb2104deeb4e4961","originalId":["od______3989::d791339867bec6d3eb2104deeb4e4961"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.586Z","dateoftransformation":"2020-03-03T13:06:57.721Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.586Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/645","datestamp":"2018-04-12T07:59:27Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Transferencia de Conocimiento e Innovación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Innovación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Investigación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Ciencia","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Tecnología","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"UCJC Open Science Day 2018","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"El UCJC Open Science Day tiene como objetivo mostrar las actividades de investigación llevadas a cabo por investigadores de la Universidad Camilo José Cela (UCJC) a todos los miembros de la UCJC.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-04-10","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/645"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-04-10","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055872022,"id":"50|od______3989::d90d3a1f64ad264b5ebed8a35b280343","originalId":["od______3989::d90d3a1f64ad264b5ebed8a35b280343"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.912Z","dateoftransformation":"2020-03-03T13:06:57.747Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.912Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/795","datestamp":"2018-11-06T15:43:10Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Comunicación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Unión Europea","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Desinformación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Política","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Periodismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Ciudadanos europeos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Democracias","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Estado de derecho","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Derechos humanos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"5910.02 Medios de Comunicación de Masas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Ramón Luis Varcárcel - La Unión Europea ante el reto de la desinformación","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Conferencia de Ramón Luis Valcárcel, vicepresidente del Parlamento Europeo, en el Campus de Almagro de la Universidad Camilo José Cela (UCJC) bajo el título \"La Unión Europea ante el reto de la desinformación\", en la que ha alertado sobre el alcance de la desinformación en las democracias y sobre cómo pone en peligro los derechos de los ciudadanos europeos. También ha participado el rector de la UCJC, Samuel Martín-Barbero.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-10-11","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/795"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-10-11","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/sample/dataset_10.json.gz b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/sample/dataset_10.json.gz deleted file mode 100644 index dccc28c879579c27eddfa3fd64c2e55a862b82b9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7128 zcmV;}8zpT17X4im?zDv9FZ`3?^$o{xt=^b%UqvwSXlvmbgQzYv9pQ{J1+XnkNMcuw`5J-E<(r>qn*$BJ1>i$Xvx zy<%O6Y$a7WV@ejqcQpNGrua%0qLgY4#r3T@&G=nzF4&T1Tt7I;c{yw9ex&Ozw)fa4 zUUS#mxBVW}TUIMj7cWJ|i;`xm-pr3Ckrzdi(PZjPq0v}V^Jg&MG zLDLEwd&*avpHNri6lyPNO8C*APoA8>Esa&=Rhqtq8dlggcy3+T_Xb0M5*@#Idh+rp z@chY<9}Gh%`fWv1F&A9z0-H#&ho1T65=0_-Q?6J18fdmx-}as9-pzCoJv3W6w^pJh z?*(4q9eE-AdF6*QFPa6Tfj9O4tykY&Dw-E_sWN(XiP0I2d@xAjX@7 z9cw>Bt8yuo?rU^D7pbu7N4?IWGg`QxRQaM$z7&8dIuQHo_Iur3cRTX6z4NOVCugU7 zCR+d2SfY7E+v>D;XgrDs{~W=8{?Rn@!=qq0n)J~8z>>ZJX9;C1)si1N@y(6=LT&zc z^FJx?lb0fsd8?t@@1~*KI)Zi`Unu&^e_?qe1TU(zMDMZZnLtGaFZaFE_SgNMXr6;^ zNTzZjuJKft@6uyPzNqM8PXJz|$6zgc0=!(&TolVI1gM{Kb;Zrqw>U0Tc?teRYpAH= zbLjp%Zu~@mqZ70s2`4EfCo-#Y!6*}(pYrRS`JSk|GVV*|T=m^ub-zJoEtWJ9MJ62# zhlJrmD9}Aj*c=Q8g<^)Mvwkkq&HEBwstX6qA%Pi%zBlkg^2$x0 z4!RayiV{`RT7-I>5|JkYoEM@2zz`^A_>IuRv* zvwK5Z`n_8m#J{G?owZRV;0tL^Kn^G+3Z4*)?+tSX2XE)#)hp>Kzo2=U=*SH&Ehp{B zPOE3X-^8TxZtY51vzp;Gm%R2L>8NO=Yc%?Q9B#OeL#HJH1Qfum=~_O(Y(EBB3OjyT})82@?oT zGbD1!Y zWPPRn)fULBEN-Sg3tRzXTVBuGfBC`T|4>}hXm}mhG@OqDHfO;YoL~}-hMZ1Dz6$Z z4|gON>c`gmASo>p`k8S=z2_^9Ac(Hk>=SM}Q$qE%KD+M#AwfU_e@S>nVJa`T*UibQ7 zoIK|y5<16MaNPhVbIG{EcHL8m0eQ)Z;uv4u{6wl8g{TsNZM3StkYtJUqirY^Gy!jv zq-ZHYN&FdM#)=rsO`#Df2eqTxxPp0jEsesZrUE`JE10MwG9Yx*^7=MtpeR8#@C2A+ zaw0)h#lS59K45eBHOw`iOoyW(8&Wo8 z@Y9@oZ0rxiF!sk0HSzQghPg)f4Rb}q4;keebt1|&@vcnHfl;m@Cg%vf+3-Uo=V-+v zZhRf>lY6hE+sAIpJ>4lJe)+pD_4NRXw2L*-jjILXzx=(X$!-H%t0sK26LG0+)P+V| zXyl_CS=#HsTX1-@h~jo)9e7(v2JRQe$hqz=ZX4D?!FX;LMvKcUqYjuu0uu~E?aT=@p*84iYUXMxqL; z6ug|;_#4M?(26Q}+rp5MHC}iGTJH=mi{0_Pt}PK7e0WhK^vYQoGz-H=+2&m-t&|vX zT5)Bi$~Hej_nKpZIw*)yC9ICu4zaFq@H)4;1{q_qd8ZpuQuqP%WQ>ENf^Bm)Kf+z5 zm6H?zS7L&1CY<&wdNfGz@EE$K#Rm+ajc_?5aJv#Z5}DY@XzYT<4;eHL{HV1zQqpsHQ57uZ#cVc7CjQt9r*ssK zeQ(a@(=iXdxj!6@0ybikTY|>^IGAFvc;Zi|@ia>OBpQt-kx%F2BxF(4#h`IC7)9Yd zgT~|0hl?6-`!Om#6^u=)YwIhe=kGJZcq$~_bv!_a3KCwBnq_VkT^IJ z&M$Qa1bCZu6qW<;xW-wxlBsYTeLK-5xQ!MT&^Pon%Yov!D-NDpxhqe1tM?LQXb&NrTV=p}3s%9{_ElsuyqC4KnN2Jr=h`{(t(h;hO@2Do9SjTcylixA^w^B`plU9C$U8+gQrjxlh3WK>n z8AtJOI-c?<3St^k>iK-syP0Yd@Ngau8J~Jl;LRxyBae=`?++)*H~}qokZKYR2U9<| z7u94q2>i*1qM9`F9G+@olfqm{H3?_Ka2EInrJDQ&JP>qwhTci1%OhubE-$uGSNoK` z+v=5w88z=3ekY$&N-NXiqTv(maS z>|q^KJFe=WI3yIG@f`P+3+JF5am2xM2ja-tZYchU*zuic8FZrZHOV=~IWBZ8JSA08 zr&wTiM%JX0xOtZBs3To8in~)sp5M&jC^)9t*!p;`Dm1I&b*?!ur&|t8BFu7#Ht(l2mWt|? zAZ+Kg9?bTETh?T9-Dzyf7Pk2b4^KxOkwW15o=#+PS(A$3?^|WeOSY{6(9wd&MQ4>ApCpH0Wt(23KFOL3?R)4 zqNFMXnxDNB#8ueaN@y}Drj5<^#0NY$f`7ll$k*2NG(cZtyS8K_imECA^sHJFFo4W> z-We&C;VCf+jfZ|<0K3^RwoQ5PKZ@_>EV@}T?NqJT2C z^o3%tDxMhpmC?p_C_upmcrJ%A5xsCKTlS%yLt(oGA`4yJFc|yRrtHI)6v5IhQzYZ( zQuS>qbDg4zf$r@u>JGz=7NzklT5XEr~jbsc(j8>9f! zVQEzu5~av-QMxcHU6akvpfLEP)v#|lhFS;Z%&8o&d%Jqrj-OW|xsr+QUfWYf;B&m4 z>>_cm0cCG+)|#vh-)ahaG<*_^A^B`%dk@mD+RR_?F#tV2*Fu5x%9@X}+bK=Q+Nm z<6Ax`-*P&+d>)h~OwNb%kWB)Q4I?j3yh%bC{N;@q{FtzL?-su0ZN)pux14yR`|>S= z=?COnT6qr5w;WH$z3cgw!3_Qc;fLj0o>Y2W@Py{FPoC0BA@|5$tG_mU$=2OHM~-|@ za^#7`IX-jwgAO0re@4|e;Mem7NsKFWa2ym4u1_}5@3g67A051JFqhHk!2Hw3|^R=5cNaOA_wDH)6Xuv6VH~Og2L`<0&b`A{Thtnu&LVCGh5r z0qMPP`;x|ZW}2p7T8^)#q?vSC$Tq5qieh#xP5QZcP^mLqrJ5MSC~pXvSorCkwTDy} zE$Eioaw1c{Ky5u&;)3Goau~o*IIi9ab12>a{pWw5Y~CwY8P3iLC?BhNoK#Gc!^j`C ztbf8Aw^2fkDe{_fq!uP(TcK9;Lg5Xz!HH=w1$}PbWq3{t*e1X^ z-MlOM00Fd-Dnq%#s=oRj3M8lp3GgK{*Q76~9$Iw1`3X$dFz^(wazoZK^$KM{DtWeD z9?_&MM)O3|-k?WTw#{p@0)1uBzL;v#qv4yDt<-5+Zl$@Pq=cKm`jHtywMmH79PT%; zuw8|@UI?wJnYBfd&Ci;cxh29upUMmu7DURqtq_R-6sc4w+7g@;)pR$g97Uvibqd8&@oh^`WL~Vm(nMLcd+C5&qbckiH`+ZWr!$8unk0_MBrOcl3>uzubA+Mh zTuPIvF59NH1FOwMjs{@Jk9G|iHS)+x++_FD=^vSh;b~1`I-4T0QEI=faclLfHpA8! zY|XU=kHb}IWVCnVdhF&O5<%293Y8*%Q6%OrXfpU-@Y04ynd_5RX{xR5l$Kj6C{o)ewS!I4CT$Sfx3vTV*_=94TM48F0+B$5a`%Ijt%s1Y@o3h zO@iy#K;tNwCte)HF=aldK4X!`@wefK$Dt2_hJJ@gRwvQfNUTu z&*9lXlip2iplCb`MlSoBBN*M0VD!8Y%x=PSF_*)pGGcNm9gFCoAe`#;b9dmyy_v8@ZCFwW)BIH<)Pf ziOlO%DA!mYvS41wf|4f{zC`bkb9N52qxGS2S7!%n5;B=&LeMx6?z~!P+6--m{F*wk z-%l{P(KE5HC(Oi!X{1abyw;M^JcM%dj%j9uR&TpE;tq8~xhPxuHVLyd0|Ix<+jXZc zeL?5>S;765*C6txh%I@gImC|YoK1zyX)6m?ew}uaOpu$$+Kf{ec9htHqN=^TZU6<5 zz85upU`g>oHfdH%c>Hu>5-08DDYF&YL_#&IX;+(l0vhMYNYUJb+cU0N^IYju9K;Ik zsgL!Z&G!(<<=o6Z7&6r3`4k9is5U9HTHcgpMX+hMOmWbmz1>nSIz#pJW2>K-#*f&Hb3XvzVro+ zl>o6{RIvbA&-q-FMPRA`O{b-*^B!HTWlW5vEjQ6U25>B-CXe)uL2FCWy%oe^nlOU5 z71cQ}b<5xjCcDxk7&N_HbFws-hNd+JW-2on1@VZARKJQ}l&JetmTqtVkX__>Z(``1qHmz(+LIis+ zxUvTz^g+}NmoYJ0BUJyS0#~OKxk;p@p*P$72yhjbd}trRpfPl++;jc0&ef z-KpRQOmeu^&w>Wg&3k-u0;q_*T60Vd=R$lBo-8YJI(5vU-zA!kIrQmb>zG3wVGd0? z%^Y$=pkod>=FmejhsKe2EpuqjVjfa23?|-a6tX1Y@jMAxJPo|iA7f&f8<|7574IH% zD2N85=zixmMuTbep@>6`IEN<=+1zcHh(lh4!iJbp$q|PfamZyR`UnS!-N-H2oR1WK`a6JqDaGbw7EV?pqVAxd1Wb?i# zVN`dx91@q$={KC@{5v=E8l zCnarP#bjPDtGIRcs7@AkF3I!F&lqUa@;t(1O_YS>ONn$gdTj= zP%UXrnw4U69BD@Jq(7?*Loh0~d4V!nCk(T1GVp_u=-21y>2yl4k}Iaw;ZlORlwd9; zm`e%vI~!nKK9rA%*yng=pR(aNo>?b&W}{B?%$!GdJTu2LdqAF9;CcRaJhNmx^+WhL z8poWCl90!N?+p{q;~^hTM}F^Sp4qL%yT>!bj8K#NomV#=j068eW~iEU;=H=i(2{$1 ziDusL$PZq5<5@JF1!G4vb40T{63t%ep4GhhlN0@(Y*!`rv2!7Y1I9@m5FG)+u@#Z6q?) zLz)DmVI1+8CleY*aWL`1;cX{-+*-Voag(Wkzy0~+!FV|SfN>Km&!OWcxgyhRgqBKzNq&;md(*HcYJ|y z?1?U#!`;~vCo*rdV0W!1wN}$77$(3?_QLFV#+|aysY+?4>9WR<@Z=P8G7HUCm39%2 z(u^*Yybn<#rsonLQo=hMZYY1<`!xoOXRS1Q?3Z}@5m$P2H~W;$c|54NtIM0h*oN)g zjI3>0v$-EPRpVVNo(gPYs-2=8vFrw2c6q8jAg}QJl+8Ol(Wf~^Cy}N-b*=(S3+a=W zG^^msGq`c{K0o@5s+cS3=py&(4tLqq<@-*jjxN%QdN{g>ql Date: Fri, 3 Nov 2023 15:30:01 +0100 Subject: [PATCH 096/148] refactoring --- .../src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java index 64eb86338..ba0c6b252 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java @@ -41,8 +41,8 @@ public class BulkTagJobTest { "\"sdg\" : \"$['subject'][?(@['qualifier']['classid']=='SDG')].value\"," + "\"hostedby\" : \"$['instance'][*]['hostedby']['key']\" , " + "\"collectedfrom\" : \"$['instance'][*]['collectedfrom']['key']\"," + - "\"publisher\":\"$['publisher'].value\"," + - "\"publicationyear\":\"$['dateofacceptance'].value\"} "; + "\"publisher\":\"$['publisher'].value\"," + + "\"publicationyear\":\"$['dateofacceptance'].value\"} "; private static SparkSession spark; From 5bc97615d50383c9adc3eab2c69f86bacbe4fb4b Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Fri, 3 Nov 2023 15:35:10 +0100 Subject: [PATCH 097/148] - --- .../dhp/actionmanager/bipfinder/SparkAtomicActionScoreJob.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/SparkAtomicActionScoreJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/SparkAtomicActionScoreJob.java index 8adc88920..040c89782 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/SparkAtomicActionScoreJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/bipfinder/SparkAtomicActionScoreJob.java @@ -95,7 +95,7 @@ public class SparkAtomicActionScoreJob implements Serializable { return projectScores.map((MapFunction) bipProjectScores -> { Project project = new Project(); - // project.setId(bipProjectScores.getProjectId()); + project.setId(bipProjectScores.getProjectId()); project.setMeasures(bipProjectScores.toMeasures()); return project; }, Encoders.bean(Project.class)) From 671ba8a5a7845df5f8718450c786814219a1bcd0 Mon Sep 17 00:00:00 2001 From: Serafeim Chatzopoulos Date: Tue, 7 Nov 2023 18:35:05 +0200 Subject: [PATCH 098/148] Clear working dir in bipranker workflow --- .../impact_indicators/oozie_app/workflow.xml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/dhp-workflows/dhp-impact-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/impact_indicators/oozie_app/workflow.xml b/dhp-workflows/dhp-impact-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/impact_indicators/oozie_app/workflow.xml index e43e7cf14..6ee5860d5 100644 --- a/dhp-workflows/dhp-impact-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/impact_indicators/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-impact-indicators/src/main/resources/eu/dnetlib/dhp/oa/graph/impact_indicators/oozie_app/workflow.xml @@ -48,16 +48,25 @@ ${wf:conf('resume') eq "format-results"} ${wf:conf('resume') eq "map-ids"} ${wf:conf('resume') eq "map-scores"} - ${wf:conf('resume') eq "start"} + ${wf:conf('resume') eq "start"} ${wf:conf('resume') eq "projects-impact"} ${wf:conf('resume') eq "create-actionset"} - + + + + + + + + + + @@ -606,6 +615,10 @@ Calculating project impact indicators failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + Re-create working dir failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + From d524e3086630f20e129ae339072e0198b16c7392 Mon Sep 17 00:00:00 2001 From: dimitrispie Date: Tue, 14 Nov 2023 09:46:52 +0200 Subject: [PATCH 099/148] Changes to actionsets Resolve comments from https://code-repo.d4science.org/D-Net/dnet-hadoop/pulls/355 --- dhp-workflows/dhp-stats-actionsets/pom.xml | 39 ---- .../dnetlib/dhp/actionmanager/Constants.java | 102 --------- .../StatsAtomicActionsJob.java | 206 ++++-------------- .../stats_actionsets/StatsDiamondOAModel.java | 29 --- .../stats_actionsets/StatsGreenOAModel.java | 29 --- .../stats_actionsets/StatsOAColourModel.java | 47 ---- .../StatsPubliclyFundedModel.java | 29 --- .../StatsResultEnhancementModel.java | 76 +++++++ .../stats_actionsets/oozie_app/workflow.xml | 28 --- 9 files changed, 116 insertions(+), 469 deletions(-) delete mode 100644 dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/Constants.java delete mode 100644 dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsDiamondOAModel.java delete mode 100644 dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsGreenOAModel.java delete mode 100644 dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsOAColourModel.java delete mode 100644 dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsPubliclyFundedModel.java create mode 100644 dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsResultEnhancementModel.java diff --git a/dhp-workflows/dhp-stats-actionsets/pom.xml b/dhp-workflows/dhp-stats-actionsets/pom.xml index 670fd4a5e..3daa8f995 100644 --- a/dhp-workflows/dhp-stats-actionsets/pom.xml +++ b/dhp-workflows/dhp-stats-actionsets/pom.xml @@ -8,46 +8,7 @@ dhp-stats-actionsets - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - org.apache.httpcomponents httpclient diff --git a/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/Constants.java b/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/Constants.java deleted file mode 100644 index 0df34364d..000000000 --- a/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/Constants.java +++ /dev/null @@ -1,102 +0,0 @@ - -package eu.dnetlib.dhp.actionmanager; - -import java.util.Optional; - -import org.apache.spark.api.java.function.MapFunction; -import org.apache.spark.sql.Dataset; -import org.apache.spark.sql.Encoders; -import org.apache.spark.sql.SparkSession; - -import com.fasterxml.jackson.databind.ObjectMapper; - -import eu.dnetlib.dhp.application.ArgumentApplicationParser; -import eu.dnetlib.dhp.schema.common.ModelConstants; -import eu.dnetlib.dhp.schema.oaf.StructuredProperty; -import eu.dnetlib.dhp.schema.oaf.Subject; -import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils; - -public class Constants { - - public static final String DOI = "doi"; - public static final String DOI_CLASSNAME = "Digital Object Identifier"; - - public static final String DEFAULT_DELIMITER = ","; - public static final String DEFAULT_FOS_DELIMITER = "\t"; - - public static final String UPDATE_DATA_INFO_TYPE = "update"; -// public static final String UPDATE_SUBJECT_FOS_CLASS_ID = "subject:fos"; - public static final String UPDATE_CLASS_NAME = "Inferred by OpenAIRE"; -// public static final String UPDATE_MEASURE_BIP_CLASS_ID = "measure:bip"; -// public static final String UPDATE_SUBJECT_SDG_CLASS_ID = "subject:sdg"; -// public static final String UPDATE_MEASURE_USAGE_COUNTS_CLASS_ID = "measure:usage_counts"; -// public static final String UPDATE_KEY_USAGE_COUNTS = "count"; - - public static final String UPDATE_MEASURE_STATS_MODEL_CLASS_ID = "measure:stats_model"; - public static final String UPDATE_KEY_STATS_MODEL = "stats_model"; - -// public static final String UPDATE_MEASURE_PUBLICLY_FUNDED_CLASS_ID = "measure:publicly_funded"; -// public static final String UPDATE_KEY_PUBLICLY_FUNDED = "publicly_funded"; - -// public static final String FOS_CLASS_ID = "FOS"; -// public static final String FOS_CLASS_NAME = "Fields of Science and Technology classification"; -// -// public static final String SDG_CLASS_ID = "SDG"; -// public static final String SDG_CLASS_NAME = "Sustainable Development Goals"; - - public static final String NULL = "NULL"; - - public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - - private Constants() { - } - - public static Boolean isSparkSessionManaged(ArgumentApplicationParser parser) { - return Optional - .ofNullable(parser.get("isSparkSessionManaged")) - .map(Boolean::valueOf) - .orElse(Boolean.TRUE); - } - - public static Dataset readPath( - SparkSession spark, String inputPath, Class clazz) { - return spark - .read() - .textFile(inputPath) - .map((MapFunction) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz)); - } - - public static Subject getSubject(String sbj, String classid, String classname, - String diqualifierclassid) { - if (sbj == null || sbj.equals(NULL)) - return null; - Subject s = new Subject(); - s.setValue(sbj); - s - .setQualifier( - OafMapperUtils - .qualifier( - classid, - classname, - ModelConstants.DNET_SUBJECT_TYPOLOGIES, - ModelConstants.DNET_SUBJECT_TYPOLOGIES)); - s - .setDataInfo( - OafMapperUtils - .dataInfo( - false, - UPDATE_DATA_INFO_TYPE, - true, - false, - OafMapperUtils - .qualifier( - diqualifierclassid, - UPDATE_CLASS_NAME, - ModelConstants.DNET_PROVENANCE_ACTIONS, - ModelConstants.DNET_PROVENANCE_ACTIONS), - "")); - - return s; - - } -} diff --git a/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsAtomicActionsJob.java b/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsAtomicActionsJob.java index f6e2cfe01..e984d68fc 100644 --- a/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsAtomicActionsJob.java +++ b/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsAtomicActionsJob.java @@ -1,7 +1,6 @@ package eu.dnetlib.dhp.actionmanager.stats_actionsets; -import static eu.dnetlib.dhp.actionmanager.Constants.*; import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession; import java.io.Serializable; @@ -12,14 +11,22 @@ import java.util.Optional; import org.apache.commons.io.IOUtils; import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.compress.GzipCodec; import org.apache.hadoop.mapred.SequenceFileOutputFormat; import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.api.java.function.MapGroupsFunction; +import org.apache.spark.sql.*; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoders; import org.apache.spark.sql.SaveMode; import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,6 +72,9 @@ public class StatsAtomicActionsJob implements Serializable { SparkConf conf = new SparkConf(); conf.set("hive.metastore.uris", parser.get("hive_metastore_uris")); + conf.set("spark.speculation", "false"); + conf.set("spark.hadoop.mapreduce.map.speculative", "false"); + conf.set("spark.hadoop.mapreduce.reduce.speculative", "false"); final String dbname = parser.get("statsDB"); @@ -75,75 +85,26 @@ public class StatsAtomicActionsJob implements Serializable { isSparkSessionManaged, spark -> { removeOutputDir(spark, outputPath); - prepareGreenData(dbname, spark, workingPath + "/greenOADB", "indi_pub_green_oa", "id"); - prepareDiamondData(dbname, spark, workingPath + "/diamondOADΒ", "indi_pub_diamond", "id"); - preparePubliclyFundedData( - dbname, spark, workingPath + "/publiclyFundedDΒ", "indi_funded_result_with_fundref", "id"); - prepareOAColourData(dbname, spark, workingPath + "/oacolourDB", "", "id"); + prepareResultEnhancement(dbname, spark, workingPath + "/resultEnhancements", "id"); writeActionSet(spark, workingPath, outputPath); }); } - private static void prepareGreenData(String dbname, SparkSession spark, String workingPath, String tableName, + private static void prepareResultEnhancement(String dbname, SparkSession spark, String workingPath, String resultAttributeName) { spark .sql( String .format( - "select %s as id, green_oa as green_oa " + - "from %s.%s", - resultAttributeName, dbname, tableName)) - .as(Encoders.bean(StatsGreenOAModel.class)) - .write() - .mode(SaveMode.Overwrite) - .option("compression", "gzip") - .json(workingPath); - } - - private static void prepareDiamondData(String dbname, SparkSession spark, String workingPath, String tableName, - String resultAttributeName) { - spark - .sql( - String - .format( - "select %s as id, in_diamond_journal as in_diamond_journal " + - "from %s.%s", - resultAttributeName, dbname, tableName)) - .as(Encoders.bean(StatsDiamondOAModel.class)) - .write() - .mode(SaveMode.Overwrite) - .option("compression", "gzip") - .json(workingPath); - } - - private static void preparePubliclyFundedData(String dbname, SparkSession spark, String workingPath, - String tableName, - String resultAttributeName) { - spark - .sql( - String - .format( - "select %s as id, fundref as publicly_funded " + - "from %s.%s", - resultAttributeName, dbname, tableName)) - .as(Encoders.bean(StatsPubliclyFundedModel.class)) - .write() - .mode(SaveMode.Overwrite) - .option("compression", "gzip") - .json(workingPath); - } - - private static void prepareOAColourData(String dbname, SparkSession spark, String workingPath, String tableName, - String resultAttributeName) { - spark - .sql( - String - .format( - "select b.%s as id, is_gold, is_bronze_oa, is_hybrid from %s.indi_pub_bronze_oa b " + + "select b.%s as id, is_gold, is_bronze_oa, is_hybrid,green_oa, in_diamond_journal,f.fundref as publicly_funded " + + "from %s.indi_pub_bronze_oa b " + "left outer join %s.indi_pub_gold_oa g on g.id=b.id " + - "left outer join %s.indi_pub_hybrid h on b.id=h.id", - resultAttributeName, dbname, dbname, dbname)) - .as(Encoders.bean(StatsOAColourModel.class)) + "left outer join %s.indi_pub_hybrid h on b.id=h.id " + + "left outer join %s.indi_pub_green_oa gr on b.id=gr.id " + + "left outer join %s.indi_pub_diamond d on b.id=d.id " + + "left outer join %s.indi_funded_result_with_fundref f on b.id=f.id ", + resultAttributeName, dbname, dbname, dbname, dbname, dbname, dbname)) + .as(Encoders.bean(StatsResultEnhancementModel.class)) .write() .mode(SaveMode.Overwrite) .option("compression", "gzip") @@ -152,125 +113,39 @@ public class StatsAtomicActionsJob implements Serializable { public static void writeActionSet(SparkSession spark, String inputPath, String outputPath) { - getFinalIndicatorsGreenResult(spark, inputPath + "/greenOADB") + getResultEnhancements(spark, inputPath + "/resultEnhancements") .toJavaRDD() .map(p -> new AtomicAction(p.getClass(), p)) - .union( - getFinalIndicatorsDiamondResult(spark, inputPath + "/diamondOADΒ") - .toJavaRDD() - .map(p -> new AtomicAction(p.getClass(), p))) - .union( - getFinalIndicatorsPubliclyFundedResult(spark, inputPath + "/publiclyFundedDΒ") - .toJavaRDD() - .map(p -> new AtomicAction(p.getClass(), p))) - .union( - getFinalIndicatorsOAColourResult(spark, inputPath + "/oacolourDB") - .toJavaRDD() - .map(p -> new AtomicAction(p.getClass(), p))) .mapToPair( aa -> new Tuple2<>(new Text(aa.getClazz().getCanonicalName()), new Text(OBJECT_MAPPER.writeValueAsString(aa)))) - .saveAsHadoopFile(outputPath, Text.class, Text.class, SequenceFileOutputFormat.class); + .saveAsHadoopFile( + outputPath, + Text.class, + Text.class, + SequenceFileOutputFormat.class, + GzipCodec.class); } - public static Measure newMeasureInstance(String id) { - Measure m = new Measure(); - m.setId(id); - m.setUnit(new ArrayList<>()); - return m; - } + private static Dataset getResultEnhancements(SparkSession spark, String inputPath) { - private static Dataset getFinalIndicatorsGreenResult(SparkSession spark, String inputPath) { - - return readPath(spark, inputPath, StatsGreenOAModel.class) - .map((MapFunction) usm -> { + return readPath(spark, inputPath, StatsResultEnhancementModel.class) + .map((MapFunction) usm -> { Result r = new Result(); r.setId("50|" + usm.getId()); - r.setMeasures(getMeasure(usm.isGreen_oa(), "green_oa")); + r.setIsInDiamondJournal(usm.isIn_diamond_journal()); + r.setIsGreen(usm.isGreen_oa()); + r.setPubliclyFunded(usm.isPublicly_funded()); + if (usm.isIs_bronze_oa()) + r.setOpenAccessColor(OpenAccessColor.bronze); + else if (usm.isIs_gold()) + r.setOpenAccessColor(OpenAccessColor.bronze); + else if (usm.isIs_gold()) + r.setOpenAccessColor(OpenAccessColor.gold); return r; }, Encoders.bean(Result.class)); } - private static Dataset getFinalIndicatorsDiamondResult(SparkSession spark, String inputPath) { - - return readPath(spark, inputPath, StatsDiamondOAModel.class) - .map((MapFunction) usm -> { - Result r = new Result(); - r.setId("50|" + usm.getId()); - r.setMeasures(getMeasure(usm.isIn_diamond_journal(), "in_diamond_journal")); - return r; - }, Encoders.bean(Result.class)); - } - - private static Dataset getFinalIndicatorsPubliclyFundedResult(SparkSession spark, String inputPath) { - - return readPath(spark, inputPath, StatsPubliclyFundedModel.class) - .map((MapFunction) usm -> { - Result r = new Result(); - r.setId("50|" + usm.getId()); - r.setMeasures(getMeasure(usm.isPublicly_funded(), "publicly_funded")); - return r; - }, Encoders.bean(Result.class)); - } - - private static Dataset getFinalIndicatorsOAColourResult(SparkSession spark, String inputPath) { - - return readPath(spark, inputPath, StatsOAColourModel.class) - .map((MapFunction) usm -> { - Result r = new Result(); - r.setId("50|" + usm.getId()); - r.setMeasures(getMeasureOAColour(usm.isIs_gold(), usm.isIs_bronze_oa(), usm.isIs_hybrid())); - return r; - }, Encoders.bean(Result.class)); - } - - private static List getMeasure(Boolean is_model_oa, String model_type) { - DataInfo dataInfo = OafMapperUtils - .dataInfo( - false, - UPDATE_DATA_INFO_TYPE, - true, - false, - OafMapperUtils - .qualifier( - UPDATE_MEASURE_STATS_MODEL_CLASS_ID, - UPDATE_CLASS_NAME, - ModelConstants.DNET_PROVENANCE_ACTIONS, - ModelConstants.DNET_PROVENANCE_ACTIONS), - ""); - - return Arrays - .asList( - OafMapperUtils - .newMeasureInstance(model_type, String.valueOf(is_model_oa), UPDATE_KEY_STATS_MODEL, dataInfo)); - } - - private static List getMeasureOAColour(Boolean is_gold, Boolean is_bronze_oa, Boolean is_hybrid) { - DataInfo dataInfo = OafMapperUtils - .dataInfo( - false, - UPDATE_DATA_INFO_TYPE, - true, - false, - OafMapperUtils - .qualifier( - UPDATE_MEASURE_STATS_MODEL_CLASS_ID, - UPDATE_CLASS_NAME, - ModelConstants.DNET_PROVENANCE_ACTIONS, - ModelConstants.DNET_PROVENANCE_ACTIONS), - ""); - - return Arrays - .asList( - OafMapperUtils - .newMeasureInstance("is_gold", String.valueOf(is_gold), UPDATE_KEY_STATS_MODEL, dataInfo), - OafMapperUtils - .newMeasureInstance("is_bronze_oa", String.valueOf(is_bronze_oa), UPDATE_KEY_STATS_MODEL, dataInfo), - OafMapperUtils - .newMeasureInstance("is_hybrid", String.valueOf(is_hybrid), UPDATE_KEY_STATS_MODEL, dataInfo)); - - } - private static void removeOutputDir(SparkSession spark, String path) { HdfsSupport.remove(path, spark.sparkContext().hadoopConfiguration()); } @@ -282,5 +157,4 @@ public class StatsAtomicActionsJob implements Serializable { .textFile(inputPath) .map((MapFunction) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz)); } - } diff --git a/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsDiamondOAModel.java b/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsDiamondOAModel.java deleted file mode 100644 index 340790fa8..000000000 --- a/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsDiamondOAModel.java +++ /dev/null @@ -1,29 +0,0 @@ - -package eu.dnetlib.dhp.actionmanager.stats_actionsets; - -import java.io.Serializable; - -/** - * @author dimitris.pierrakos - * @Date 30/10/23 - */ -public class StatsDiamondOAModel implements Serializable { - private String id; - private boolean in_diamond_journal; - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public boolean isIn_diamond_journal() { - return in_diamond_journal; - } - - public void setIn_diamond_journal(boolean in_diamond_journal) { - this.in_diamond_journal = in_diamond_journal; - } -} diff --git a/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsGreenOAModel.java b/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsGreenOAModel.java deleted file mode 100644 index 3d59bdf95..000000000 --- a/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsGreenOAModel.java +++ /dev/null @@ -1,29 +0,0 @@ - -package eu.dnetlib.dhp.actionmanager.stats_actionsets; - -import java.io.Serializable; - -/** - * @author dimitris.pierrakos - * @Date 30/10/23 - */ -public class StatsGreenOAModel implements Serializable { - private String id; - private boolean green_oa; - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public boolean isGreen_oa() { - return green_oa; - } - - public void setGreen_oa(boolean green_oa) { - this.green_oa = green_oa; - } -} diff --git a/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsOAColourModel.java b/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsOAColourModel.java deleted file mode 100644 index a83eb4f60..000000000 --- a/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsOAColourModel.java +++ /dev/null @@ -1,47 +0,0 @@ - -package eu.dnetlib.dhp.actionmanager.stats_actionsets; - -import java.io.Serializable; - -/** - * @author dimitris.pierrakos - * @Date 30/10/23 - */ -public class StatsOAColourModel implements Serializable { - private String id; - private boolean is_gold; - private boolean is_bronze_oa; - private boolean is_hybrid; - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public boolean isIs_gold() { - return is_gold; - } - - public void setIs_gold(boolean is_gold) { - this.is_gold = is_gold; - } - - public boolean isIs_bronze_oa() { - return is_bronze_oa; - } - - public void setIs_bronze_oa(boolean is_bronze_oa) { - this.is_bronze_oa = is_bronze_oa; - } - - public boolean isIs_hybrid() { - return is_hybrid; - } - - public void setIs_hybrid(boolean is_hybrid) { - this.is_hybrid = is_hybrid; - } -} diff --git a/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsPubliclyFundedModel.java b/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsPubliclyFundedModel.java deleted file mode 100644 index 6bdc78220..000000000 --- a/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsPubliclyFundedModel.java +++ /dev/null @@ -1,29 +0,0 @@ - -package eu.dnetlib.dhp.actionmanager.stats_actionsets; - -import java.io.Serializable; - -/** - * @author dimitris.pierrakos - * @Date 30/10/23 - */ -public class StatsPubliclyFundedModel implements Serializable { - private String id; - private boolean publicly_funded; - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public boolean isPublicly_funded() { - return publicly_funded; - } - - public void setPublicly_funded(boolean publicly_funded) { - this.publicly_funded = publicly_funded; - } -} diff --git a/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsResultEnhancementModel.java b/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsResultEnhancementModel.java new file mode 100644 index 000000000..f794c5591 --- /dev/null +++ b/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsResultEnhancementModel.java @@ -0,0 +1,76 @@ + +package eu.dnetlib.dhp.actionmanager.stats_actionsets; + +import java.io.Serializable; + +import eu.dnetlib.dhp.schema.oaf.*; + +/** + * @author dimitris.pierrakos + * @Date 30/10/23 + */ +public class StatsResultEnhancementModel implements Serializable { + private String id; + private Boolean is_gold; + private Boolean is_bronze_oa; + private Boolean is_hybrid; + private boolean in_diamond_journal; + private boolean green_oa; + private boolean publicly_funded; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public Boolean isIs_gold() { + return is_gold; + } + + public void setIs_gold(Boolean is_gold) { + this.is_gold = is_gold; + } + + public Boolean isIs_bronze_oa() { + return is_bronze_oa; + } + + public void setIs_bronze_oa(Boolean is_bronze_oa) { + this.is_bronze_oa = is_bronze_oa; + } + + public Boolean isIs_hybrid() { + return is_hybrid; + } + + public void setIs_hybrid(Boolean is_hybrid) { + this.is_hybrid = is_hybrid; + } + + public boolean isIn_diamond_journal() { + return in_diamond_journal; + } + + public void setIn_diamond_journal(boolean in_diamond_journal) { + this.in_diamond_journal = in_diamond_journal; + } + + public boolean isGreen_oa() { + return green_oa; + } + + public void setGreen_oa(boolean green_oa) { + this.green_oa = green_oa; + } + + public boolean isPublicly_funded() { + return publicly_funded; + } + + public void setPublicly_funded(boolean publicly_funded) { + this.publicly_funded = publicly_funded; + } +} diff --git a/dhp-workflows/dhp-stats-actionsets/src/main/resources/eu/dnetlib/dhp/actionmanager/stats_actionsets/oozie_app/workflow.xml b/dhp-workflows/dhp-stats-actionsets/src/main/resources/eu/dnetlib/dhp/actionmanager/stats_actionsets/oozie_app/workflow.xml index 1aa4e1050..79fd3f56e 100644 --- a/dhp-workflows/dhp-stats-actionsets/src/main/resources/eu/dnetlib/dhp/actionmanager/stats_actionsets/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-stats-actionsets/src/main/resources/eu/dnetlib/dhp/actionmanager/stats_actionsets/oozie_app/workflow.xml @@ -68,39 +68,11 @@ Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] - - - yarn - cluster - Produces the atomic action with the usage stats count for results - - eu.dnetlib.dhp.actionmanager.stats_actionsets.SparkAtomicActionGreenOAJob - dhp-stats-actionsets-${projectVersion}.jar - - --executor-memory=${sparkExecutorMemory} - --executor-cores=${sparkExecutorCores} - --driver-memory=${sparkDriverMemory} - --conf spark.extraListeners=${spark2ExtraListeners} - --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} - --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} - --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} - - --hive_metastore_uris${hiveMetastoreUris} - --outputPath${outputPath} - --statsDB${statsDB} - --workingPath${workingDir} - - - - - yarn cluster Produces the atomic action with the stats green_oa for results - eu.dnetlib.dhp.actionmanager.stats_actionsets.StatsAtomicActionsJob dhp-stats-actionsets-${projectVersion}.jar From 6ce36b3e412020619afd8a242d863ec3a4f632b5 Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Tue, 14 Nov 2023 12:04:29 +0100 Subject: [PATCH 100/148] Implemented ORCID Workflow on DHP-Aggregation for retrieving ORCID DUMP and generating tables --- .../orcid/DownloadORCIDDumpApplication.java | 102 +++ .../collection/orcid/ExtractORCIDDump.java | 71 +++ .../dhp/collection/orcid/ORCIDExtractor.java | 171 ++++++ .../dhp/collection/orcid/OrcidParser.java | 251 ++++++++ .../dhp/collection/orcid/model/Author.java | 83 +++ .../collection/orcid/model/Employment.java | 54 ++ .../dhp/collection/orcid/model/ORCIDItem.java | 14 + .../dhp/collection/orcid/model/Pid.java | 33 + .../dhp/collection/orcid/model/Work.java | 35 ++ .../orcid/download_orcid_parameter.json | 21 + .../orcid/extract_orcid_parameter.json | 21 + .../orcid/generate_orcid_table_parameter.json | 21 + .../orcid/oozie_app/config-default.xml | 23 + .../collection/orcid/oozie_app/workflow.xml | 81 +++ .../preprocess_orcid_dump_parameter.json | 21 + .../orcid/SparkGenerateORCIDTable.scala | 101 +++ .../collection/orcid/DownloadORCIDTest.java | 158 +++++ .../activity_work_0000-0002-2536-4498.xml | 69 +++ .../activity_work_0000-0002-5982-8983.xml | 79 +++ ...ty_work_0000-0003-2760-1191-similarity.xml | 113 ++++ .../activity_work_0000-0003-2760-1191.xml | 106 ++++ ..._work_0000-0003-2760-1191_contributors.xml | 101 +++ .../dhp/collection/orcid/employment.xml | 50 ++ .../dhp/collection/orcid/employment_2.xml | 55 ++ .../dhp/collection/orcid/employment_3.xml | 62 ++ .../dnetlib/dhp/collection/orcid/summary.xml | 581 ++++++++++++++++++ 26 files changed, 2477 insertions(+) create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/DownloadORCIDDumpApplication.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/ExtractORCIDDump.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/ORCIDExtractor.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/OrcidParser.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/model/Author.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/model/Employment.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/model/ORCIDItem.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/model/Pid.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/model/Work.java create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/download_orcid_parameter.json create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/extract_orcid_parameter.json create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/generate_orcid_table_parameter.json create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/oozie_app/config-default.xml create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/oozie_app/workflow.xml create mode 100644 dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/preprocess_orcid_dump_parameter.json create mode 100644 dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/orcid/SparkGenerateORCIDTable.scala create mode 100644 dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/orcid/DownloadORCIDTest.java create mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/activity_work_0000-0002-2536-4498.xml create mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/activity_work_0000-0002-5982-8983.xml create mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/activity_work_0000-0003-2760-1191-similarity.xml create mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/activity_work_0000-0003-2760-1191.xml create mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/activity_work_0000-0003-2760-1191_contributors.xml create mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/employment.xml create mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/employment_2.xml create mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/employment_3.xml create mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/summary.xml diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/DownloadORCIDDumpApplication.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/DownloadORCIDDumpApplication.java new file mode 100644 index 000000000..69661bbf6 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/DownloadORCIDDumpApplication.java @@ -0,0 +1,102 @@ + +package eu.dnetlib.dhp.collection.orcid; + +import static eu.dnetlib.dhp.utils.DHPUtils.getHadoopConfiguration; + +import java.io.InputStream; +import java.net.URL; +import java.net.URLConnection; +import java.util.Objects; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; + +public class DownloadORCIDDumpApplication { + private static final Logger log = LoggerFactory.getLogger(DownloadORCIDDumpApplication.class); + + private final FileSystem fileSystem; + + public DownloadORCIDDumpApplication(FileSystem fileSystem) { + this.fileSystem = fileSystem; + } + + public static void main(String[] args) throws Exception { + final ArgumentApplicationParser argumentParser = new ArgumentApplicationParser( + IOUtils + .toString( + Objects + .requireNonNull( + DownloadORCIDDumpApplication.class + .getResourceAsStream( + "/eu/dnetlib/dhp/collection/orcid/download_orcid_parameter.json")))); + argumentParser.parseArgument(args); + + final String hdfsuri = argumentParser.get("namenode"); + log.info("hdfsURI is {}", hdfsuri); + + final String targetPath = argumentParser.get("targetPath"); + log.info("targetPath is {}", targetPath); + + final String apiURL = argumentParser.get("apiURL"); + log.info("apiURL is {}", apiURL); + + final FileSystem fileSystem = FileSystem.get(getHadoopConfiguration(hdfsuri)); + + new DownloadORCIDDumpApplication(fileSystem).run(targetPath, apiURL); + + } + + private void downloadItem(final String name, final String itemURL, final String basePath) { + try { + final Path hdfsWritePath = new Path(String.format("%s/%s", basePath, name)); + final FSDataOutputStream fsDataOutputStream = fileSystem.create(hdfsWritePath, true); + final HttpGet request = new HttpGet(itemURL); + final int timeout = 60; // seconds + final RequestConfig config = RequestConfig + .custom() + .setConnectTimeout(timeout * 1000) + .setConnectionRequestTimeout(timeout * 1000) + .setSocketTimeout(timeout * 1000) + .build(); + log.info("Downloading url {} into {}", itemURL, hdfsWritePath.getName()); + try (CloseableHttpClient client = HttpClientBuilder.create().setDefaultRequestConfig(config).build(); + CloseableHttpResponse response = client.execute(request)) { + int responseCode = response.getStatusLine().getStatusCode(); + log.info("Response code is {}", responseCode); + if (responseCode >= 200 && responseCode < 400) { + IOUtils.copy(response.getEntity().getContent(), fsDataOutputStream); + } + } catch (Throwable eu) { + throw new RuntimeException(eu); + } + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + protected void run(final String targetPath, final String apiURL) throws Exception { + final ObjectMapper mapper = new ObjectMapper(); + final URL url = new URL(apiURL); + URLConnection conn = url.openConnection(); + InputStream is = conn.getInputStream(); + final String json = IOUtils.toString(is); + JsonNode jsonNode = mapper.readTree(json); + jsonNode + .get("files") + .forEach(i -> downloadItem(i.get("name").asText(), i.get("download_url").asText(), targetPath)); + } +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/ExtractORCIDDump.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/ExtractORCIDDump.java new file mode 100644 index 000000000..4aefdb5e9 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/ExtractORCIDDump.java @@ -0,0 +1,71 @@ + +package eu.dnetlib.dhp.collection.orcid; + +import static eu.dnetlib.dhp.utils.DHPUtils.getHadoopConfiguration; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; + +public class ExtractORCIDDump { + private static final Logger log = LoggerFactory.getLogger(ExtractORCIDDump.class); + + private final FileSystem fileSystem; + + public ExtractORCIDDump(FileSystem fileSystem) { + this.fileSystem = fileSystem; + } + + public static void main(String[] args) throws Exception { + final ArgumentApplicationParser argumentParser = new ArgumentApplicationParser( + IOUtils + .toString( + Objects + .requireNonNull( + DownloadORCIDDumpApplication.class + .getResourceAsStream( + "/eu/dnetlib/dhp/collection/orcid/extract_orcid_parameter.json")))); + argumentParser.parseArgument(args); + + final String hdfsuri = argumentParser.get("namenode"); + log.info("hdfsURI is {}", hdfsuri); + + final String sourcePath = argumentParser.get("sourcePath"); + log.info("sourcePath is {}", sourcePath); + + final String targetPath = argumentParser.get("targetPath"); + log.info("targetPath is {}", targetPath); + + final FileSystem fileSystem = FileSystem.get(getHadoopConfiguration(hdfsuri)); + + new ExtractORCIDDump(fileSystem).run(sourcePath, targetPath); + + } + + public void run(final String sourcePath, final String targetPath) throws IOException, InterruptedException { + RemoteIterator ls = fileSystem.listFiles(new Path(sourcePath), false); + final List workers = new ArrayList<>(); + int i = 0; + while (ls.hasNext()) { + LocatedFileStatus current = ls.next(); + if (current.getPath().getName().endsWith("tar.gz")) { + workers.add(new ORCIDExtractor(fileSystem, "" + i++, current.getPath(), targetPath)); + } + } + workers.forEach(Thread::start); + for (ORCIDExtractor worker : workers) { + worker.join(); + } + } +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/ORCIDExtractor.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/ORCIDExtractor.java new file mode 100644 index 000000000..11f4c55d8 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/ORCIDExtractor.java @@ -0,0 +1,171 @@ + +package eu.dnetlib.dhp.collection.orcid; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.compress.CompressionCodec; +import org.apache.hadoop.io.compress.CompressionCodecFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/**\ + * The ORCIDExtractor class extracts ORCID data from a TAR archive. + * The class creates a map of SequenceFile.Writer objects, one for each type of data that is to be extracted (e.g., employments, works, summaries). + * Then, it iterates over the TAR archive and writes each entry to the appropriate SequenceFile.Writer object. + * Finally, it closes all the SequenceFile.Writer objects. + */ +public class ORCIDExtractor extends Thread { + + private static final Logger log = LoggerFactory.getLogger(ORCIDExtractor.class); + + private final FileSystem fileSystem; + + private final String id; + + private final Path sourcePath; + + private final String baseOutputPath; + + public ORCIDExtractor(FileSystem fileSystem, String id, Path sourcePath, String baseOutputPath) { + this.fileSystem = fileSystem; + this.id = id; + this.sourcePath = sourcePath; + this.baseOutputPath = baseOutputPath; + } + + /** + * creates a map of SequenceFile.Writer objects, + * one for each type of data that is to be extracted. The map is created based on the filename in the TAR archive. + * For example, if the filename is employments.json, the map will contain an entry for the SequenceFile.Writer + * object that writes employment data. + * @return the Map + */ + private Map createMap() { + try { + log.info("Thread {} Creating sequence files starting from this input Path {}", id, sourcePath.getName()); + Map res = new HashMap<>(); + if (sourcePath.getName().contains("summaries")) { + + final String summaryPath = String.format("%s/summaries_%s", baseOutputPath, id); + final SequenceFile.Writer summary_file = SequenceFile + .createWriter( + fileSystem.getConf(), + SequenceFile.Writer.file(new Path(summaryPath)), + SequenceFile.Writer.keyClass(Text.class), + SequenceFile.Writer.valueClass(Text.class)); + + log.info("Thread {} Creating only summary path here {}", id, summaryPath); + res.put("summary", summary_file); + return res; + } else { + String employmentsPath = String.format("%s/employments_%s", baseOutputPath, id); + final SequenceFile.Writer employments_file = SequenceFile + .createWriter( + fileSystem.getConf(), + SequenceFile.Writer.file(new Path(employmentsPath)), + SequenceFile.Writer.keyClass(Text.class), + SequenceFile.Writer.valueClass(Text.class)); + res.put("employments", employments_file); + log.info("Thread {} Creating employments path here {}", id, employmentsPath); + + final String worksPath = String.format("%s/works_%s", baseOutputPath, id); + final SequenceFile.Writer works_file = SequenceFile + .createWriter( + fileSystem.getConf(), + SequenceFile.Writer.file(new Path(worksPath)), + SequenceFile.Writer.keyClass(Text.class), + SequenceFile.Writer.valueClass(Text.class)); + res.put("works", works_file); + log.info("Thread {} Creating works path here {}", id, worksPath); + + return res; + } + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public void run() { + + CompressionCodecFactory factory = new CompressionCodecFactory(fileSystem.getConf()); + CompressionCodec codec = factory.getCodec(sourcePath); + if (codec == null) { + System.err.println("No codec found for " + sourcePath.getName()); + System.exit(1); + } + + InputStream gzipInputStream = null; + try { + gzipInputStream = codec.createInputStream(fileSystem.open(sourcePath)); + final Map fileMap = createMap(); + iterateTar(fileMap, gzipInputStream); + + } catch (IOException e) { + throw new RuntimeException(e); + } finally { + log.info("Closing gzip stream"); + IOUtils.closeStream(gzipInputStream); + } + + } + + private SequenceFile.Writer retrieveFile(Map fileMap, final String path) { + if (sourcePath.getName().contains("summaries")) { + return fileMap.get("summary"); + } + + if (path.contains("works")) { + return fileMap.get("works"); + } + if (path.contains("employments")) + return fileMap.get("employments"); + return null; + } + + private void iterateTar(Map fileMap, InputStream gzipInputStream) throws IOException { + + int extractedItem = 0; + try (final TarArchiveInputStream tais = new TarArchiveInputStream(gzipInputStream)) { + + TarArchiveEntry entry; + while ((entry = tais.getNextTarEntry()) != null) { + + if (entry.isFile()) { + + final SequenceFile.Writer fl = retrieveFile(fileMap, entry.getName()); + if (fl != null) { + final Text key = new Text(entry.getName()); + final Text value = new Text( + org.apache.commons.io.IOUtils.toString(new BufferedReader(new InputStreamReader(tais)))); + fl.append(key, value); + extractedItem++; + if (extractedItem % 100000 == 0) { + log.info("Thread {}: Extracted {} items", id, extractedItem); + break; + } + } + } + } + } finally { + for (SequenceFile.Writer k : fileMap.values()) { + log.info("Thread {}: Completed processed {} items", id, extractedItem); + k.hflush(); + k.close(); + } + } + + } +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/OrcidParser.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/OrcidParser.java new file mode 100644 index 000000000..159b8a5fc --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/OrcidParser.java @@ -0,0 +1,251 @@ + +package eu.dnetlib.dhp.collection.orcid; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.ximpleware.*; + +import eu.dnetlib.dhp.collection.orcid.model.*; +import eu.dnetlib.dhp.parser.utility.VtdException; +import eu.dnetlib.dhp.parser.utility.VtdUtilityParser; + +public class OrcidParser { + + final Logger log = LoggerFactory.getLogger(OrcidParser.class); + private VTDNav vn; + + private AutoPilot ap; + private static final String NS_COMMON_URL = "http://www.orcid.org/ns/common"; + private static final String NS_COMMON = "common"; + private static final String NS_PERSON_URL = "http://www.orcid.org/ns/person"; + private static final String NS_PERSON = "person"; + private static final String NS_DETAILS_URL = "http://www.orcid.org/ns/personal-details"; + private static final String NS_DETAILS = "personal-details"; + private static final String NS_OTHER_URL = "http://www.orcid.org/ns/other-name"; + private static final String NS_OTHER = "other-name"; + private static final String NS_RECORD_URL = "http://www.orcid.org/ns/record"; + private static final String NS_RECORD = "record"; + private static final String NS_ERROR_URL = "http://www.orcid.org/ns/error"; + private static final String NS_ACTIVITIES = "activities"; + private static final String NS_ACTIVITIES_URL = "http://www.orcid.org/ns/activities"; + private static final String NS_WORK = "work"; + private static final String NS_WORK_URL = "http://www.orcid.org/ns/work"; + + private static final String NS_ERROR = "error"; + private static final String NS_HISTORY = "history"; + private static final String NS_HISTORY_URL = "http://www.orcid.org/ns/history"; + private static final String NS_BULK_URL = "http://www.orcid.org/ns/bulk"; + private static final String NS_BULK = "bulk"; + private static final String NS_EXTERNAL = "external-identifier"; + private static final String NS_EXTERNAL_URL = "http://www.orcid.org/ns/external-identifier"; + + private void generateParsedDocument(final String xml) throws ParseException { + final VTDGen vg = new VTDGen(); + vg.setDoc(xml.getBytes()); + vg.parse(true); + this.vn = vg.getNav(); + this.ap = new AutoPilot(vn); + ap.declareXPathNameSpace(NS_COMMON, NS_COMMON_URL); + ap.declareXPathNameSpace(NS_PERSON, NS_PERSON_URL); + ap.declareXPathNameSpace(NS_DETAILS, NS_DETAILS_URL); + ap.declareXPathNameSpace(NS_OTHER, NS_OTHER_URL); + ap.declareXPathNameSpace(NS_RECORD, NS_RECORD_URL); + ap.declareXPathNameSpace(NS_ERROR, NS_ERROR_URL); + ap.declareXPathNameSpace(NS_HISTORY, NS_HISTORY_URL); + ap.declareXPathNameSpace(NS_WORK, NS_WORK_URL); + ap.declareXPathNameSpace(NS_EXTERNAL, NS_EXTERNAL_URL); + ap.declareXPathNameSpace(NS_ACTIVITIES, NS_ACTIVITIES_URL); + } + + public Author parseSummary(final String xml) { + + try { + final Author author = new Author(); + generateParsedDocument(xml); + List recordNodes = VtdUtilityParser + .getTextValuesWithAttributes( + ap, vn, "//record:record", Arrays.asList("path")); + if (!recordNodes.isEmpty()) { + final String oid = (recordNodes.get(0).getAttributes().get("path")).substring(1); + author.setOrcid(oid); + } else { + return null; + } + List personNodes = VtdUtilityParser + .getTextValuesWithAttributes( + ap, vn, "//person:name", Arrays.asList("visibility")); + final String visibility = (personNodes.get(0).getAttributes().get("visibility")); + author.setVisibility(visibility); + final String name = VtdUtilityParser.getSingleValue(ap, vn, "//personal-details:given-names"); + author.setGivenName(name); + + final String surnames = VtdUtilityParser.getSingleValue(ap, vn, "//personal-details:family-name"); + author.setFamilyName(surnames); + + final String creditNames = VtdUtilityParser.getSingleValue(ap, vn, "//personal-details:credit-name"); + author.setCreditName(creditNames); + + final String biography = VtdUtilityParser + .getSingleValue(ap, vn, "//person:biography/personal-details:content"); + author.setBiography(biography); + + final List otherNames = VtdUtilityParser.getTextValue(ap, vn, "//other-name:content"); + if (!otherNames.isEmpty()) { + author.setOtherNames(otherNames); + } + + ap.selectXPath("//external-identifier:external-identifier"); + + while (ap.evalXPath() != -1) { + final Pid pid = new Pid(); + + final AutoPilot ap1 = new AutoPilot(ap.getNav()); + + ap1.selectXPath("./common:external-id-type"); + while (ap1.evalXPath() != -1) { + int it = vn.getText(); + pid.setSchema(vn.toNormalizedString(it)); + } + ap1.selectXPath("./common:external-id-value"); + while (ap1.evalXPath() != -1) { + int it = vn.getText(); + pid.setValue(vn.toNormalizedString(it)); + } + + author.addOtherPid(pid); + } + + return author; + } catch (Throwable e) { + log.error("Error on parsing {}", xml); + log.error(e.getMessage()); + return null; + } + } + + public Work parseWork(final String xml) { + + try { + final Work work = new Work(); + generateParsedDocument(xml); + List workNodes = VtdUtilityParser + .getTextValuesWithAttributes(ap, vn, "//work:work", Arrays.asList("path", "visibility")); + if (!workNodes.isEmpty()) { + final String oid = (workNodes.get(0).getAttributes().get("path")).split("/")[1]; + work.setOrcid(oid); + } else { + return null; + } + + ap.selectXPath("//common:external-id"); + + while (ap.evalXPath() != -1) { + final Pid pid = new Pid(); + + final AutoPilot ap1 = new AutoPilot(ap.getNav()); + + ap1.selectXPath("./common:external-id-type"); + while (ap1.evalXPath() != -1) { + int it = vn.getText(); + pid.setSchema(vn.toNormalizedString(it)); + } + ap1.selectXPath("./common:external-id-value"); + while (ap1.evalXPath() != -1) { + int it = vn.getText(); + pid.setValue(vn.toNormalizedString(it)); + } + + work.addPid(pid); + } + + work.setTitle(VtdUtilityParser.getSingleValue(ap, vn, "//work:title/common:title")); + + return work; + } catch (Throwable e) { + log.error("Error on parsing {}", xml); + log.error(e.getMessage()); + return null; + } + + } + + private String extractEmploymentDate(final String xpath) throws Exception { + + ap.selectXPath(xpath); + StringBuilder sb = new StringBuilder(); + while (ap.evalXPath() != -1) { + final AutoPilot ap1 = new AutoPilot(ap.getNav()); + ap1.selectXPath("./common:year"); + while (ap1.evalXPath() != -1) { + int it = vn.getText(); + sb.append(vn.toNormalizedString(it)); + } + ap1.selectXPath("./common:month"); + while (ap1.evalXPath() != -1) { + int it = vn.getText(); + sb.append("-"); + sb.append(vn.toNormalizedString(it)); + } + ap1.selectXPath("./common:day"); + while (ap1.evalXPath() != -1) { + int it = vn.getText(); + sb.append("-"); + sb.append(vn.toNormalizedString(it)); + } + } + return sb.toString(); + + } + + public Employment parseEmployment(final String xml) { + try { + final Employment employment = new Employment(); + generateParsedDocument(xml); + final String oid = VtdUtilityParser + .getSingleValue(ap, vn, "//common:source-orcid/common:path"); + if (StringUtils.isNotBlank(oid)) { + employment.setOrcid(oid); + } else { + return null; + } + final String depName = VtdUtilityParser + .getSingleValue(ap, vn, "//common:department-name"); + final String rolTitle = VtdUtilityParser + .getSingleValue(ap, vn, "//common:role-title"); + if (StringUtils.isNotBlank(rolTitle)) + employment.setRoleTitle(rolTitle); + if (StringUtils.isNotBlank(depName)) + employment.setDepartmentName(depName); + else + employment + .setDepartmentName( + VtdUtilityParser + .getSingleValue(ap, vn, "//common:organization/common:name")); + + employment.setStartDate(extractEmploymentDate("//common:start-date")); + employment.setEndDate(extractEmploymentDate("//common:end-date")); + + final String affiliationId = VtdUtilityParser + .getSingleValue(ap, vn, "//common:disambiguated-organization-identifier"); + final String affiliationIdType = VtdUtilityParser + .getSingleValue(ap, vn, "//common:disambiguation-source"); + + if (StringUtils.isNotBlank(affiliationId) || StringUtils.isNotBlank(affiliationIdType)) + employment.setAffiliationId(new Pid(affiliationId, affiliationIdType)); + + return employment; + } catch (Throwable e) { + log.error("Error on parsing {}", xml); + log.error(e.getMessage()); + return null; + } + + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/model/Author.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/model/Author.java new file mode 100644 index 000000000..32c321b41 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/model/Author.java @@ -0,0 +1,83 @@ + +package eu.dnetlib.dhp.collection.orcid.model; + +import java.util.ArrayList; +import java.util.List; + +public class Author extends ORCIDItem { + private String givenName; + private String familyName; + + private String visibility; + + private String creditName; + + private List otherNames; + + private List otherPids; + + private String biography; + + public String getBiography() { + return biography; + } + + public void setBiography(String biography) { + this.biography = biography; + } + + public String getGivenName() { + return givenName; + } + + public void setGivenName(String givenName) { + this.givenName = givenName; + } + + public String getFamilyName() { + return familyName; + } + + public void setFamilyName(String familyName) { + this.familyName = familyName; + } + + public String getCreditName() { + return creditName; + } + + public void setCreditName(String creditName) { + this.creditName = creditName; + } + + public List getOtherNames() { + return otherNames; + } + + public void setOtherNames(List otherNames) { + this.otherNames = otherNames; + } + + public String getVisibility() { + return visibility; + } + + public void setVisibility(String visibility) { + this.visibility = visibility; + } + + public List getOtherPids() { + return otherPids; + } + + public void setOtherPids(List otherPids) { + this.otherPids = otherPids; + } + + public void addOtherPid(final Pid pid) { + + if (otherPids == null) + otherPids = new ArrayList<>(); + otherPids.add(pid); + } +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/model/Employment.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/model/Employment.java new file mode 100644 index 000000000..baee67d46 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/model/Employment.java @@ -0,0 +1,54 @@ + +package eu.dnetlib.dhp.collection.orcid.model; + +public class Employment extends ORCIDItem { + + private String startDate; + private String EndDate; + + private Pid affiliationId; + + private String departmentName; + + private String roleTitle; + + public String getStartDate() { + return startDate; + } + + public void setStartDate(String startDate) { + this.startDate = startDate; + } + + public String getEndDate() { + return EndDate; + } + + public void setEndDate(String endDate) { + EndDate = endDate; + } + + public Pid getAffiliationId() { + return affiliationId; + } + + public void setAffiliationId(Pid affiliationId) { + this.affiliationId = affiliationId; + } + + public String getDepartmentName() { + return departmentName; + } + + public void setDepartmentName(String departmentName) { + this.departmentName = departmentName; + } + + public String getRoleTitle() { + return roleTitle; + } + + public void setRoleTitle(String roleTitle) { + this.roleTitle = roleTitle; + } +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/model/ORCIDItem.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/model/ORCIDItem.java new file mode 100644 index 000000000..6bc47bc26 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/model/ORCIDItem.java @@ -0,0 +1,14 @@ + +package eu.dnetlib.dhp.collection.orcid.model; + +public class ORCIDItem { + private String orcid; + + public String getOrcid() { + return orcid; + } + + public void setOrcid(String orcid) { + this.orcid = orcid; + } +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/model/Pid.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/model/Pid.java new file mode 100644 index 000000000..077dc2550 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/model/Pid.java @@ -0,0 +1,33 @@ + +package eu.dnetlib.dhp.collection.orcid.model; + +public class Pid { + + private String value; + + private String schema; + + public Pid() { + } + + public Pid(String value, String schema) { + this.value = value; + this.schema = schema; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public String getSchema() { + return schema; + } + + public void setSchema(String schema) { + this.schema = schema; + } +} diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/model/Work.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/model/Work.java new file mode 100644 index 000000000..670170323 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/collection/orcid/model/Work.java @@ -0,0 +1,35 @@ + +package eu.dnetlib.dhp.collection.orcid.model; + +import java.util.ArrayList; +import java.util.List; + +public class Work extends ORCIDItem { + + private String title; + + private List pids; + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public List getPids() { + return pids; + } + + public void setPids(List pids) { + this.pids = pids; + } + + public void addPid(Pid pid) { + if (pids == null) + pids = new ArrayList<>(); + pids.add(pid); + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/download_orcid_parameter.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/download_orcid_parameter.json new file mode 100644 index 000000000..4a84cbbb9 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/download_orcid_parameter.json @@ -0,0 +1,21 @@ +[ + { + "paramName": "n", + "paramLongName": "namenode", + "paramDescription": "the Name Node URI", + "paramRequired": true + }, + { + "paramName": "t", + "paramLongName": "targetPath", + "paramDescription": "the target PATH where download the files", + "paramRequired": true + }, + { + "paramName": "a", + "paramLongName": "apiURL", + "paramDescription": "the FIGSHARE API id URL to retrieve all the dump files", + "paramRequired": true + } + +] \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/extract_orcid_parameter.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/extract_orcid_parameter.json new file mode 100644 index 000000000..4af371875 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/extract_orcid_parameter.json @@ -0,0 +1,21 @@ +[ + { + "paramName": "n", + "paramLongName": "namenode", + "paramDescription": "the Name Node URI", + "paramRequired": true + }, + { + "paramName": "t", + "paramLongName": "targetPath", + "paramDescription": "the target PATH to extract files", + "paramRequired": true + }, + { + "paramName": "s", + "paramLongName": "sourcePath", + "paramDescription": "the PATH where the tar.gz files were downloaded", + "paramRequired": true + } + +] \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/generate_orcid_table_parameter.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/generate_orcid_table_parameter.json new file mode 100644 index 000000000..01d81ea97 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/generate_orcid_table_parameter.json @@ -0,0 +1,21 @@ +[ + { + "paramName": "m", + "paramLongName": "master", + "paramDescription": "the master name", + "paramRequired": true + }, + { + "paramName": "t", + "paramLongName": "targetPath", + "paramDescription": "the target PATH of the DF tables", + "paramRequired": true + }, + { + "paramName": "s", + "paramLongName": "sourcePath", + "paramDescription": "the PATH of the ORCID sequence file", + "paramRequired": true + } + +] \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/oozie_app/config-default.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/oozie_app/config-default.xml new file mode 100644 index 000000000..dd3c32c62 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/oozie_app/config-default.xml @@ -0,0 +1,23 @@ + + + jobTracker + yarnRM + + + nameNode + hdfs://nameservice1 + + + oozie.use.system.libpath + true + + + oozie.action.sharelib.for.spark + spark2 + + + + oozie.launcher.mapreduce.user.classpath.first + true + + \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/oozie_app/workflow.xml new file mode 100644 index 000000000..1a5f425e5 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/oozie_app/workflow.xml @@ -0,0 +1,81 @@ + + + + targetPath + the path to store the original ORCID dump + + + apiURL + The figshare API URL to retrieve the list file to download + + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + + + oozie.launcher.mapreduce.user.classpath.first + true + + + eu.dnetlib.dhp.collection.orcid.DownloadORCIDDumpApplication + --namenode${nameNode} + --targetPath${targetPath} + --apiURL${apiURL} + + + + + + + + + + oozie.launcher.mapreduce.user.classpath.first + true + + + + eu.dnetlib.dhp.collection.orcid.ExtractORCIDDump + -Xmx6g + --namenode${nameNode} + --sourcePath${targetPath} + --targetPath${targetPath}/extracted + + + + + + + + yarn + cluster + Generate ORCID Tables + eu.dnetlib.dhp.collection.orcid.SparkGenerateORCIDTable + dhp-aggregation-${projectVersion}.jar + + --executor-memory=${sparkExecutorMemory} + --executor-cores=${sparkExecutorCores} + --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=2g + --conf spark.sql.shuffle.partitions=3000 + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + + --sourcePath${targetPath}/extracted + --targetPath${targetPath}/tables + --masteryarn + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/preprocess_orcid_dump_parameter.json b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/preprocess_orcid_dump_parameter.json new file mode 100644 index 000000000..4a84cbbb9 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/collection/orcid/preprocess_orcid_dump_parameter.json @@ -0,0 +1,21 @@ +[ + { + "paramName": "n", + "paramLongName": "namenode", + "paramDescription": "the Name Node URI", + "paramRequired": true + }, + { + "paramName": "t", + "paramLongName": "targetPath", + "paramDescription": "the target PATH where download the files", + "paramRequired": true + }, + { + "paramName": "a", + "paramLongName": "apiURL", + "paramDescription": "the FIGSHARE API id URL to retrieve all the dump files", + "paramRequired": true + } + +] \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/orcid/SparkGenerateORCIDTable.scala b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/orcid/SparkGenerateORCIDTable.scala new file mode 100644 index 000000000..f0c4cd214 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/collection/orcid/SparkGenerateORCIDTable.scala @@ -0,0 +1,101 @@ +package eu.dnetlib.dhp.collection.orcid + +import eu.dnetlib.dhp.application.AbstractScalaApplication +import eu.dnetlib.dhp.collection.orcid.model.{Author, Employment, Pid, Work} +import org.apache.hadoop.io.Text +import org.apache.spark.SparkContext +import org.apache.spark.sql.{Encoder, Encoders, SaveMode, SparkSession} +import org.slf4j.{Logger, LoggerFactory} + +class SparkGenerateORCIDTable(propertyPath: String, args: Array[String], log: Logger) + extends AbstractScalaApplication(propertyPath, args, log: Logger) { + + /** Here all the spark applications runs this method + * where the whole logic of the spark node is defined + */ + override def run(): Unit = { + val sourcePath: String = parser.get("sourcePath") + log.info("found parameters sourcePath: {}", sourcePath) + val targetPath: String = parser.get("targetPath") + log.info("found parameters targetPath: {}", targetPath) + extractORCIDTable(spark, sourcePath, targetPath) + extractORCIDEmploymentsTable(spark, sourcePath, targetPath) + extractORCIDWorksTable(spark, sourcePath, targetPath) + } + + def extractORCIDTable(spark: SparkSession, sourcePath: String, targetPath: String): Unit = { + val sc: SparkContext = spark.sparkContext + import spark.implicits._ + val df = sc + .sequenceFile(sourcePath, classOf[Text], classOf[Text]) + .map { case (x, y) => (x.toString, y.toString) } + .toDF + .as[(String, String)] + implicit val orcidAuthor: Encoder[Author] = Encoders.bean(classOf[Author]) +// implicit val orcidPID:Encoder[Pid] = Encoders.bean(classOf[Pid]) + df.filter(r => r._1.contains("summaries")) + .map { r => + val p = new OrcidParser + p.parseSummary(r._2) + } + .filter(p => p != null) + .write + .mode(SaveMode.Overwrite) + .save(s"$targetPath/Authors") + } + + def extractORCIDWorksTable(spark: SparkSession, sourcePath: String, targetPath: String): Unit = { + val sc: SparkContext = spark.sparkContext + import spark.implicits._ + val df = sc + .sequenceFile(sourcePath, classOf[Text], classOf[Text]) + .map { case (x, y) => (x.toString, y.toString) } + .toDF + .as[(String, String)] + implicit val orcidWorkAuthor: Encoder[Work] = Encoders.bean(classOf[Work]) + implicit val orcidPID: Encoder[Pid] = Encoders.bean(classOf[Pid]) + df.filter(r => r._1.contains("works")) + .map { r => + val p = new OrcidParser + p.parseWork(r._2) + } + .filter(p => p != null) + .write + .mode(SaveMode.Overwrite) + .save(s"$targetPath/Works") + } + + def extractORCIDEmploymentsTable(spark: SparkSession, sourcePath: String, targetPath: String): Unit = { + val sc: SparkContext = spark.sparkContext + import spark.implicits._ + val df = sc + .sequenceFile(sourcePath, classOf[Text], classOf[Text]) + .map { case (x, y) => (x.toString, y.toString) } + .toDF + .as[(String, String)] + implicit val orcidEmploymentAuthor: Encoder[Employment] = Encoders.bean(classOf[Employment]) + implicit val orcidPID: Encoder[Pid] = Encoders.bean(classOf[Pid]) + df.filter(r => r._1.contains("employments")) + .map { r => + val p = new OrcidParser + p.parseEmployment(r._2) + } + .filter(p => p != null) + .write + .mode(SaveMode.Overwrite) + .save(s"$targetPath/Employments") + } +} + +object SparkGenerateORCIDTable { + + val log: Logger = LoggerFactory.getLogger(SparkGenerateORCIDTable.getClass) + + def main(args: Array[String]): Unit = { + + new SparkGenerateORCIDTable("/eu/dnetlib/dhp/collection/orcid/generate_orcid_table_parameter.json", args, log) + .initialize() + .run() + + } +} diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/orcid/DownloadORCIDTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/orcid/DownloadORCIDTest.java new file mode 100644 index 000000000..be5555fc0 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/orcid/DownloadORCIDTest.java @@ -0,0 +1,158 @@ + +package eu.dnetlib.dhp.collection.orcid; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.io.Text; +import org.apache.spark.SparkContext; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.ximpleware.NavException; +import com.ximpleware.ParseException; +import com.ximpleware.XPathEvalException; +import com.ximpleware.XPathParseException; + +import eu.dnetlib.dhp.collection.orcid.model.Author; +import eu.dnetlib.dhp.collection.orcid.model.ORCIDItem; +import eu.dnetlib.dhp.parser.utility.VtdException; + +public class DownloadORCIDTest { + private final Logger log = LoggerFactory.getLogger(DownloadORCIDTest.class); + +// public void test() throws Exception { +// +// Configuration conf = new Configuration(); +// // Set FileSystem URI +//// conf.set("fs.defaultFS", "file://"); +// // Because of Maven +// conf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName()); +// conf.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName()); +// +// System.setProperty("hadoop.home.dir", "file:///Users/sandro/orcid/"); +// +// final FileSystem fileSystem = FileSystem.get(conf); +// +// new ExtractORCIDDump(fileSystem).run("/Users/sandro/orcid/", "/Users/sandro/orcid/extracted"); +// +//// final GZIPInputStream gzip = new GZIPInputStream(Files.newInputStream(Paths.get("/Users/sandro/orcid/ORCID_2023_10_activities_1.tar.gz"))); +//// try(final TarArchiveInputStream tais = new TarArchiveInputStream(gzip)) { +//// +//// TarArchiveEntry entry; +//// while ((entry = tais.getNextTarEntry()) != null) { +//// +//// if (entry.isFile() && entry.getName().contains("employments")) { +//// +//// System.out.println(entry.getName()); +//// final String [] items = entry.getName().split("/"); +//// +//// final String res = IOUtils.toString(new BufferedReader(new InputStreamReader(tais))); +//// System.out.println("res = " + res); +//// +//// System.out.println(items[items.length-2]); +//// break; +//// } +//// +//// +//// } +//// } +// +// } + + @Test + public void testSummary() throws Exception { + final String xml = IOUtils + .toString( + Objects.requireNonNull(getClass().getResourceAsStream("/eu/dnetlib/dhp/collection/orcid/summary.xml"))); + + final OrcidParser parser = new OrcidParser(); + ORCIDItem orcidItem = parser.parseSummary(xml); + + final ObjectMapper mapper = new ObjectMapper(); + System.out.println(mapper.writeValueAsString(orcidItem)); + + } + + @Test + public void testParsingWork() throws Exception { + + final List works_path = Arrays + .asList( + "/eu/dnetlib/dhp/collection/orcid/activity_work_0000-0002-2536-4498.xml", + "/eu/dnetlib/dhp/collection/orcid/activity_work_0000-0002-5982-8983.xml", + "/eu/dnetlib/dhp/collection/orcid/activity_work_0000-0003-2760-1191.xml", + "/eu/dnetlib/dhp/collection/orcid/activity_work_0000-0003-2760-1191-similarity.xml", + "/eu/dnetlib/dhp/collection/orcid/activity_work_0000-0003-2760-1191_contributors.xml" + + ); + + final OrcidParser parser = new OrcidParser(); + final ObjectMapper mapper = new ObjectMapper(); + works_path.stream().map(s -> { + try { + return IOUtils + .toString( + Objects + .requireNonNull( + getClass() + .getResourceAsStream( + s))); + } catch (IOException e) { + throw new RuntimeException(e); + } + }).forEach(s -> { + try { + System.out.println(mapper.writeValueAsString(parser.parseWork(s))); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + + @Test + public void testParsingEmployments() throws Exception { + + final List works_path = Arrays + .asList( + "/eu/dnetlib/dhp/collection/orcid/employment.xml", + "/eu/dnetlib/dhp/collection/orcid/employment_2.xml", + "/eu/dnetlib/dhp/collection/orcid/employment_3.xml" + + ); + + final OrcidParser parser = new OrcidParser(); + final ObjectMapper mapper = new ObjectMapper(); + works_path.stream().map(s -> { + try { + return IOUtils + .toString( + Objects + .requireNonNull( + getClass() + .getResourceAsStream( + s))); + } catch (IOException e) { + throw new RuntimeException(e); + } + }).forEach(s -> { + try { + System.out.println(mapper.writeValueAsString(parser.parseEmployment(s))); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/activity_work_0000-0002-2536-4498.xml b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/activity_work_0000-0002-2536-4498.xml new file mode 100644 index 000000000..2c89b83f6 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/activity_work_0000-0002-2536-4498.xml @@ -0,0 +1,69 @@ + + + 2016-09-01T19:22:46.768Z + 2022-05-25T03:48:56.968Z + + + https://orcid.org/client/0000-0002-5982-8983 + 0000-0002-5982-8983 + orcid.org + + Scopus - Elsevier + + https://orcid.org/0000-0001-5010-5001 + 0000-0001-5010-5001 + orcid.org + + Quang Nguyen + + + Vision outcomes and major complications after endovascular coil embolization of ophthalmic segment aneurysms + + American Journal of Neuroradiology + + bibtex + @article{Nguyen2014,title = {Vision outcomes and major complications after endovascular coil embolization of ophthalmic segment aneurysms},journal = {American Journal of Neuroradiology},year = {2014},volume = {35},number = {11},pages = {2140-2145},author = {Durst, C. and Starke, R.M. and Gaughen, J. and Nguyen, Q. and Patrie, J. and Jensen, M.E. and Evans, A.J.}} + + journal-article + + 2014 + + + + doi + 10.3174/ajnr.A4032 + 10.3174/ajnr.a4032 + self + + + eid + 2-s2.0-84911865199 + 2-s2.0-84911865199 + self + + + http://www.scopus.com/inward/record.url?eid=2-s2.0-84911865199&partnerID=MN8TOARS + + + Durst, C. + + + Starke, R.M. + + + Gaughen, J. + + + Nguyen, Q. + + + Patrie, J. + + + Jensen, M.E. + + + Evans, A.J. + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/activity_work_0000-0002-5982-8983.xml b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/activity_work_0000-0002-5982-8983.xml new file mode 100644 index 000000000..63b4405f1 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/activity_work_0000-0002-5982-8983.xml @@ -0,0 +1,79 @@ + + + 2018-11-01T19:49:45.562Z + 2018-11-01T19:49:45.562Z + + + https://orcid.org/client/0000-0002-5982-8983 + 0000-0002-5982-8983 + orcid.org + + Scopus - Elsevier + + + "Calling Out" in class: Degrees of candor in addressing social injustices in + racially homogenous and heterogeneous U.S. history classrooms + + Journal of Social Studies Research + + bibtex + @article{Massaro2018,title = {{"}Calling Out{"} in class: Degrees of + candor in addressing social injustices in racially homogenous and heterogeneous U.S. + history classrooms},journal = {Journal of Social Studies Research},year = {2018},author + = {Parkhouse, H. and Massaro, V.R.}} + + journal-article + + 2018 + + + + doi + 10.1016/j.jssr.2018.01.004 + 10.1016/j.jssr.2018.01.004 + self + + + eid + 2-s2.0-85041949043 + 2-s2.0-85041949043 + self + + + http://www.scopus.com/inward/record.url?eid=2-s2.0-85041949043&partnerID=MN8TOARS + + + Parkhouse, H. + + + Massaro, V.R. + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/activity_work_0000-0003-2760-1191-similarity.xml b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/activity_work_0000-0003-2760-1191-similarity.xml new file mode 100644 index 000000000..650d5a4cb --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/activity_work_0000-0003-2760-1191-similarity.xml @@ -0,0 +1,113 @@ + + + 2016-12-12T23:02:05.233Z + 2016-12-13T09:08:16.412Z + + + https://orcid.org/0000-0002-9157-3431 + 0000-0002-9157-3431 + orcid.org + + Europe PubMed Central + + + Cutoff Value of Admission N-Terminal Pro-Brain Natriuretic Peptide Which + Predicts Poor Myocardial Perfusion after Primary Percutaneous Coronary Intervention for + ST-Segment-Elevation Myocardial Infarction. + + + formatted-unspecified + Abdel-Dayem K, Eweda II, El-Sherbiny A, Dimitry MO, Nammas W, Acta + Cardiologica Sinica, 2016, vol. 32, no. 6, pp. 649-655, 2016 + + journal-article + + 2016 + 11 + + + + pmid + 27899851 + 27899851 + self + + + pmc + PMC5126442 + PMC5126442 + self + + + http://europepmc.org/abstract/med/27899851 + + + Abdel-Dayem K + + first + author + + + + Abdel-Dayem Fake + + first + author + + + + Eweda II + + first + author + + + + El-Sherbiny A + + first + author + + + + Dimitry MO + + first + author + + + + Nammas W + + first + author + + + + diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/activity_work_0000-0003-2760-1191.xml b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/activity_work_0000-0003-2760-1191.xml new file mode 100644 index 000000000..83752b145 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/activity_work_0000-0003-2760-1191.xml @@ -0,0 +1,106 @@ + + + 2016-12-12T23:02:05.233Z + 2016-12-13T09:08:16.412Z + + + https://orcid.org/0000-0002-9157-3431 + 0000-0002-9157-3431 + orcid.org + + Europe PubMed Central + + + Cutoff Value of Admission N-Terminal Pro-Brain Natriuretic Peptide Which + Predicts Poor Myocardial Perfusion after Primary Percutaneous Coronary Intervention for + ST-Segment-Elevation Myocardial Infarction. + + + formatted-unspecified + Abdel-Dayem K, Eweda II, El-Sherbiny A, Dimitry MO, Nammas W, Acta + Cardiologica Sinica, 2016, vol. 32, no. 6, pp. 649-655, 2016 + + journal-article + + 2016 + 11 + + + + pmid + 27899851 + 27899851 + self + + + pmc + PMC5126442 + PMC5126442 + self + + + http://europepmc.org/abstract/med/27899851 + + + Khair Abde Daye + + first + author + + + + Eweda II + + first + author + + + + El-Sherbiny A + + first + author + + + + Dimitry MO + + first + author + + + + Nammas W + + first + author + + + + diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/activity_work_0000-0003-2760-1191_contributors.xml b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/activity_work_0000-0003-2760-1191_contributors.xml new file mode 100644 index 000000000..26e64aeda --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/activity_work_0000-0003-2760-1191_contributors.xml @@ -0,0 +1,101 @@ + + + 2016-12-12T23:02:05.233Z + 2016-12-13T09:08:16.412Z + + + https://orcid.org/0000-0002-9157-3431 + 0000-0002-9157-3431 + orcid.org + + Europe PubMed Central + + + Cutoff Value of Admission N-Terminal Pro-Brain Natriuretic Peptide Which + Predicts Poor Myocardial Perfusion after Primary Percutaneous Coronary Intervention for + ST-Segment-Elevation Myocardial Infarction. + + + formatted-unspecified + Abdel-Dayem K, Eweda II, El-Sherbiny A, Dimitry MO, Nammas W, Acta + Cardiologica Sinica, 2016, vol. 32, no. 6, pp. 649-655, 2016 + + journal-article + + 2016 + 11 + + + + pmid + 27899851 + 27899851 + self + + + pmc + PMC5126442 + PMC5126442 + self + + + http://europepmc.org/abstract/med/27899851 + + + + seq0 + role0 + + + + creditname1 + + + creditname2 + + seq2 + + + + + creditname3 + + + role3 + + + + + + seq4 + role4 + + + + diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/employment.xml b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/employment.xml new file mode 100644 index 000000000..89c7f7020 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/employment.xml @@ -0,0 +1,50 @@ + + + 2016-09-01T19:21:05.791Z + 2016-09-01T19:21:05.791Z + + + https://orcid.org/0000-0001-5010-5001 + 0000-0001-5010-5001 + orcid.org + + Quang Nguyen + + + Beth Israel Deaconess Medical Center + + Boston + MA + US + + + 1859 + RINGGOLD + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/employment_2.xml b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/employment_2.xml new file mode 100644 index 000000000..c0e88e236 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/employment_2.xml @@ -0,0 +1,55 @@ + + + 2018-09-03T01:46:19.474Z + 2018-09-03T01:46:19.474Z + + + https://orcid.org/0000-0001-5011-3001 + 0000-0001-5011-3001 + orcid.org + + zhengyan li + + + 2008 + 09 + 01 + + + Anhui Academy of Agricultural Sciences + + Hefei + Anhui + CN + + + 125385 + RINGGOLD + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/employment_3.xml b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/employment_3.xml new file mode 100644 index 000000000..8e7857fb8 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/employment_3.xml @@ -0,0 +1,62 @@ + + + 2021-03-11T14:48:29.603Z + 2021-03-11T14:48:29.603Z + + + https://orcid.org/0000-0001-5012-1001 + 0000-0001-5012-1001 + orcid.org + + Asma Bazzi + + Pathology and Laboratory Medicine + Medical Laboratory Technologist + + 1994 + 10 + 01 + + + 2000 + 06 + 30 + + + American University of Beirut + + Hamra + Beirut + LB + + + 11238 + RINGGOLD + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/summary.xml b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/summary.xml new file mode 100644 index 000000000..1f5a1bff1 --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/collection/orcid/summary.xml @@ -0,0 +1,581 @@ + + + + https://orcid.org/0000-0001-5045-1000 + 0000-0001-5045-1000 + orcid.org + + + es + + + Direct + 2023-01-17T23:50:40.215Z + 2023-09-04T17:51:57.749Z + true + true + true + + + + 2023-01-17T23:50:40.472Z + 2023-01-17T23:50:40.472Z + Patricio + Sánchez Quinchuela + + + + 2023-01-19T13:47:33.653Z + 2023-01-19T13:47:33.653Z + Especialista de vinculación con la sociedad y docente de la Universidad de las Artes. Magister en Economía Social y Solidaria por el IAEN; Magister en Proyectos Sociales y Productivos por la UNACH. Licenciado en Artes UCE. Licenciado en Castellano y Literatura por la UNACH. Doctorando del programa de Sociología de la UNED España. Larga trayectoria vinculado a las organizaciones sociales acompañando procesos de gestión cultural, formación de liderazgos y economía solidaria. + + + + + + + 2018-02-05T23:27:36.636Z + + 2013-03-08T03:20:39.347Z + 2018-02-05T23:27:36.636Z + + + https://orcid.org/client/0000-0002-5982-8983 + 0000-0002-5982-8983 + orcid.org + + Scopus - Elsevier + + https://orcid.org/0000-0001-7291-3210 + 0000-0001-7291-3210 + orcid.org + + Paolo Manghi + + Scopus Author ID + 6602255248 + http://www.scopus.com/inward/authorDetails.url?authorID=6602255248&partnerID=MN8TOARS + self + + + + + 2023-09-04T17:51:57.749Z + + 2023-01-19T13:49:48.482Z + + 2023-01-19T13:49:48.482Z + + + 2023-01-19T13:49:48.482Z + 2023-01-19T13:49:48.482Z + + + https://orcid.org/0000-0001-5045-1000 + 0000-0001-5045-1000 + orcid.org + + Patricio Sánchez Quinchuela + + Programa de Maestría + Becario del programa de Maestría en Economía Social y Solidaria + + 2014 + 10 + 20 + + + Instituto de Altos Estudios Nacionales + + Quito + Pichincha + EC + + + https://ror.org/011g3me54 + ROR + + + + + + + 2023-01-18T21:41:03.175Z + + 2023-01-18T21:41:03.175Z + + + 2023-01-18T21:41:03.175Z + 2023-01-18T21:41:03.175Z + + + https://orcid.org/0000-0001-5045-1000 + 0000-0001-5045-1000 + orcid.org + + Patricio Sánchez Quinchuela + + Programa de Doctorado en Sociología + Doctorando del Programa de Sociología + + 2020 + 11 + 06 + + + Universidad Nacional de Educación a Distancia Facultad de Ciencias Políticas y Sociología + + Madrid + Comunidad de Madrid + ES + + + 223339 + RINGGOLD + + + + + + + 2023-01-18T21:25:07.138Z + + 2023-01-18T21:22:21.513Z + + + 2023-01-17T23:57:08.246Z + 2023-01-18T21:22:21.513Z + + + https://orcid.org/0000-0001-5045-1000 + 0000-0001-5045-1000 + orcid.org + + Patricio Sánchez Quinchuela + + Dirección de Vinculación con la Sociedad + Especialista de Proyectos y docente + + 2021 + 11 + 01 + + + Universidad de las Artes + + Guayaquil + Guayas + EC + + + https://ror.org/016drwn73 + ROR + + + + + + 2023-01-18T21:25:07.138Z + + + 2023-01-18T21:25:07.138Z + 2023-01-18T21:25:07.138Z + + + https://orcid.org/0000-0001-5045-1000 + 0000-0001-5045-1000 + orcid.org + + Patricio Sánchez Quinchuela + + Dirección de Vinculación con la Sociedad + Director + + 2019 + 11 + 05 + + + 2021 + 10 + 31 + + + Universidad Regional Amazónica IKIAM + + Tena + Napo + EC + + + https://ror.org/05xedqd83 + ROR + + + http://ikiam.edu.ec + + + + + + + 2023-03-24T18:16:09.131Z + + 2023-03-24T18:16:09.131Z + + + 2023-03-24T18:16:09.131Z + 2023-03-24T18:16:09.131Z + + + https://orcid.org/0000-0001-5045-1000 + 0000-0001-5045-1000 + orcid.org + + Patricio Sánchez Quinchuela + + Artes Escénicas + Miembro + + 2000 + 07 + 15 + + + Casa de la Cultura Ecuatoriana + + Riobamba + Sierra Centro + EC + + + + + + + + 2023-01-18T21:45:07.379Z + + 2023-01-18T21:29:11.300Z + + + 2023-01-18T21:29:11.300Z + 2023-01-18T21:29:11.300Z + + + https://orcid.org/0000-0001-5045-1000 + 0000-0001-5045-1000 + orcid.org + + Patricio Sánchez Quinchuela + + Programa de Gobernabilidad + Magister en Economïa Social y Solidaria + + 2014 + 10 + 20 + + + 2017 + 01 + 26 + + + Instituto de Altos Estudios Nacionales + + Quito + Pichincha + EC + + + https://ror.org/011g3me54 + ROR + + + + + + 2023-01-18T21:34:32.093Z + + + 2023-01-18T21:34:32.093Z + 2023-01-18T21:34:32.093Z + + + https://orcid.org/0000-0001-5045-1000 + 0000-0001-5045-1000 + orcid.org + + Patricio Sánchez Quinchuela + + Posgrados + Magister en Proyectos Sociales y Productivos + + 2001 + 03 + 09 + + + 2003 + 02 + 27 + + + Universidad Nacional de Chimborazo + + Riobamba + Chimborazo + EC + + + https://ror.org/059wmd288 + ROR + + + + + + 2023-01-18T21:45:07.379Z + + + 2023-01-18T21:45:07.379Z + 2023-01-18T21:45:07.379Z + + + https://orcid.org/0000-0001-5045-1000 + 0000-0001-5045-1000 + orcid.org + + Patricio Sánchez Quinchuela + + Ciencias de la Educación + Licenciado en Ciencias de la Educación en Castellano y Literatura + + 1994 + 10 + 03 + + + 2000 + 01 + 31 + + + Universidad Nacional de Chimborazo + + Riobamba + Chimborazo + EC + + + https://ror.org/059wmd288 + ROR + + + + + + 2023-01-18T21:37:42.186Z + + + 2023-01-18T21:37:42.186Z + 2023-01-18T21:37:42.186Z + + + https://orcid.org/0000-0001-5045-1000 + 0000-0001-5045-1000 + orcid.org + + Patricio Sánchez Quinchuela + + Facultad de Artes + Licenciado en Artes + + 1989 + 09 + 05 + + + 1997 + 08 + 07 + + + Universidad Central del Ecuador + + Quito + Pichincha + EC + + + http://dx.doi.org/10.13039/100019134 + FUNDREF + + + + + + + + + 2023-09-04T17:51:57.749Z + + 2023-06-09T22:15:12.910Z + + + 2023-03-24T18:36:56.180Z + 2023-06-09T22:15:12.910Z + + + https://orcid.org/0000-0001-5045-1000 + 0000-0001-5045-1000 + orcid.org + + Patricio Sánchez Quinchuela + + + Experience in a non-capitalist way: solidarity funds that do not tax interest on the use of money + + + + isbn + 978-9942-29-089-2 + 9789942290892 + part-of + + + book-chapter + + 2023 + 06 + 07 + + Finanzas éticas y solidarias en América Latina: diagnósticos, debates y propuestas + + + + 2023-03-24T19:05:36.384Z + + + 2023-03-24T19:05:36.384Z + 2023-03-24T19:05:36.384Z + + + https://orcid.org/0000-0001-5045-1000 + 0000-0001-5045-1000 + orcid.org + + Patricio Sánchez Quinchuela + + + Incidence of artistic practices in the social transformation of the territory. study of case: Hilarte Association, Guayaquil-Ecuador + + + conference-abstract + + 2022 + 10 + 06 + + + + + 2023-09-04T17:40:30.215Z + + + other-id + 2018 + 2018 + self + + + + 2023-09-04T17:40:30.215Z + 2023-09-04T17:40:30.215Z + + + https://orcid.org/0000-0001-5045-1000 + 0000-0001-5045-1000 + orcid.org + + Patricio Sánchez Quinchuela + + + Más allá de la transferencia de conocimientos, un espacio para el interaprendizaje y el diálogo de saberes + + + + other-id + 2018 + 2018 + self + + + https://drive.google.com/drive/folders/1Tclz6isxGzSjTq-hfTnxe6M1nux-88wF?usp=drive_link + conference-poster + + 2018 + 11 + 30 + + Más allá de la transferencia de conocimientos, un espacio para el interaprendizaje y el diálogo de saberes + + + + 2023-03-24T18:57:10.095Z + + + 2023-03-24T18:57:10.095Z + 2023-03-24T18:57:10.095Z + + + https://orcid.org/0000-0001-5045-1000 + 0000-0001-5045-1000 + orcid.org + + Patricio Sánchez Quinchuela + + + Promotion of the popular and solidarity economy from the state: principles and challenges in the experience of Ecuador + + + dissertation-thesis + + 2017 + 01 + 26 + + + + + 2023-09-04T17:51:57.749Z + + + 2023-09-04T17:51:57.749Z + 2023-09-04T17:51:57.749Z + + + https://orcid.org/0000-0001-5045-1000 + 0000-0001-5045-1000 + orcid.org + + Patricio Sánchez Quinchuela + + + La Rebelión de los Dioses + + + https://drive.google.com/drive/folders/1Tclz6isxGzSjTq-hfTnxe6M1nux-88wF?usp=drive_link + registered-copyright + + 2001 + 08 + 28 + + Editorial pedagógica freire + + + + + \ No newline at end of file From eaf0a702dea01c77a01bfa38acb4c9b1b6c72627 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Tue, 14 Nov 2023 14:53:34 +0100 Subject: [PATCH 101/148] - --- .../eu/dnetlib/dhp/api/QueryCommunityAPI.java | 50 +++++++++---------- .../main/java/eu/dnetlib/dhp/api/Utils.java | 25 +++++----- .../dnetlib/dhp/bulktag/SparkBulkTagJob.java | 6 +-- .../PrepareResultCommunitySet.java | 6 +-- .../PrepareResultCommunitySet.java | 7 ++- 5 files changed, 47 insertions(+), 47 deletions(-) diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java index 262ca0290..dc10fce15 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java @@ -14,8 +14,8 @@ import org.jetbrains.annotations.NotNull; * @Date 06/10/23 */ public class QueryCommunityAPI { - private static final String PRODUCTION_BASE_URL = "https://services.openaire.eu/openaire/"; - private static final String BETA_BASE_URL = "https://beta.services.openaire.eu/openaire/"; + + private static String get(String geturl) throws IOException { URL url = new URL(geturl); @@ -32,35 +32,35 @@ public class QueryCommunityAPI { return body; } - public static String communities(boolean production) throws IOException { - if (production) - return get(PRODUCTION_BASE_URL + "community/communities"); - return get(BETA_BASE_URL + "community/communities"); - } + public static String communities(String baseURL) throws IOException { - public static String community(String id, boolean production) throws IOException { - if (production) - return get(PRODUCTION_BASE_URL + "community/" + id); - return get(BETA_BASE_URL + "community/" + id); - } - - public static String communityDatasource(String id, boolean production) throws IOException { - if (production) - return get(PRODUCTION_BASE_URL + "community/" + id + "/contentproviders"); - return (BETA_BASE_URL + "community/" + id + "/contentproviders"); + return get(baseURL + "community/communities"); } - public static String communityPropagationOrganization(String id, boolean production) throws IOException { - if (production) - return get(PRODUCTION_BASE_URL + "community/" + id + "/propagationOrganizations"); - return get(BETA_BASE_URL + "community/" + id + "/propagationOrganizations"); + public static String community(String id, String baseURL ) throws IOException { + + return get(baseURL + "community/" + id); + } - public static String communityProjects(String id, String page, String size, boolean production) throws IOException { - if (production) - return get(PRODUCTION_BASE_URL + "community/" + id + "/projects/" + page + "/" + size); - return get(BETA_BASE_URL + "community/" + id + "/projects/" + page + "/" + size); + public static String communityDatasource(String id, String baseURL ) throws IOException { + + return get(baseURL + "community/" + id + "/contentproviders"); + + + } + + public static String communityPropagationOrganization(String id, String baseURL ) throws IOException { + + return get(baseURL + "community/" + id + "/propagationOrganizations"); + + } + + public static String communityProjects(String id, String page, String size, String baseURL ) throws IOException { + + return get(baseURL + "community/" + id + "/projects/" + page + "/" + size); + } @NotNull diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java index 43d5e7e98..d121b8b7e 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/Utils.java @@ -36,14 +36,14 @@ public class Utils implements Serializable { private static final Logger log = LoggerFactory.getLogger(Utils.class); - public static CommunityConfiguration getCommunityConfiguration(boolean production) throws IOException { + public static CommunityConfiguration getCommunityConfiguration(String baseURL) throws IOException { final Map communities = Maps.newHashMap(); List validCommunities = new ArrayList<>(); - getValidCommunities(production) + getValidCommunities(baseURL) .forEach(community -> { try { CommunityModel cm = MAPPER - .readValue(QueryCommunityAPI.community(community.getId(), production), CommunityModel.class); + .readValue(QueryCommunityAPI.community(community.getId(), baseURL), CommunityModel.class); validCommunities.add(getCommunity(cm)); } catch (IOException e) { throw new RuntimeException(e); @@ -53,7 +53,7 @@ public class Utils implements Serializable { try { DatasourceList dl = MAPPER .readValue( - QueryCommunityAPI.communityDatasource(community.getId(), production), DatasourceList.class); + QueryCommunityAPI.communityDatasource(community.getId(), baseURL), DatasourceList.class); community.setProviders(dl.stream().map(d -> { if (d.getEnabled() == null || Boolean.FALSE.equals(d.getEnabled())) return null; @@ -98,9 +98,9 @@ public class Utils implements Serializable { return c; } - public static List getValidCommunities(boolean production) throws IOException { + public static List getValidCommunities(String baseURL) throws IOException { return MAPPER - .readValue(QueryCommunityAPI.communities(production), CommunitySummary.class) + .readValue(QueryCommunityAPI.communities(baseURL), CommunitySummary.class) .stream() .filter( community -> !community.getStatus().equals("hidden") && @@ -111,15 +111,15 @@ public class Utils implements Serializable { /** * it returns for each organization the list of associated communities */ - public static CommunityEntityMap getCommunityOrganization(boolean production) throws IOException { + public static CommunityEntityMap getCommunityOrganization(String baseURL) throws IOException { CommunityEntityMap organizationMap = new CommunityEntityMap(); - getValidCommunities(production) + getValidCommunities(baseURL) .forEach(community -> { String id = community.getId(); try { List associatedOrgs = MAPPER .readValue( - QueryCommunityAPI.communityPropagationOrganization(id, production), OrganizationList.class); + QueryCommunityAPI.communityPropagationOrganization(id, baseURL), OrganizationList.class); associatedOrgs.forEach(o -> { if (!organizationMap .keySet() @@ -136,9 +136,10 @@ public class Utils implements Serializable { return organizationMap; } - public static CommunityEntityMap getCommunityProjects(boolean production) throws IOException { + public static CommunityEntityMap getCommunityProjects(String baseURL) throws IOException { CommunityEntityMap projectMap = new CommunityEntityMap(); - getValidCommunities(production) + + getValidCommunities(baseURL) .forEach(community -> { int page = -1; int size = 100; @@ -150,7 +151,7 @@ public class Utils implements Serializable { .readValue( QueryCommunityAPI .communityProjects( - community.getId(), String.valueOf(page), String.valueOf(size), production), + community.getId(), String.valueOf(page), String.valueOf(size), baseURL), ContentModel.class); if (cm.getContent().size() > 0) { cm.getContent().forEach(p -> { diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java index 15712ad66..d972cecac 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java @@ -62,8 +62,8 @@ public class SparkBulkTagJob { final String outputPath = parser.get("outputPath"); log.info("outputPath: {}", outputPath); - final boolean production = Boolean.valueOf(parser.get("production")); - log.info("production: {}", production); + final String baseURL = parser.get("baseURL"); + log.info("baseURL: {}", baseURL); ProtoMap protoMappingParams = new Gson().fromJson(parser.get("pathMap"), ProtoMap.class); log.info("pathMap: {}", new Gson().toJson(protoMappingParams)); @@ -79,7 +79,7 @@ public class SparkBulkTagJob { if (taggingConf != null) { cc = CommunityConfigurationFactory.newInstance(taggingConf); } else { - cc = Utils.getCommunityConfiguration(production); + cc = Utils.getCommunityConfiguration(baseURL); } runWithSparkSession( diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunitySet.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunitySet.java index e32e94a4b..54fa60168 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunitySet.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromorganization/PrepareResultCommunitySet.java @@ -48,10 +48,10 @@ public class PrepareResultCommunitySet { final String outputPath = parser.get("outputPath"); log.info("outputPath: {}", outputPath); - final boolean production = Boolean.valueOf(parser.get("production")); - log.info("production: {}", production); + final String baseURL = parser.get("baseURL"); + log.info("baseURL: {}", baseURL); - final CommunityEntityMap organizationMap = Utils.getCommunityOrganization(production); + final CommunityEntityMap organizationMap = Utils.getCommunityOrganization(baseURL); log.info("organizationMap: {}", new Gson().toJson(organizationMap)); SparkConf conf = new SparkConf(); diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java index 7bee1ea0c..2c2eb69dd 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java @@ -52,11 +52,10 @@ public class PrepareResultCommunitySet { final String outputPath = parser.get("outputPath"); log.info("outputPath: {}", outputPath); - final boolean production = Boolean.valueOf(parser.get("production")); - log.info("production: {}", production); + final String baseURL = parser.get("baseURL"); + log.info("baseUEL: {}", baseURL); - final CommunityEntityMap projectsMap = Utils.getCommunityProjects(production); - // log.info("projectsMap: {}", new Gson().toJson(projectsMap)); + final CommunityEntityMap projectsMap = Utils.getCommunityProjects(baseURL); SparkConf conf = new SparkConf(); From b177cd5a0a30b4e982d7a37953ba9b33a339c8c1 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Tue, 14 Nov 2023 16:25:09 +0100 Subject: [PATCH 102/148] Project propagation via communityAPI instead of using IS via IIS --- .../eu/dnetlib/dhp/PropagationConstant.java | 4 +- .../PrepareResultCommunitySet.java | 124 +++++++++++++ .../ResultProjectList.java | 26 +++ .../SparkResultToCommunityFromProject.java | 163 ++++++++++++++++++ .../input_communitytoresult_parameters.json | 28 +++ ...t_preparecommunitytoresult_parameters.json | 28 +++ .../oozie_app/config-default.xml | 58 +++++++ .../oozie_app/workflow.xml | 144 ++++++++++++++++ .../ResultToCommunityJobTest.java | 133 ++++++++++++++ .../preparedInfo/resultcommunitylist | 4 + .../relation/relation | 20 +++ .../sample/dataset/dataset | 10 ++ .../otherresearchproduct/otherresearchproduct | 0 .../sample/publication/publication | 0 .../sample/software/software | 0 15 files changed, 741 insertions(+), 1 deletion(-) create mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java create mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultProjectList.java create mode 100644 dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java create mode 100644 dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/input_communitytoresult_parameters.json create mode 100644 dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/input_preparecommunitytoresult_parameters.json create mode 100644 dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/oozie_app/config-default.xml create mode 100644 dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/oozie_app/workflow.xml create mode 100644 dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultToCommunityJobTest.java create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromproject/preparedInfo/resultcommunitylist create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromproject/relation/relation create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromproject/sample/dataset/dataset create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromproject/sample/otherresearchproduct/otherresearchproduct create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromproject/sample/publication/publication create mode 100644 dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromproject/sample/software/software diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java index 1b381e970..7c4186663 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/PropagationConstant.java @@ -5,7 +5,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; -import org.apache.hadoop.conf.Configuration; import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoders; @@ -72,6 +71,9 @@ public class PropagationConstant { public static final String PROPAGATION_RESULT_COMMUNITY_ORGANIZATION_CLASS_ID = "result:community:organization"; public static final String PROPAGATION_RESULT_COMMUNITY_ORGANIZATION_CLASS_NAME = " Propagation of result belonging to community through organization"; + public static final String PROPAGATION_RESULT_COMMUNITY_PROJECT_CLASS_ID = "result:community:project"; + public static final String PROPAGATION_RESULT_COMMUNITY_PROJECT_CLASS_NAME = " Propagation of result belonging to community through project"; + public static final String PROPAGATION_ORCID_TO_RESULT_FROM_SEM_REL_CLASS_ID = "authorpid:result"; public static final String PROPAGATION_ORCID_TO_RESULT_FROM_SEM_REL_CLASS_NAME = "Propagation of authors pid to result through semantic relations"; diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java new file mode 100644 index 000000000..82625a122 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java @@ -0,0 +1,124 @@ + +package eu.dnetlib.dhp.resulttocommunityfromproject; + +import static eu.dnetlib.dhp.PropagationConstant.*; +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession; +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.util.*; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.io.compress.GzipCodec; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.api.java.function.MapGroupsFunction; +import org.apache.spark.sql.*; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.gson.Gson; + +import eu.dnetlib.dhp.api.Utils; +import eu.dnetlib.dhp.api.model.CommunityEntityMap; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.resulttocommunityfromorganization.ResultCommunityList; +import eu.dnetlib.dhp.resulttocommunityfromorganization.ResultOrganizations; +import eu.dnetlib.dhp.schema.common.ModelConstants; +import eu.dnetlib.dhp.schema.oaf.Relation; +import scala.Tuple2; + +public class PrepareResultCommunitySet { + + private static final Logger log = LoggerFactory.getLogger(PrepareResultCommunitySet.class); + + public static void main(String[] args) throws Exception { + String jsonConfiguration = IOUtils + .toString( + PrepareResultCommunitySet.class + .getResourceAsStream( + "/eu/dnetlib/dhp/resulttocommunityfromproject/input_preparecommunitytoresult_parameters.json")); + + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + parser.parseArgument(args); + + Boolean isSparkSessionManaged = isSparkSessionManaged(parser); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + String inputPath = parser.get("sourcePath"); + log.info("inputPath: {}", inputPath); + + final String outputPath = parser.get("outputPath"); + log.info("outputPath: {}", outputPath); + + final boolean production = Boolean.valueOf(parser.get("production")); + log.info("production: {}", production); + + final CommunityEntityMap projectsMap = Utils.getCommunityProjects(production); + // log.info("projectsMap: {}", new Gson().toJson(projectsMap)); + + SparkConf conf = new SparkConf(); + + runWithSparkSession( + conf, + isSparkSessionManaged, + spark -> { + removeOutputDir(spark, outputPath); + prepareInfo(spark, inputPath, outputPath, projectsMap); + }); + } + + private static void prepareInfo( + SparkSession spark, + String inputPath, + String outputPath, + CommunityEntityMap projectMap) { + + final StructType structureSchema = new StructType() + .add( + "dataInfo", new StructType() + .add("deletedbyinference", DataTypes.BooleanType) + .add("invisible", DataTypes.BooleanType)) + .add("source", DataTypes.StringType) + .add("target", DataTypes.StringType) + .add("relClass", DataTypes.StringType); + + spark + .read() + .schema(structureSchema) + .json(inputPath) + .filter( + "dataInfo.deletedbyinference != true " + + "and relClass == '" + ModelConstants.IS_PRODUCED_BY + "'") + .select( + new Column("source").as("resultId"), + new Column("target").as("projectId")) + .groupByKey((MapFunction) r -> (String) r.getAs("resultId"), Encoders.STRING()) + .mapGroups((MapGroupsFunction) (k, v) -> { + ResultProjectList rpl = new ResultProjectList(); + rpl.setResultId(k); + ArrayList cl = new ArrayList<>(); + cl.addAll(projectMap.get(v.next().getAs("projectId"))); + v.forEachRemaining(r -> { + projectMap + .get(r.getAs("projectId")) + .forEach(c -> { + if (!cl.contains(c)) + cl.add(c); + }); + + }); + if (cl.size() == 0) + return null; + rpl.setCommunityList(cl); + return rpl; + }, Encoders.bean(ResultProjectList.class)) + .filter(Objects::nonNull) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(outputPath); + } + +} diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultProjectList.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultProjectList.java new file mode 100644 index 000000000..ce96bbfac --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultProjectList.java @@ -0,0 +1,26 @@ + +package eu.dnetlib.dhp.resulttocommunityfromproject; + +import java.io.Serializable; +import java.util.ArrayList; + +public class ResultProjectList implements Serializable { + private String resultId; + private ArrayList communityList; + + public String getResultId() { + return resultId; + } + + public void setResultId(String resultId) { + this.resultId = resultId; + } + + public ArrayList getCommunityList() { + return communityList; + } + + public void setCommunityList(ArrayList communityList) { + this.communityList = communityList; + } +} diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java new file mode 100644 index 000000000..13202d79d --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java @@ -0,0 +1,163 @@ + +package eu.dnetlib.dhp.resulttocommunityfromproject; + +import static eu.dnetlib.dhp.PropagationConstant.*; +import static eu.dnetlib.dhp.PropagationConstant.PROPAGATION_RESULT_COMMUNITY_ORGANIZATION_CLASS_NAME; +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkHiveSession; +import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.apache.commons.io.IOUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.function.MapFunction; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.SaveMode; +import org.apache.spark.sql.SparkSession; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import eu.dnetlib.dhp.application.ArgumentApplicationParser; +import eu.dnetlib.dhp.resulttocommunityfromorganization.ResultCommunityList; +import eu.dnetlib.dhp.resulttocommunityfromorganization.SparkResultToCommunityFromOrganizationJob; +import eu.dnetlib.dhp.schema.common.ModelConstants; +import eu.dnetlib.dhp.schema.common.ModelSupport; +import eu.dnetlib.dhp.schema.oaf.Context; +import eu.dnetlib.dhp.schema.oaf.Result; +import scala.Tuple2; + +/** + * @author miriam.baglioni + * @Date 11/10/23 + */ +public class SparkResultToCommunityFromProject implements Serializable { + private static final Logger log = LoggerFactory.getLogger(SparkResultToCommunityFromProject.class); + + public static void main(String[] args) throws Exception { + String jsonConfiguration = IOUtils + .toString( + SparkResultToCommunityFromProject.class + .getResourceAsStream( + "/eu/dnetlib/dhp/resulttocommunityfromproject/input_communitytoresult_parameters.json")); + + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + + parser.parseArgument(args); + + Boolean isSparkSessionManaged = isSparkSessionManaged(parser); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + + String inputPath = parser.get("sourcePath"); + log.info("inputPath: {}", inputPath); + + final String outputPath = parser.get("outputPath"); + log.info("outputPath: {}", outputPath); + + final String possibleupdatespath = parser.get("preparedInfoPath"); + log.info("preparedInfoPath: {}", possibleupdatespath); + + SparkConf conf = new SparkConf(); + + runWithSparkSession( + conf, + isSparkSessionManaged, + spark -> { + + execPropagation(spark, inputPath, outputPath, possibleupdatespath); + + }); + } + + private static void execPropagation( + SparkSession spark, + String inputPath, + String outputPath, + + String possibleUpdatesPath) { + + Dataset possibleUpdates = readPath(spark, possibleUpdatesPath, ResultProjectList.class); + + ModelSupport.entityTypes + .keySet() + .parallelStream() + .forEach(e -> { + if (ModelSupport.isResult(e)) { + removeOutputDir(spark, outputPath + e.name()); + Class resultClazz = ModelSupport.entityTypes.get(e); + Dataset result = readPath(spark, inputPath + e.name(), resultClazz); + + result + .joinWith( + possibleUpdates, + result.col("id").equalTo(possibleUpdates.col("resultId")), + "left_outer") + .map(resultCommunityFn(), Encoders.bean(resultClazz)) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(outputPath + e.name()); + } + }); + + } + + private static MapFunction, R> resultCommunityFn() { + return value -> { + R ret = value._1(); + Optional rcl = Optional.ofNullable(value._2()); + if (rcl.isPresent()) { + // ArrayList communitySet = rcl.get().getCommunityList(); + List contextList = ret + .getContext() + .stream() + .map(Context::getId) + .collect(Collectors.toList()); + + @SuppressWarnings("unchecked") + R res = (R) ret.getClass().newInstance(); + + res.setId(ret.getId()); + List propagatedContexts = new ArrayList<>(); + for (String cId : rcl.get().getCommunityList()) { + if (!contextList.contains(cId)) { + Context newContext = new Context(); + newContext.setId(cId); + newContext + .setDataInfo( + Arrays + .asList( + getDataInfo( + PROPAGATION_DATA_INFO_TYPE, + PROPAGATION_RESULT_COMMUNITY_PROJECT_CLASS_ID, + PROPAGATION_RESULT_COMMUNITY_PROJECT_CLASS_NAME, + ModelConstants.DNET_PROVENANCE_ACTIONS))); + propagatedContexts.add(newContext); + } else { + ret + .getContext() + .stream() + .filter(c -> c.getId().equals(cId)) + .findFirst() + .get() + .getDataInfo() + .add( + getDataInfo( + PROPAGATION_DATA_INFO_TYPE, + PROPAGATION_RESULT_COMMUNITY_PROJECT_CLASS_ID, + PROPAGATION_RESULT_COMMUNITY_PROJECT_CLASS_NAME, + ModelConstants.DNET_PROVENANCE_ACTIONS)); + } + } + res.setContext(propagatedContexts); + ret.mergeFrom(res); + } + return ret; + }; + } +} diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/input_communitytoresult_parameters.json b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/input_communitytoresult_parameters.json new file mode 100644 index 000000000..0db8085d1 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/input_communitytoresult_parameters.json @@ -0,0 +1,28 @@ +[ + { + "paramName":"s", + "paramLongName":"sourcePath", + "paramDescription": "the path of the sequencial file to read", + "paramRequired": true + }, + + { + "paramName": "out", + "paramLongName": "outputPath", + "paramDescription": "the path used to store temporary output files", + "paramRequired": true + }, + { + "paramName": "ssm", + "paramLongName": "isSparkSessionManaged", + "paramDescription": "true if the spark session is managed, false otherwise", + "paramRequired": false + }, + { + "paramName": "p", + "paramLongName": "preparedInfoPath", + "paramDescription": "the path where prepared info have been stored", + "paramRequired": true + } + +] \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/input_preparecommunitytoresult_parameters.json b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/input_preparecommunitytoresult_parameters.json new file mode 100644 index 000000000..9a50c79fa --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/input_preparecommunitytoresult_parameters.json @@ -0,0 +1,28 @@ +[ + { + "paramName":"s", + "paramLongName":"sourcePath", + "paramDescription": "the path of the sequencial file to read", + "paramRequired": true + }, + + { + "paramName": "ssm", + "paramLongName": "isSparkSessionManaged", + "paramDescription": "true if the spark session is managed, false otherwise", + "paramRequired": false + }, + { + "paramName": "out", + "paramLongName": "outputPath", + "paramDescription": "the path used to store temporary output files", + "paramRequired": true + }, + { + "paramName": "p", + "paramLongName": "production", + "paramDescription": "the path used to store temporary output files", + "paramRequired": true + } + +] \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/oozie_app/config-default.xml b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/oozie_app/config-default.xml new file mode 100644 index 000000000..2744ea92b --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/oozie_app/config-default.xml @@ -0,0 +1,58 @@ + + + jobTracker + yarnRM + + + nameNode + hdfs://nameservice1 + + + oozie.use.system.libpath + true + + + oozie.action.sharelib.for.spark + spark2 + + + hive_metastore_uris + thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083 + + + spark2YarnHistoryServerAddress + http://iis-cdh5-test-gw.ocean.icm.edu.pl:18089 + + + spark2EventLogDir + /user/spark/spark2ApplicationHistory + + + spark2ExtraListeners + com.cloudera.spark.lineage.NavigatorAppListener + + + spark2SqlQueryExecutionListeners + com.cloudera.spark.lineage.NavigatorQueryListener + + + sparkExecutorNumber + 4 + + + sparkDriverMemory + 15G + + + sparkExecutorMemory + 6G + + + sparkExecutorCores + 1 + + + spark2MaxExecutors + 50 + + \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/oozie_app/workflow.xml b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/oozie_app/workflow.xml new file mode 100644 index 000000000..21cc2d887 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/oozie_app/workflow.xml @@ -0,0 +1,144 @@ + + + + sourcePath + the source path + + + + outputPath + the output path + + + + + ${jobTracker} + ${nameNode} + + + oozie.action.sharelib.for.spark + ${oozieActionShareLibForSpark2} + + + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + + + + + + + + + + + + + + + + + ${nameNode}/${sourcePath}/relation + ${nameNode}/${outputPath}/relation + + + + + + + + ${nameNode}/${sourcePath}/organization + ${nameNode}/${outputPath}/organization + + + + + + + + ${nameNode}/${sourcePath}/project + ${nameNode}/${outputPath}/project + + + + + + + + ${nameNode}/${sourcePath}/datasource + ${nameNode}/${outputPath}/datasource + + + + + + + + + + yarn + cluster + Prepare-Community-Result-Organization + eu.dnetlib.dhp.resulttocommunityfromproject.PrepareResultCommunitySet + dhp-enrichment-${projectVersion}.jar + + --executor-cores=6 + --executor-memory=5G + --conf spark.executor.memoryOverhead=3g + --conf spark.sql.shuffle.partitions=3284 + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + + --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors} + + --sourcePath${sourcePath}/relation + --outputPath${workingDir}/preparedInfo/resultCommunityList + --production${production} + + + + + + + + yarn + cluster + community2resultfromproject + eu.dnetlib.dhp.resulttocommunityfromproject.SparkResultToCommunityFromProject + dhp-enrichment-${projectVersion}.jar + + --executor-cores=6 + --executor-memory=5G + --conf spark.executor.memoryOverhead=3g + --conf spark.sql.shuffle.partitions=3284 + --driver-memory=${sparkDriverMemory} + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors} + + --preparedInfoPath${workingDir}/preparedInfo/resultCommunityList + --sourcePath${sourcePath}/ + --outputPath${outputPath}/ + + + + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultToCommunityJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultToCommunityJobTest.java new file mode 100644 index 000000000..40b9745c5 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultToCommunityJobTest.java @@ -0,0 +1,133 @@ + +package eu.dnetlib.dhp.resulttocommunityfromproject; + +import static org.apache.spark.sql.functions.desc; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.apache.commons.io.FileUtils; +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.orcidtoresultfromsemrel.OrcidPropagationJobTest; +import eu.dnetlib.dhp.resulttocommunityfromorganization.SparkResultToCommunityFromOrganizationJob; +import eu.dnetlib.dhp.schema.oaf.Context; +import eu.dnetlib.dhp.schema.oaf.Dataset; + +public class ResultToCommunityJobTest { + + private static final Logger log = LoggerFactory.getLogger(ResultToCommunityJobTest.class); + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static SparkSession spark; + + private static Path workingDir; + + @BeforeAll + public static void beforeAll() throws IOException { + workingDir = Files.createTempDirectory(ResultToCommunityJobTest.class.getSimpleName()); + log.info("using work dir {}", workingDir); + + SparkConf conf = new SparkConf(); + conf.setAppName(ResultToCommunityJobTest.class.getSimpleName()); + + conf.setMaster("local[*]"); + conf.set("spark.driver.host", "localhost"); + conf.set("hive.metastore.local", "true"); + conf.set("spark.ui.enabled", "false"); + conf.set("spark.sql.warehouse.dir", workingDir.toString()); + conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString()); + + spark = SparkSession + .builder() + .appName(OrcidPropagationJobTest.class.getSimpleName()) + .config(conf) + .getOrCreate(); + } + + @AfterAll + public static void afterAll() throws IOException { + FileUtils.deleteDirectory(workingDir.toFile()); + spark.stop(); + } + + @Test + void testSparkResultToCommunityFromProjectJob() throws Exception { + final String preparedInfoPath = getClass() + .getResource("/eu/dnetlib/dhp/resulttocommunityfromproject/preparedInfo") + .getPath(); + SparkResultToCommunityFromProject + .main( + new String[] { + + "-isSparkSessionManaged", Boolean.FALSE.toString(), + "-sourcePath", getClass() + .getResource("/eu/dnetlib/dhp/resulttocommunityfromproject/sample/") + .getPath(), + + "-outputPath", workingDir.toString() + "/", + "-preparedInfoPath", preparedInfoPath + }); + + final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .textFile(workingDir.toString() + "/dataset") + .map(item -> OBJECT_MAPPER.readValue(item, Dataset.class)); + + Assertions.assertEquals(10, tmp.count()); + /** + * {"resultId":"50|57a035e5b1ae::d5be548ca7ae489d762f893be67af52f","communityList":["aurora"]} + * {"resultId":"50|57a035e5b1ae::a77232ffca9115fcad51c3503dbc7e3e","communityList":["aurora"]} + * {"resultId":"50|57a035e5b1ae::803aaad4decab7e27cd4b52a1931b3a1","communityList":["sdsn-gr"]} + * {"resultId":"50|57a035e5b1ae::a02e9e4087bca50687731ae5c765b5e1","communityList":["netherlands"]} + */ + List context = tmp + .filter(r -> r.getId().equals("50|57a035e5b1ae::d5be548ca7ae489d762f893be67af52f")) + .first() + .getContext(); + Assertions.assertTrue(context.stream().anyMatch(c -> containsResultCommunityProject(c))); + + context = tmp + .filter(r -> r.getId().equals("50|57a035e5b1ae::a77232ffca9115fcad51c3503dbc7e3e")) + .first() + .getContext(); + Assertions.assertTrue(context.stream().anyMatch(c -> containsResultCommunityProject(c))); + + Assertions + .assertEquals( + 0, tmp.filter(r -> r.getId().equals("50|57a035e5b1ae::803aaad4decab7e27cd4b52a1931b3a1")).count()); + + Assertions + .assertEquals( + 0, tmp.filter(r -> r.getId().equals("50|57a035e5b1ae::a02e9e4087bca50687731ae5c765b5e1")).count()); + + Assertions + .assertEquals( + 2, tmp.filter(r -> r.getContext().stream().anyMatch(c -> c.getId().equals("aurora"))).count()); + + } + + private static boolean containsResultCommunityProject(Context c) { + return c + .getDataInfo() + .stream() + .anyMatch(di -> di.getProvenanceaction().getClassid().equals("result:community:project")); + } +} diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromproject/preparedInfo/resultcommunitylist b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromproject/preparedInfo/resultcommunitylist new file mode 100644 index 000000000..a9bbccb5b --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromproject/preparedInfo/resultcommunitylist @@ -0,0 +1,4 @@ +{"resultId":"50|57a035e5b1ae::d5be548ca7ae489d762f893be67af52f","communityList":["aurora"]} +{"resultId":"50|57a035e5b1ae::a77232ffca9115fcad51c3503dbc7e3e","communityList":["aurora"]} +{"resultId":"50|57a035e5b1ae::803aaad4decab7e27cd4b52a1931b3a1","communityList":["sdsn-gr"]} +{"resultId":"50|57a035e5b1ae::a02e9e4087bca50687731ae5c765b5e1","communityList":["netherlands"]} \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromproject/relation/relation b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromproject/relation/relation new file mode 100644 index 000000000..ecf89016f --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromproject/relation/relation @@ -0,0 +1,20 @@ +{"subRelType": "affiliation", "relClass": "isProducedBy", "dataInfo": {"provenanceaction": {"classid": "iis", "classname": "Inferred by OpenAIRE", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": true, "inferenceprovenance": "iis::document_affiliations", "invisible": false, "trust": "0.7731"}, "target": "40|corda__h2020::5c16b849965ee04493a5e244471aae16", "lastupdatetimestamp": 1694431186898, "relType": "resultOrganization", "source": "50|57a035e5b1ae::803aaad4decab7e27cd4b52a1931b3a1", "collectedfrom": [], "validated": false, "properties": []} +{"subRelType": "affiliation", "relClass": "isProducedBy", "dataInfo": {"provenanceaction": {"classid": "iis", "classname": "Inferred by OpenAIRE", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": true, "inferenceprovenance": "iis::document_affiliations", "invisible": false, "trust": "0.9"}, "target": "40|nwo_________::a9f6d38fb3626d6659d385f71be9657e", "lastupdatetimestamp": 1694431155490, "relType": "resultOrganization", "source": "50|57a035e5b1ae::a02e9e4087bca50687731ae5c765b5e1", "validated": false, "properties": []} +{"subRelType": "affiliation", "relClass": "isProducedBy", "dataInfo": {"provenanceaction": {"classid": "iis", "classname": "Inferred by OpenAIRE", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": true, "inferenceprovenance": "iis::document_affiliations", "invisible": false, "trust": "0.9"}, "target": "40|ukri________::93b0983bc4fb1a17a8c3d1e6ab45c03b", "lastupdatetimestamp": 1694431195409, "relType": "resultOrganization", "source": "50|57a035e5b1ae::a77232ffca9115fcad51c3503dbc7e3e", "collectedfrom": [], "validated": false, "properties": []} +{"subRelType": "affiliation", "relClass": "isProducedBy", "dataInfo": {"provenanceaction": {"classid": "iis", "classname": "Inferred by OpenAIRE", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": true, "inferenceprovenance": "iis::document_affiliations", "invisible": false, "trust": "0.7731"}, "target": "40|ukri________::93b0983bc4fb1a17a8c3d1e6ab45c03b", "lastupdatetimestamp": 1694431195538, "relType": "resultOrganization", "source": "50|57a035e5b1ae::d5be548ca7ae489d762f893be67af52f", "collectedfrom": [], "validated": false, "properties": []} +{"subRelType": "affiliation", "relClass": "isProducedBy", "dataInfo": {"provenanceaction": {"classid": "iis", "classname": "Inferred by OpenAIRE", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": true, "inferenceprovenance": "iis::document_affiliations", "invisible": false, "trust": "0.9"}, "target": "40|openorgs____::eb0669daa9efeb898a3090d8aac7c953", "lastupdatetimestamp": 1694431216929, "relType": "resultOrganization", "source": "50|RECOLECTA___::031d8312287a0108202acd8c5957fcb5", "collectedfrom": [], "validated": false, "properties": []} +{"subRelType": "affiliation", "relClass": "isProducedBy", "dataInfo": {"provenanceaction": {"classid": "iis", "classname": "Inferred by OpenAIRE", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": true, "inferenceprovenance": "iis::document_affiliations", "invisible": false, "trust": "0.9"}, "target": "40|pending_org_::26d4324bfca459ab17e1efd966fba8d7", "lastupdatetimestamp": 1694431156296, "relType": "resultOrganization", "source": "50|RECOLECTA___::0ac43c9933175f1d0e091ba4dc814565", "collectedfrom": [], "validated": false, "properties": []} +{"subRelType": "affiliation", "relClass": "isProducedBy", "dataInfo": {"provenanceaction": {"classid": "iis", "classname": "Inferred by OpenAIRE", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": true, "inferenceprovenance": "iis::document_affiliations", "invisible": false, "trust": "0.9"}, "target": "40|pending_org_::fc8f734ba211cfc8a3769189808338c2", "lastupdatetimestamp": 1694431218717, "relType": "resultOrganization", "source": "50|RECOLECTA___::1c6c582aa2d57c932069bc4382653229", "collectedfrom": [], "validated": false, "properties": []} +{"subRelType": "affiliation", "relClass": "isProducedBy", "dataInfo": {"provenanceaction": {"classid": "iis", "classname": "Inferred by OpenAIRE", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": true, "inferenceprovenance": "iis::document_affiliations", "invisible": false, "trust": "0.9"}, "target": "40|pending_org_::15c4bd8602727bff4ecbaf69e7ac43af", "lastupdatetimestamp": 1694431211627, "relType": "resultOrganization", "source": "50|RECOLECTA___::21c4e85b761c898dd0f6d59d9f9b85f1", "collectedfrom": [], "validated": false, "properties": []} +{"subRelType": "affiliation", "relClass": "isProducedBy", "dataInfo": {"provenanceaction": {"classid": "iis", "classname": "Inferred by OpenAIRE", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": true, "inferenceprovenance": "iis::document_affiliations", "invisible": false, "trust": "0.9"}, "target": "40|pending_org_::6e4694bb54928162e3736e2042663ff1", "lastupdatetimestamp": 1694431213910, "relType": "resultOrganization", "source": "50|RECOLECTA___::22436b0491ae186b64671a0667551830", "collectedfrom": [], "validated": false, "properties": []} +{"subRelType": "affiliation", "relClass": "isProducedBy", "dataInfo": {"provenanceaction": {"classid": "iis", "classname": "Inferred by OpenAIRE", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": true, "inferenceprovenance": "iis::document_affiliations", "invisible": false, "trust": "0.9"}, "target": "40|openorgs____::5665ec851e301a1f816ee0be3e98757b", "lastupdatetimestamp": 1694431212739, "relType": "resultOrganization", "source": "50|RECOLECTA___::6739f9dda9d6e4703b70f908a4ab6259", "collectedfrom": [], "validated": false, "properties": []} +{"subRelType": "affiliation", "relClass": "isProducedBy", "dataInfo": {"provenanceaction": {"classid": "iis", "classname": "Inferred by OpenAIRE", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": true, "inferenceprovenance": "iis::document_affiliations", "invisible": false, "trust": "0.9"}, "target": "40|pending_org_::7ccc3885d6c13b55952c4af07e95f6a6", "lastupdatetimestamp": 1694431203420, "relType": "resultOrganization", "source": "50|RECOLECTA___::a3baa9c23ff95de4f09bf711bcf934a5", "validated": false, "properties": []} +{"subRelType": "affiliation", "relClass": "isProducedBy", "dataInfo": {"provenanceaction": {"classid": "iis", "classname": "Inferred by OpenAIRE", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": true, "inferenceprovenance": "iis::document_affiliations", "invisible": false, "trust": "0.8997"}, "target": "40|openorgs____::2cd85ac41e550e32a488a1b26a71fdd8", "lastupdatetimestamp": 1694431162512, "relType": "resultOrganization", "source": "50|RECOLECTA___::ce1f62947f2773709b75e57ec91eca51", "collectedfrom": [], "validated": false, "properties": []} +{"subRelType": "affiliation", "relClass": "isProducedBy", "dataInfo": {"provenanceaction": {"classid": "iis", "classname": "Inferred by OpenAIRE", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": true, "inferenceprovenance": "iis::document_affiliations", "invisible": false, "trust": "0.9"}, "target": "40|pending_org_::2f224aa8c572c740fac1ed8b1a38aa21", "lastupdatetimestamp": 1694431204317, "relType": "resultOrganization", "source": "50|RECOLECTA___::df660a72ea2d758e542cdd0d63dcdbd1", "collectedfrom": [], "validated": false, "properties": []} +{"subRelType": "affiliation", "relClass": "isProducedBy", "dataInfo": {"provenanceaction": {"classid": "iis", "classname": "Inferred by OpenAIRE", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": true, "inferenceprovenance": "iis::document_affiliations", "invisible": false, "trust": "0.7731"}, "target": "40|openorgs____::d4c564d89cc281f00a27ff7684da5668", "lastupdatetimestamp": 1694431222761, "relType": "resultOrganization", "source": "50|altaiap_____::5c53480b2300ae410d183822b0c1ee9b", "collectedfrom": [], "validated": false, "properties": []} +{"subRelType": "affiliation", "relClass": "isProducedBy", "dataInfo": {"provenanceaction": {"classid": "iis", "classname": "Inferred by OpenAIRE", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": true, "inferenceprovenance": "iis::document_affiliations", "invisible": false, "trust": "0.9"}, "target": "40|pending_org_::5f8192f98afda4a06afb4386b7c913cc", "lastupdatetimestamp": 1694431180171, "relType": "resultOrganization", "source": "50|altaiap_____::d036acbdb3155339d2fa77455a0b3d95", "validated": false, "properties": []} +{"subRelType": "affiliation", "relClass": "isProducedBy", "dataInfo": {"provenanceaction": {"classid": "iis", "classname": "Inferred by OpenAIRE", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": true, "inferenceprovenance": "iis::document_affiliations", "invisible": false, "trust": "0.8998"}, "target": "40|pending_org_::afc6d809fd8a8db5ee592469c12a7dc8", "lastupdatetimestamp": 1694431193653, "relType": "resultOrganization", "source": "50|arXiv_______::0007ee48e3aa0c409dc68f172bbc76a8", "collectedfrom": [], "validated": false, "properties": []} +{"subRelType": "affiliation", "relClass": "isProducedBy", "dataInfo": {"provenanceaction": {"classid": "iis", "classname": "Inferred by OpenAIRE", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": true, "inferenceprovenance": "iis::document_affiliations", "invisible": false, "trust": "0.8998"}, "target": "40|openorgs____::5ab85d71ee109cc81a2bc28cbdeba96a", "lastupdatetimestamp": 1694431224324, "relType": "resultOrganization", "source": "50|arXiv_______::00285f6ee59920bcdb4d96d6cd240095", "collectedfrom": [], "validated": false, "properties": []} +{"subRelType": "affiliation", "relClass": "isProducedBy", "dataInfo": {"provenanceaction": {"classid": "iis", "classname": "Inferred by OpenAIRE", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": true, "inferenceprovenance": "iis::document_affiliations", "invisible": false, "trust": "0.9"}, "target": "40|pending_org_::7d22ea802b3405becbde1e2e1ea59197", "lastupdatetimestamp": 1694431220671, "relType": "resultOrganization", "source": "50|arXiv_______::002eb5d6ee54ba872b882bbd3824aa96", "collectedfrom": [], "validated": false, "properties": []} +{"subRelType": "affiliation", "relClass": "isProducedBy", "dataInfo": {"provenanceaction": {"classid": "iis", "classname": "Inferred by OpenAIRE", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": true, "inferenceprovenance": "iis::document_affiliations", "invisible": false, "trust": "0.8847"}, "target": "40|pending_org_::c44367f733bf59e75fc7e92eb9ad4489", "lastupdatetimestamp": 1694431179461, "relType": "resultOrganization", "source": "50|arXiv_______::00480a6b5ae2cd88dbd7781e1ec341aa", "collectedfrom": [], "validated": false, "properties": []} +{"subRelType": "affiliation", "relClass": "isProducedBy", "dataInfo": {"provenanceaction": {"classid": "iis", "classname": "Inferred by OpenAIRE", "schemeid": "dnet:provenanceActions", "schemename": "dnet:provenanceActions"}, "deletedbyinference": false, "inferred": true, "inferenceprovenance": "iis::document_affiliations", "invisible": false, "trust": "0.8998"}, "target": "40|openorgs____::293e60068d63e17e20f2de54944cae01", "lastupdatetimestamp": 1694431219918, "relType": "resultOrganization", "source": "50|arXiv_______::00766c4ea96b8b797528b511b793ba62", "collectedfrom": [], "validated": false, "properties": []} \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromproject/sample/dataset/dataset b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromproject/sample/dataset/dataset new file mode 100644 index 000000000..a264fbc26 --- /dev/null +++ b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromproject/sample/dataset/dataset @@ -0,0 +1,10 @@ +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055868909,"id":"50|57a035e5b1ae::a77232ffca9115fcad51c3503dbc7e3e","originalId":["od______3989::02dd5d2c222191b0b9bd4f33c8e96529"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[{"value":"10.4185/RLCS-2018-1243","qualifier":{"classid":"doi","classname":"doi","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofcollection":"2020-03-03T13:05:26.091Z","dateoftransformation":"2020-03-03T13:06:53.161Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:26.091Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/562","datestamp":"2018-01-23T15:06:07Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"affiliation":[],"fullname":"Jung, Nicole","name":"Nicole","pid":[{"dataInfo":null,"qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":null,"schemename":null},"value":"0000-0001-9513-2468"}],"rank":1,"surname":"Jung"},{"affiliation":[],"fullname":"Gräßle, Simone","name":"Simone","pid":[],"rank":2,"surname":"Gräßle"}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Comunicación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"yihadismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"viralidad","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"vídeo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"propaganda","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"terrorismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"terrorism","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"jihadism","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"virality","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"video","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"5903 Ideologías Políticas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"6310.13 Terrorismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"El individualismo como estrategia","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Individualism as an improvised strategy","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Se analiza como el Yihadismo distribuye su mensaje propaganístico en You Tube.\nUniversidad Camilo José Cela","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"MediaLab - UCJC","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[{"dataInfo":[{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"propagation:community:organizationtoresult","classname":"Propagation of community result through organization to result association","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.85"}],"id":"beopen"}],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/562"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-01-01","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055868954,"id":"50|57a035e5b1ae::d5be548ca7ae489d762f893be67af52f","originalId":["od______3989::05d8c751462f9bb8d2b06956dfbc5c7b"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:26.079Z","dateoftransformation":"2020-03-03T13:06:53.239Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:26.079Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/227","datestamp":"2017-11-15T16:29:55Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"affiliation":[],"fullname":"Fischer, Malte","name":"Malte","pid":[{"dataInfo":null,"qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":null,"schemename":null},"value":"0000-0002-2806-1302"}],"rank":1,"surname":"Fischer"},{"affiliation":[],"fullname":"Barbul, Daniel","name":"Daniel","pid":[],"rank":2,"surname":"Barbul"},{"affiliation":[],"fullname":"Schmidtmann, Marc","name":"Marc","pid":[{"dataInfo":null,"qualifier":{"classid":"MAG Identifier","classname":"MAG Identifier","schemeid":null,"schemename":null},"value":"2024269002"}],"rank":3,"surname":"Schmidtmann"},{"affiliation":[],"fullname":"Beckhaus, Ruediger","name":"Ruediger","pid":[],"rank":4,"surname":"Beckhaus"}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Actividad Física y del Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Eventos deportivos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Retos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"I Congreso Nuevos Retos en los Eventos Deportivos","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"I Congreso Nuevos Retos en los Eventos Deportivos, celebrado en la Universidad Camilo José Cela.\n\n\n\nDerivado del proyecto de investigación \"La situación de los eventos deportivos en España a través de su estructuras y de sus herramientas\", tiene como finalidad conocer la situación de los eventos deportivos en España. Se realizó un estudio descriptivo holístico en el que se investigaron elementos culturales, el perfil de los recursos humanos, la estructura de los comités, entre otras.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2015-06-20","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/227"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2015-06-20","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055869097,"id":"50|od________18::8887b1df8b563c4ea851eb9c882c9d7b","originalId":["od______3989::0f89464c4ac4c398fe0c71433b175a62"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:26.069Z","dateoftransformation":"2020-03-03T13:06:53.54Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:26.069Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/802","datestamp":"2018-11-14T15:30:47Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Ciencias de la Actividad Física y del Deporte","name":"Ciencias La Actividad Física Y. Del Deporte","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Actividad Física y del Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Actividades deportivas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Impacto físico","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Proyectos de investigación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"2411.06 Fisiología del Ejercicio","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Investigaciones en Ciencias de la Actividad Física y del Deporte CCAFD - UCJC","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Blanca Romero, investigadora y docente de Ciencias de la Actividad Física y del Deporte en la Universidad Camilo José Cela, describe dos de las líneas de investigación que se han desarrollado en el departamento CCAFD de UCJC: el impacto físico de las diferentes actividades deportivas y los métodos de recuperación más eficaces tras el ejercicio físico.\nBlanca también destaca la importancia en el ámbito de la salud los estudios en Ciencias de la Actividad Física y del Deporte.\nCiencias de la Actividad Física y del Deporte en la Universidad Camilo José Cela","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2016-03-21","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Universidad Camilo José Cela, Ciencias de la Actividad Física y del Deporte","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[{"dataInfo":[{"deletedbyinference":false,"inferenceprovenance":"","inferred":true,"invisible":false,"provenanceaction":{"classid":"propagation:community:organizationtoresult","classname":"Propagation of community result through organization to result association","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.85"}],"id":"beopen"}],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/802"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2016-03-21","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055869215,"id":"50|doajarticles::8d817039a63710fcf97e30f14662c6c8","originalId":["od______3989::1729c3988199b95d1d566851af7d3c55"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:25.653Z","dateoftransformation":"2020-03-03T13:06:53.764Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:25.653Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/801","datestamp":"2018-11-14T15:19:38Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Ciencias de la Actividad Física y del Deporte","name":"Ciencias La Actividad Física Y. Del Deporte","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Actividad Física y del Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Prevención","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Hidratación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deportistas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Electrolitos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deportes de resistencia","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"2411.06 Fisiología del Ejercicio","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"La importancia de la hidratación en competiciones resistencia, por Juan del Coso","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Juan del Coso, investigador y profesor en el Grado en Ciencias de la Actividad Física y del Deporte UCJC, presenta el proyecto de investigación que ha realizado el departamento CCAFD de UCJC: la prevención de desequilibrios de agua y electrolitos que se dan en los deportes de resistencia para que los deportistas realicen su actividad de una manera más segura.\nMás información: http://www.ucjc.edu/2016/03/investiga...\nUniversidad Camilo José Cela, Ciencias de la Actividad Física y del Deporte","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2016-03-21","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Grado en Ciencias de la Actividad Física y del Deporte UCJC","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[{"dataInfo":[{"deletedbyinference":false,"inferenceprovenance":"","inferred":true,"invisible":false,"provenanceaction":{"classid":"propagation:community:organizationtoresult","classname":"Propagation of community result through organization to result association","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.85"}],"id":"euromarine"}],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/801"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2016-03-21","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055869487,"id":"50|doajarticles::3c98f0632f1875b4979e552ba3aa01e6","originalId":["od______3989::2e3f34ce90520fae350a7e1148d7dcea"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.912Z","dateoftransformation":"2020-03-03T13:06:54.218Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.912Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/800","datestamp":"2018-11-14T13:43:21Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Actividad Física y del Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Deporte","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Cafeína","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Efectos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Alto rendimiento","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Competiciones","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Beneficios","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"2411.06 Fisiología del Ejercicio","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Investigación - Efectos positivos y negativos del uso de la cafeína en el deporte","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Proyecto de investigación que se están desarrollando en la Universidad Camilo José Cela (UCJC) sobre la cafeína y sus efectos cuando se realiza deporte. https://www.ucjc.edu/2016/08/la-bbc-s...\n\nJuan Del Coso, director del Laboratorio de Fisiología del Ejercicio de la Universidad Camilo José Cela, nos describe cómo está siendo la investigación que desarrolla con deportistas de alto rendimiento y el consumo de cafeína.\n\n\"Nuestra investigación se enmarca dentro del campo de la fisiología del ejercicio y el objetivo que tenemos es investigar los efectos que tiene la cafeína en los deportistas, principalmente saber si es una ayuda ergogénica en la mayor parte de los deportes, pero también conocer qué perjuicios existen para aquellos deportistas que la utilizan para incrementar el rendimiento deportivo\", explica Del Coso.\n\nY es que los datos son muy llamativos: tres de cada cuatro deportistas de alto rendimiento utilizan la cafeína antes de la competición y muchas veces se utiliza sin saber los perjuicios que puede tener para el deportista. \"Lo que estamos investigando es cómo una sustancia, que te puede hacer rendir mejor, cómo puede afectar negativamente cuando termina una competición\".\n\nPero esta investigación no sólo puede extraer conclusiones interesantes y útiles para los deportistas profesionales, \"investigando el deporte de alto rendimiento podemos conocer los beneficios y perjuicios de la cafeína que también afectarían al deportista amateur y hacer un deporte en general más seguro\".\n\nPara el desarrollo de este proyecto de investigación están colaborando el Laboratorio de Fisiología del Ejercicio de la Universidad Camilo José Cela y la Agencia para la Protección de la Salud en el Deporte.\nPara el desarrollo de este proyecto de investigación están colaborando el Laboratorio de Fisiología del Ejercicio de la Universidad Camilo José Cela y la Agencia para la Protección de la Salud en el Deporte.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2017-10-06","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Universidad Camilo José Cela, Vicerrectorado de Innovación","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[{"dataInfo":[{"deletedbyinference":false,"inferenceprovenance":"","inferred":true,"invisible":false,"provenanceaction":{"classid":"propagation:community:organizationtoresult","classname":"Propagation of community result through organization to result association","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.85"}],"id":"euromarine"},{"dataInfo":[{"deletedbyinference":false,"inferenceprovenance":"","inferred":true,"invisible":false,"provenanceaction":{"classid":"propagation:community:organizationtoresult","classname":"Propagation of community result through organization to result association","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.85"}],"id":"ni"}],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/800"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2017-10-06","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055869498,"id":"50|od______3989::2f4f3c820c450bd08dac08d07cc82dcf","originalId":["od______3989::2f4f3c820c450bd08dac08d07cc82dcf"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.806Z","dateoftransformation":"2020-03-03T13:06:54.229Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.806Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/797","datestamp":"2018-11-14T12:12:23Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Cuesta Cano, Laura","name":"Laura","surname":"Cuesta Cano","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Comunicación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Marketing digital","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Medios sociales","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Marcas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Empresas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Emprendedores","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"II Liga de Debate CICAE - UCJC","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"6114.13 Marketing","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Laura Cuesta - Liga de Debate CICAE - UCJC","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Laura Cuesta, profesora en la Universidad Camilo José Cela (UCJC) y Especialista en Marketing Digital y Medios Sociales para marcas y emprendedores. \n\nEn esta ocasión, Laura Cuesta, nos explica el significado de redes sociales y nos habla sobre el uso que le dan los usuarios.\n\nII Liga de Debate CICAE - UCJC http://ligacicae.ucjc.edu/\n\nLa Universidad Camilo José Cela (UCJC) y la Asociación de Colegios Privados e Independientes –Círculo de Calidad Educativa (CICAE) – organizan la segunda edición de esta gran liga de debate académico. El formato es al estilo inglés, similar al de la Oxford Union, donde algunos jóvenes se reunían a debatir sobre temas de la más candente actualidad y polemizaban con la intención de formarse en tan valoradas habilidades.\n\nLa Liga de Debate CICAE-UCJC es una ocasión única para los alumnos de ponerse en contacto con herramientas tan útiles como la oratoria y la comunicación efectiva. Pero, sobretodo, una oportunidad de fomentar el pensamiento crítico y la socialización entre alumnos con espíritu emprendedor. La Liga de Debate CICAE - Universidad Camilo José Cela se diferencia del resto de torneos en que potenciamos que sea eminentemente formativa. Alumnos y profesores reciben formación en comunicación, oratoria y debate por parte de alguno de los expertos en debate españoles más reconocidos en la escena internacional.\n\nDatos de la III Liga de Debate CICAE - UCJC http://ligacicae.ucjc.edu/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-10-23","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Universidad Camilo José Cela, Vicerrectorado de Innovación","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/797"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-10-23","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055870582,"id":"50|od______3989::752fd0b2bbac1ea1cc50e52fd46eb663","originalId":["od______3989::752fd0b2bbac1ea1cc50e52fd46eb663"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.561Z","dateoftransformation":"2020-03-03T13:06:55.716Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.561Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/798","datestamp":"2018-11-14T12:57:26Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Psicología","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Alcohol","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Dependencia","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Proyectos de investigación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Alcoholismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Realidad virtual","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Psicología conductual","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Investigación - Intervención basada en realidad virtual sobre la dependencia del alcohol","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Proyecto de investigación que se están desarrollando en la Universidad Camilo José Cela (UCJC) con tecnología aplicada al abandono de la dependencia del alcohol. \nRosa Jurado, profesora e investigadora de UCJC https://www.ucjc.edu/, explica cómo se conjuga el alcoholismo con la realidad virtual. \"Mi investigación consiste en desarrollar una técnica de intervención o rehabilitación que tiene como propósito fomentar la capacidad de inhibición para que las personas que tienen dependencia del alcohol sean capaces de controlar su conducta de aproximación hacia aquello que tiene que ver con el consumo de alcohol\".\n\nEn esta investigación, liderada por la Universidad Camilo José Cela, están participando, el Instituto de Investigación Biomédica del Hospital 12 de Octubre y la Universidad Politécnica de Madrid, con el CeDint. \"En nuestro equipo de investigación pensamos que, además del deseo de consumo, existen una carencia clara de capacidad de control inhibitorio en este tipo de personas\", destaca Rosa Jurado.\n\nLa realidad virtual sirve, según explica Gabriel Rubio, jefe de Psiquiatría del Hospital 12 de Octubre, \"para dar un paso más, para ver cómo reacciona un sujeto en un ambiente que no es el real, pero se parece mucho al real\". El objetivo final de estas acciones es \"mejorar la capacidad del individuo para poner en marcha mecanismos de inhibición conductual, para que el sujeto sea capaz de decir que no\".\n\nY no acaba ahí el proceso. La fase final del tratamiento se enfoca en que el paciente \"sea capaz de reconstruir su vida\" a través de programas de entrenamiento de habilidades sociales, asertividad, familias... \"Juntando todas las áreas de intervención, la abstinencia se mantiene mucho más tiempo y las recaídas disminuyen\".","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2017-10-05","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[{"value":"Universidad Camilo José Cela, Vicerrectorado de Innovación","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/798"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2017-10-05","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055870750,"id":"50|od______3989::7fcbe3a03280663cddebfd3cb9203177","originalId":["od______3989::7fcbe3a03280663cddebfd3cb9203177"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:25.652Z","dateoftransformation":"2020-03-03T13:06:55.95Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:25.652Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/796","datestamp":"2018-11-05T13:36:54Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Comunicación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Redes sociales","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Comunicación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Información","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Desinformación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"III Liga de Debate CICAE","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Instantaneidad","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Información periodística","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Fake news","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"5910.02 Medios de Comunicación de Masas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Jorge Gallardo - Liga de Debate CICAE","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Jorge Gallardo, profesor de Derecho en la Universidad Camilo José Cela (UCJC), Doctor en Comunicación Audiovisual y MBA en Empresas Audiovisuales. Subdirector de Espejo Público en Antena 3 Noticias. \n\nEn esta ocasión, Jorge Gallardo habla sobre las redes sociales, cómo han transformado la manera en la que nos comunicamos y nos informamos. \n\nIII Liga de Debate CICAE - UCJC http://ligacicae.ucjc.edu/\n\nLa Universidad Camilo José Cela (UCJC) y la Asociación de Colegios Privados e Independientes –Círculo de Calidad Educativa (CICAE) – organizan la tercera edición de esta gran liga de debate académico. El formato es al estilo inglés, similar al de la Oxford Union, donde algunos jóvenes se reunían a debatir sobre temas de la más candente actualidad y polemizaban con la intención de formarse en tan valoradas habilidades.\n\nLa Liga de Debate CICAE-UCJC es una ocasión única para los alumnos de ponerse en contacto con herramientas tan útiles como la oratoria y la comunicación efectiva. Pero, sobretodo, una oportunidad de fomentar el pensamiento crítico y la socialización entre alumnos con espíritu emprendedor. La Liga de Debate CICAE - Universidad Camilo José Cela se diferencia del resto de torneos en que potenciamos que sea eminentemente formativa. Alumnos y profesores reciben formación en comunicación, oratoria y debate por parte de alguno de los expertos en debate españoles más reconocidos en la escena internacional.\n\nDatos de la III Liga de Debate CICAE - UCJC http://ligacicae.ucjc.edu/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-10-23","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/796"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-10-23","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055872001,"id":"50|od______3989::d791339867bec6d3eb2104deeb4e4961","originalId":["od______3989::d791339867bec6d3eb2104deeb4e4961"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.586Z","dateoftransformation":"2020-03-03T13:06:57.721Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.586Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/645","datestamp":"2018-04-12T07:59:27Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Transferencia de Conocimiento e Innovación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Innovación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Investigación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Ciencia","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Tecnología","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"UCJC Open Science Day 2018","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"El UCJC Open Science Day tiene como objetivo mostrar las actividades de investigación llevadas a cabo por investigadores de la Universidad Camilo José Cela (UCJC) a todos los miembros de la UCJC.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-04-10","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/645"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-04-10","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} +{"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1585055872022,"id":"50|od______3989::d90d3a1f64ad264b5ebed8a35b280343","originalId":["od______3989::d90d3a1f64ad264b5ebed8a35b280343"],"collectedfrom":[{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null}],"pid":[],"dateofcollection":"2020-03-03T13:05:34.912Z","dateoftransformation":"2020-03-03T13:06:57.747Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2020-03-03T13:05:34.912Z","altered":true,"baseURL":"http%3A%2F%2Frepositorio.ucjc.edu%2Foai%2Frequest","identifier":"oai:repositorio.ucjc.edu:20.500.12020/795","datestamp":"2018-11-06T15:43:10Z","metadataNamespace":"http://www.openarchives.org/OAI/2.0/oai_dc/"}},"author":[{"fullname":"Universidad Camilo José Cela, Vicerrectorado de Innovación","name":"Vicerrectorado Innovación","surname":"Universidad Camilo José Cela","rank":1,"pid":null,"affiliation":null}],"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"language":{"classid":"esl/spa","classname":"Spanish","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Ciencias de la Comunicación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Unión Europea","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Desinformación","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Política","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Periodismo","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Ciudadanos europeos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Democracias","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Estado de derecho","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Derechos humanos","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"5910.02 Medios de Comunicación de Masas","qualifier":{"classid":"","classname":"","schemeid":"","schemename":""},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Ramón Luis Varcárcel - La Unión Europea ante el reto de la desinformación","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Conferencia de Ramón Luis Valcárcel, vicepresidente del Parlamento Europeo, en el Campus de Almagro de la Universidad Camilo José Cela (UCJC) bajo el título \"La Unión Europea ante el reto de la desinformación\", en la que ha alertado sobre el alcance de la desinformación en las democracias y sobre cómo pone en peligro los derechos de los ciudadanos europeos. También ha participado el rector de la UCJC, Samuel Martín-Barbero.","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2018-10-11","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":null,"embargoenddate":null,"source":[],"fulltext":[],"format":[],"contributor":[],"resourcetype":null,"coverage":[],"bestaccessright":null,"context":[],"externalReference":[],"instance":[{"license":{"value":"http://creativecommons.org/licenses/by-nc-nd/4.0/","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"instancetype":{"classid":"0024","classname":"Film","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"hostedby":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"url":["http://hdl.handle.net/20.500.12020/795"],"distributionlocation":"","collectedfrom":{"key":"opendoar____::3989","value":"Depósito Digital e-UCJC","dataInfo":null},"dateofacceptance":{"value":"2018-10-11","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"sysimport:crosswalk:repository","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":null}],"storagedate":null,"device":null,"size":null,"version":null,"lastmetadataupdate":null,"metadataversionnumber":null,"geolocation":[]} \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromproject/sample/otherresearchproduct/otherresearchproduct b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromproject/sample/otherresearchproduct/otherresearchproduct new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromproject/sample/publication/publication b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromproject/sample/publication/publication new file mode 100644 index 000000000..e69de29bb diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromproject/sample/software/software b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/resulttocommunityfromproject/sample/software/software new file mode 100644 index 000000000..e69de29bb From a94a54a2d0a9bdd9e6178f7f63b5369bd639d440 Mon Sep 17 00:00:00 2001 From: dimitrispie Date: Wed, 15 Nov 2023 14:32:18 +0200 Subject: [PATCH 103/148] Changes for tables and creation of the new indicator indi_is_result_accessible - Drop table statements for all tables to avoid duplicates in case of wf rerun - Add pdfsaggregated step to create the indi_is_result_accessible table. This step is executed on the new impala cluster only, since the pdfaggregation_i is updated on this cluster. --- .../oozie_app/copyDataToImpalaCluster.sh | 1 - .../stats/oozie_app/createPDFsAggregated.sh | 42 +++++ .../graph/stats/oozie_app/scripts/step10.sql | 2 + .../graph/stats/oozie_app/scripts/step11.sql | 1 + .../graph/stats/oozie_app/scripts/step12.sql | 10 ++ .../graph/stats/oozie_app/scripts/step13.sql | 17 +- .../graph/stats/oozie_app/scripts/step14.sql | 16 +- .../graph/stats/oozie_app/scripts/step15.sql | 12 ++ .../stats/oozie_app/scripts/step15_5.sql | 10 ++ .../scripts/step16-createIndicatorsTables.sql | 18 +- .../scripts/step16_1-definitions.sql | 6 + .../stats/oozie_app/scripts/step16_5.sql | 1 + .../graph/stats/oozie_app/scripts/step2.sql | 17 ++ .../scripts/step20-createMonitorDB.sql | 155 +++--------------- .../scripts/step20-createMonitorDBAll.sql | 85 +++------- .../scripts/step20-createMonitorDB_RIs.sql | 3 +- .../step20-createMonitorDB_RIs_tail.sql | 3 +- .../scripts/step20-createMonitorDB_funded.sql | 4 +- .../step20-createMonitorDB_institutions.sql | 4 +- .../scripts/step21-createObservatoryDB.sql | 33 ---- .../graph/stats/oozie_app/scripts/step7.sql | 6 + .../graph/stats/oozie_app/scripts/step8.sql | 11 ++ .../graph/stats/oozie_app/scripts/step9.sql | 2 + .../dhp/oa/graph/stats/oozie_app/workflow.xml | 17 ++ 24 files changed, 234 insertions(+), 242 deletions(-) create mode 100644 dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/createPDFsAggregated.sh diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh index 431978997..18ff6dca8 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh @@ -6,7 +6,6 @@ then ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder} fi -#export HADOOP_USER_NAME="dimitris.pierrakos" export HADOOP_USER_NAME=$6 export PROD_USAGE_STATS_DB="openaire_prod_usage_stats" function copydb() { diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/createPDFsAggregated.sh b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/createPDFsAggregated.sh new file mode 100644 index 000000000..46631a0c2 --- /dev/null +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/createPDFsAggregated.sh @@ -0,0 +1,42 @@ +export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs +export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami) +if ! [ -L $link_folder ] +then + rm -Rf "$link_folder" + ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder} +fi + +function createPDFsAggregated() { + db=$1 + +impala-shell --user $HADOOP_USER_NAME -i impala-cluster-dn1.openaire.eu -d ${db} -q "drop table if exists indi_is_result_accessible"; + +impala-shell --user $HADOOP_USER_NAME -i impala-cluster-dn1.openaire.eu -d ${db} -q "create table indi_is_result_accessible stored as parquet as + select distinct p.id, coalesce(is_result_accessible, 0) as is_result_accessible from result p + left outer join + (select id, 1 as is_result_accessible from (select pl.* from result r + join pdfaggregation_i.publication p on r.id=p.id + join pdfaggregation_i.payload pl on pl.id=p.id + union all + select pl.* from result r + join pdfaggregation_i.publication p on r.id=p.dedupid + join pdfaggregation_i.payload pl on pl.id=p.id) foo) tmp on p.id=tmp.id"; +} + +STATS_DB=$1 +MONITOR_DB=$2 +HADOOP_USER_NAME=$3 + +createPDFsAggregated $STATS_DB +createPDFsAggregated $MONITOR_DB + +createPDFsAggregated $MONITOR_DB'_funded' +createPDFsAggregated $MONITOR_DB'_institutions' +createPDFsAggregated $MONITOR_DB'_ris_tail' + +contexts="knowmad::other dh-ch::other enermaps::other gotriple::other neanias-atmospheric::other rural-digital-europe::other covid-19::other aurora::other neanias-space::other north-america-studies::other north-american-studies::other eutopia::other" +for i in ${contexts} +do + tmp=`echo "$i" | sed 's/'-'/'_'/g' | sed 's/'::'/'_'/g'` + createPDFsAggregated ${MONITOR_DB}'_'${tmp} +done \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step10.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step10.sql index 92dedf243..bbd7b3bbc 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step10.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step10.sql @@ -49,5 +49,7 @@ select * from openaire_prod_usage_stats.views_stats; -- Creation date of the database ------------------------------------------------------------------------------------------------ ------------------------------------------------------------------------------------------------ +DROP TABLE IF EXISTS ${stats_db_name}.creation_date purge; + create table ${stats_db_name}.creation_date STORED AS PARQUET as select date_format(current_date(), 'dd-MM-yyyy') as date; diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step11.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step11.sql index 41c3ed751..638fb0f7a 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step11.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step11.sql @@ -20,6 +20,7 @@ WHERE project_tmp.id IN (SELECT pr.id ${stats_db_name}.result r WHERE pr.result = r.id AND r.type = 'publication'); +DROP TABLE IF EXISTS ${stats_db_name}.stored purge; CREATE TABLE ${stats_db_name}.project stored as parquet as SELECT p.id, diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step12.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step12.sql index 47d147f75..0a1904de7 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step12.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step12.sql @@ -1,22 +1,32 @@ ------------------------------------------------------------------------------------------------------ -- Creating parquet tables from the updated temporary tables and removing unnecessary temporary tables ------------------------------------------------------------------------------------------------------ +DROP TABLE IF EXISTS ${stats_db_name}.datasource purge; + CREATE TABLE ${stats_db_name}.datasource stored AS parquet AS SELECT * FROM ${stats_db_name}.datasource_tmp; +DROP TABLE IF EXISTS ${stats_db_name}.publication purge; + CREATE TABLE ${stats_db_name}.publication stored AS parquet AS SELECT * FROM ${stats_db_name}.publication_tmp; +DROP TABLE IF EXISTS ${stats_db_name}.dataset purge; + CREATE TABLE ${stats_db_name}.dataset stored AS parquet AS SELECT * FROM ${stats_db_name}.dataset_tmp; +DROP TABLE IF EXISTS ${stats_db_name}.software purge; + CREATE TABLE ${stats_db_name}.software stored AS parquet AS SELECT * FROM ${stats_db_name}.software_tmp; +DROP TABLE IF EXISTS ${stats_db_name}.otherresearchproduct purge; + CREATE TABLE ${stats_db_name}.otherresearchproduct stored AS parquet AS SELECT * FROM ${stats_db_name}.otherresearchproduct_tmp; diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step13.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step13.sql index 24e1a1355..6493fa7d0 100755 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step13.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step13.sql @@ -5,6 +5,8 @@ -- Sources related tables/views ------------------------------------------------------ ------------------------------------------------------ +DROP TABLE IF EXISTS ${stats_db_name}.publication_sources purge; + CREATE TABLE IF NOT EXISTS ${stats_db_name}.publication_sources STORED AS PARQUET as SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource FROM ( @@ -16,6 +18,8 @@ LEFT OUTER JOIN from ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference=false and d.datainfo.invisible = FALSE) d on p.datasource = d.id; +DROP TABLE IF EXISTS ${stats_db_name}.dataset_sources purge; + CREATE TABLE IF NOT EXISTS ${stats_db_name}.dataset_sources STORED AS PARQUET as SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource FROM ( @@ -27,6 +31,8 @@ LEFT OUTER JOIN from ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference=false and d.datainfo.invisible = FALSE) d on p.datasource = d.id; +DROP TABLE IF EXISTS ${stats_db_name}.software_sources purge; + CREATE TABLE IF NOT EXISTS ${stats_db_name}.software_sources STORED AS PARQUET as SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource FROM ( @@ -38,6 +44,8 @@ LEFT OUTER JOIN from ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference=false and d.datainfo.invisible = FALSE) d on p.datasource = d.id; +DROP TABLE IF EXISTS ${stats_db_name}.otherresearchproduct_sources purge; + CREATE TABLE IF NOT EXISTS ${stats_db_name}.otherresearchproduct_sources STORED AS PARQUET as SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource FROM ( @@ -48,7 +56,7 @@ LEFT OUTER JOIN SELECT substr(d.id, 4) id from ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference=false and d.datainfo.invisible = FALSE) d on p.datasource = d.id; - + CREATE VIEW IF NOT EXISTS ${stats_db_name}.result_sources AS SELECT * FROM ${stats_db_name}.publication_sources UNION ALL @@ -58,6 +66,7 @@ SELECT * FROM ${stats_db_name}.software_sources UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_sources; +DROP TABLE IF EXISTS ${stats_db_name}.result_orcid purge; CREATE TABLE IF NOT EXISTS ${stats_db_name}.result_orcid STORED AS PARQUET as select distinct res.id, regexp_replace(res.orcid, 'http://orcid.org/' ,'') as orcid @@ -69,6 +78,8 @@ from ( LATERAL VIEW explode(auth.pid.qualifier.classid) apt as author_pid_type WHERE res.datainfo.deletedbyinference = FALSE and res.datainfo.invisible = FALSE and author_pid_type = 'orcid') as res; +DROP TABLE IF EXISTS ${stats_db_name}.result_result purge; + CREATE TABLE IF NOT EXISTS ${stats_db_name}.result_result stored as parquet as select substr(rel.source, 4) as source, substr(rel.target, 4) as target, relclass, subreltype from ${openaire_db_name}.relation rel @@ -82,6 +93,8 @@ where reltype='resultResult' and r2.resulttype.classname != 'other' and rel.datainfo.deletedbyinference=false and rel.datainfo.invisible = FALSE; +DROP TABLE IF EXISTS ${stats_db_name}.result_citations_oc purge; + CREATE TABLE IF NOT EXISTS ${stats_db_name}.result_citations_oc stored as parquet as select substr(target, 4) as id, count(distinct substr(source, 4)) as citations from ${openaire_db_name}.relation rel @@ -97,6 +110,8 @@ where relClass='Cites' and rel.datainfo.provenanceaction.classid = 'sysimport:cr and rel.datainfo.deletedbyinference=false and rel.datainfo.invisible = FALSE group by substr(target, 4); +DROP TABLE IF EXISTS ${stats_db_name}.result_references_oc purge; + CREATE TABLE IF NOT EXISTS ${stats_db_name}.result_references_oc stored as parquet as select substr(source, 4) as id, count(distinct substr(target, 4)) as references from ${openaire_db_name}.relation rel diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step14.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step14.sql index 39755d68e..f50c13521 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step14.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step14.sql @@ -5,21 +5,29 @@ -- Licences related tables/views ------------------------------------------------------ ------------------------------------------------------ +DROP TABLE IF EXISTS ${stats_db_name}.publication_licenses purge; + CREATE TABLE IF NOT EXISTS ${stats_db_name}.publication_licenses STORED AS PARQUET AS -SELECT substr(p.id, 4) as id, licenses.value as type +SELECT substr(p.id, 4) as id, licenses.value as type from ${openaire_db_name}.publication p LATERAL VIEW explode(p.instance.license) instances as licenses where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false and p.datainfo.invisible = FALSE; +DROP TABLE IF EXISTS ${stats_db_name}.dataset_licenses purge; + CREATE TABLE IF NOT EXISTS ${stats_db_name}.dataset_licenses STORED AS PARQUET AS SELECT substr(p.id, 4) as id, licenses.value as type from ${openaire_db_name}.dataset p LATERAL VIEW explode(p.instance.license) instances as licenses where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false and p.datainfo.invisible = FALSE; +DROP TABLE IF EXISTS ${stats_db_name}.software_licenses purge; + CREATE TABLE IF NOT EXISTS ${stats_db_name}.software_licenses STORED AS PARQUET AS SELECT substr(p.id, 4) as id, licenses.value as type from ${openaire_db_name}.software p LATERAL VIEW explode(p.instance.license) instances as licenses where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false and p.datainfo.invisible = FALSE; +DROP TABLE IF EXISTS ${stats_db_name}.otherresearchproduct_licenses purge; + CREATE TABLE IF NOT EXISTS ${stats_db_name}.otherresearchproduct_licenses STORED AS PARQUET AS SELECT substr(p.id, 4) as id, licenses.value as type from ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.instance.license) instances as licenses @@ -34,10 +42,14 @@ SELECT * FROM ${stats_db_name}.software_licenses UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_licenses; +DROP TABLE IF EXISTS ${stats_db_name}.organization_pids purge; + CREATE TABLE IF NOT EXISTS ${stats_db_name}.organization_pids STORED AS PARQUET AS select substr(o.id, 4) as id, ppid.qualifier.classname as type, ppid.value as pid from ${openaire_db_name}.organization o lateral view explode(o.pid) pids as ppid; +DROP TABLE IF EXISTS ${stats_db_name}.organization_sources purge; + CREATE TABLE IF NOT EXISTS ${stats_db_name}.organization_sources STORED AS PARQUET as SELECT o.id, case when d.id is null then 'other' else o.datasource end as datasource FROM ( @@ -48,6 +60,8 @@ FROM ( from ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference=false and d.datainfo.invisible = FALSE) d on o.datasource = d.id; +DROP TABLE IF EXISTS ${stats_db_name}.result_accessroute purge; + CREATE TABLE IF NOT EXISTS ${stats_db_name}.result_accessroute STORED AS PARQUET as select distinct substr(id,4) as id, accessroute from ${openaire_db_name}.result lateral view explode (instance.accessright.openaccessroute) openaccessroute as accessroute; diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql index 4a8f81943..066b197e6 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql @@ -6,21 +6,29 @@ ------------------------------------------------------ ------------------------------------------------------ +DROP TABLE IF EXISTS ${stats_db_name}.publication_refereed purge; + CREATE TABLE IF NOT EXISTS ${stats_db_name}.publication_refereed STORED AS PARQUET as select substr(r.id, 4) as id, inst.refereed.classname as refereed from ${openaire_db_name}.publication r lateral view explode(r.instance) instances as inst where r.datainfo.deletedbyinference=false and r.datainfo.invisible = FALSE; +DROP TABLE IF EXISTS ${stats_db_name}.dataset_refereed purge; + CREATE TABLE IF NOT EXISTS ${stats_db_name}.dataset_refereed STORED AS PARQUET as select substr(r.id, 4) as id, inst.refereed.classname as refereed from ${openaire_db_name}.dataset r lateral view explode(r.instance) instances as inst where r.datainfo.deletedbyinference=false and r.datainfo.invisible = FALSE; +DROP TABLE IF EXISTS ${stats_db_name}.software_refereed purge; + CREATE TABLE IF NOT EXISTS ${stats_db_name}.software_refereed STORED AS PARQUET as select substr(r.id, 4) as id, inst.refereed.classname as refereed from ${openaire_db_name}.software r lateral view explode(r.instance) instances as inst where r.datainfo.deletedbyinference=false and r.datainfo.invisible = FALSE; +DROP TABLE IF EXISTS ${stats_db_name}.otherresearchproduct_refereed purge; + CREATE TABLE IF NOT EXISTS ${stats_db_name}.otherresearchproduct_refereed STORED AS PARQUET as select substr(r.id, 4) as id, inst.refereed.classname as refereed from ${openaire_db_name}.otherresearchproduct r lateral view explode(r.instance) instances as inst @@ -35,12 +43,16 @@ select * from ${stats_db_name}.software_refereed union all select * from ${stats_db_name}.otherresearchproduct_refereed; +DROP TABLE IF EXISTS ${stats_db_name}.indi_impact_measures purge; + create table if not exists ${stats_db_name}.indi_impact_measures STORED AS PARQUET as select substr(id, 4) as id, measures_ids.id impactmetric, cast(measures_ids.unit.value[0] as double) score, cast(measures_ids.unit.value[0] as decimal(6,3)) score_dec, measures_ids.unit.value[1] impact_class from ${openaire_db_name}.result lateral view explode(measures) measures as measures_ids where measures_ids.id!='views' and measures_ids.id!='downloads'; +DROP TABLE IF EXISTS ${stats_db_name}.result_apc_affiliations purge; + create table if not exists ${stats_db_name}.result_apc_affiliations STORED AS PARQUET as select distinct substr(rel.target,4) id, substr(rel.source,4) organization, o.legalname.value name, cast(rel.properties[0].value as double) apc_amount, diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15_5.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15_5.sql index 615f523ce..2c606fb92 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15_5.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15_5.sql @@ -1,6 +1,8 @@ ------------------------------------------- --- Extra tables, mostly used by indicators +DROP TABLE IF EXISTS ${stats_db_name}.result_projectcount purge; + create table if not exists ${stats_db_name}.result_projectcount STORED AS PARQUET as select r.id, count(distinct p.id) as count from ${stats_db_name}.result r @@ -8,6 +10,8 @@ left outer join ${stats_db_name}.result_projects rp on rp.id=r.id left outer join ${stats_db_name}.project p on p.id=rp.project group by r.id; +DROP TABLE IF EXISTS ${stats_db_name}.result_fundercount purge; + create table if not exists ${stats_db_name}.result_fundercount STORED AS PARQUET as select r.id, count(distinct p.funder) as count from ${stats_db_name}.result r @@ -15,6 +19,8 @@ left outer join ${stats_db_name}.result_projects rp on rp.id=r.id left outer join ${stats_db_name}.project p on p.id=rp.project group by r.id; +DROP TABLE IF EXISTS ${stats_db_name}.project_resultcount purge; + create table if not exists ${stats_db_name}.project_resultcount STORED AS PARQUET as with rcount as ( select p.id as pid, count(distinct r.id) as `count`, r.type as type @@ -37,6 +43,8 @@ create or replace view ${stats_db_name}.totalresearchersft as select * from stat create or replace view ${stats_db_name}.hrrst as select * from stats_ext.hrrst; create or replace view ${stats_db_name}.graduatedoctorates as select * from stats_ext.graduatedoctorates; +DROP TABLE IF EXISTS ${stats_db_name}.result_instance purge; + create table if not exists ${stats_db_name}.result_instance stored as parquet as select distinct r.* from ( @@ -45,6 +53,8 @@ from ( from ${openaire_db_name}.result r lateral view explode(r.instance) instances as inst lateral view explode(inst.pid) pids as p) r join ${stats_db_name}.result res on res.id=r.id; +DROP TABLE IF EXISTS ${stats_db_name}.result_apc purge; + create table if not exists ${stats_db_name}.result_apc STORED AS PARQUET as select r.id, r.amount, r.currency from ( diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql index 6af486340..8180e6527 100755 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql @@ -104,7 +104,7 @@ from ${stats_db_name}.tmp as o1 join ${stats_db_name}.tmp as o2 where o1.id=o2.id and o1.organization!=o2.organization and o1.name!=o2.name group by o1.organization, o2.organization, o1.name, o2.name; -drop table ${stats_db_name}.tmp purge; +drop table if exists ${stats_db_name}.tmp purge; create TEMPORARY TABLE ${stats_db_name}.tmp AS select distinct ro.organization organization, ro.id, o.name, o.country from ${stats_db_name}.result_organization ro @@ -118,7 +118,7 @@ from ${stats_db_name}.tmp as o1 join ${stats_db_name}.tmp as o2 on o1.id=o2.id where o1.id=o2.id and o1.country!=o2.country group by o1.organization, o1.id, o1.name, o2.country; -drop table ${stats_db_name}.tmp purge; +drop table if exists ${stats_db_name}.tmp purge; create TEMPORARY TABLE ${stats_db_name}.tmp AS select o.id organization, o.name, ro.project as project from ${stats_db_name}.organization o @@ -133,7 +133,7 @@ from ${stats_db_name}.tmp as o1 where o1.organization<>o2.organization and o1.name<>o2.name group by o1.name,o2.name, o1.organization, o2.organization; -drop table ${stats_db_name}.tmp purge; +drop table if exists ${stats_db_name}.tmp purge; create TEMPORARY TABLE ${stats_db_name}.tmp AS select o.id organization, o.name, o.country , ro.project as project from ${stats_db_name}.organization o @@ -149,7 +149,7 @@ from ${stats_db_name}.tmp as o1 where o1.organization<>o2.organization and o1.country<>o2.country group by o1.organization, o2.country, o1.name; -drop table ${stats_db_name}.tmp purge; +drop table if exists ${stats_db_name}.tmp purge; drop table if exists ${stats_db_name}.indi_funder_country_collab purge; @@ -178,7 +178,7 @@ from ${stats_db_name}.tmp as o1 where o1.country<>o2.country group by o1.country, o2.country; -drop table ${stats_db_name}.tmp purge; +drop table if exists ${stats_db_name}.tmp purge; ---- Sprint 4 ---- drop table if exists ${stats_db_name}.indi_pub_diamond purge; @@ -422,7 +422,7 @@ drop table if exists ${stats_db_name}.indi_pub_hybrid purge; -- on pd.id=tmp.id; create table if not exists ${stats_db_name}.indi_pub_hybrid stored as parquet as -select pd.id,coalesce(is_hybrid,0) is_hybrid from ${stats_db_name}.publication_datasources pd +select distinct pd.id,coalesce(is_hybrid,0) is_hybrid from ${stats_db_name}.publication_datasources pd left outer join (select pd.id, 1 as is_hybrid from ${stats_db_name}.publication_datasources pd join ${stats_db_name}.datasource d on pd.datasource=d.id join ${stats_db_name}.result_instance ri on ri.id=pd.id @@ -492,7 +492,7 @@ CREATE TEMPORARY TABLE ${stats_db_name}.allresults as select year, ro.organizati drop table if exists ${stats_db_name}.indi_org_fairness_pub_year purge; create table if not exists ${stats_db_name}.indi_org_fairness_pub_year stored as parquet as -select allresults.year, allresults.organization, result_fair.no_result_fair/allresults.no_allresults org_fairness +select cast(allresults.year as int) year, allresults.organization, result_fair.no_result_fair/allresults.no_allresults org_fairness from ${stats_db_name}.allresults join ${stats_db_name}.result_fair on result_fair.organization=allresults.organization and result_fair.year=allresults.year; @@ -813,8 +813,8 @@ drop table if exists ${stats_db_name}.indi_pub_bronze_oa purge; --and (d.type='Journal' or d.type='Journal Aggregator/Publisher') --and ri.accessright='Open Access') tmp on tmp.id=p.id; -create table ${stats_db_name}.indi_pub_bronze stored as parquet as -select pd.id,coalesce(is_bronze_oa,0) is_bronze_oa from ${stats_db_name}.publication_datasources pd +create table ${stats_db_name}.indi_pub_bronze_oa stored as parquet as +select distinct pd.id,coalesce(is_bronze_oa,0) is_bronze_oa from ${stats_db_name}.publication_datasources pd left outer join (select pd.id, 1 as is_bronze_oa from ${stats_db_name}.publication_datasources pd join ${stats_db_name}.datasource d on pd.datasource=d.id join ${stats_db_name}.result_instance ri on ri.id=pd.id diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16_1-definitions.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16_1-definitions.sql index 41c95758c..b55af13d4 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16_1-definitions.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16_1-definitions.sql @@ -3,6 +3,8 @@ ---------------------------------------------------- -- Peer reviewed: +drop table if exists ${stats_db_name}.result_peerreviewed purge; + create table IF NOT EXISTS ${stats_db_name}.result_peerreviewed STORED AS PARQUET as select r.id as id, case when doi.doi_from_crossref=1 and grey.grey_lit=0 then true else false end as peer_reviewed from ${stats_db_name}.result r @@ -10,12 +12,16 @@ left outer join ${stats_db_name}.indi_pub_doi_from_crossref doi on doi.id=r.id left outer join ${stats_db_name}.indi_pub_grey_lit grey on grey.id=r.id; -- Green OA: +drop table if exists ${stats_db_name}.result_greenoa purge; + create table IF NOT EXISTS ${stats_db_name}.result_greenoa STORED AS PARQUET as select r.id, case when green.green_oa=1 then true else false end as green from ${stats_db_name}.result r left outer join ${stats_db_name}.indi_pub_green_oa green on green.id=r.id; -- GOLD OA: +drop table if exists ${stats_db_name}.result_gold purge; + create table IF NOT EXISTS ${stats_db_name}.result_gold STORED AS PARQUET as select r.id, case when gold.is_gold=1 then true else false end as gold from ${stats_db_name}.result r diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16_5.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16_5.sql index f737c1ea6..7faa91697 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16_5.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16_5.sql @@ -1,6 +1,7 @@ -- replace the creation of the result view to include the boolean fields from the previous tables (green, gold, -- peer reviewed) drop table if exists ${stats_db_name}.result_tmp; + CREATE TABLE ${stats_db_name}.result_tmp ( id STRING, title STRING, diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step2.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step2.sql index 4ffbd384b..8e56f98fc 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step2.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step2.sql @@ -5,6 +5,7 @@ -------------------------------------------------------------- -- Publication temporary table +DROP TABLE IF EXISTS ${stats_db_name}.publication_tmp purge; CREATE TABLE ${stats_db_name}.publication_tmp ( id STRING, @@ -40,12 +41,16 @@ SELECT substr(p.id, 4) as id, from ${openaire_db_name}.publication p where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.publication_classifications purge; + CREATE TABLE ${stats_db_name}.publication_classifications STORED AS PARQUET AS SELECT substr(p.id, 4) as id, instancetype.classname as type from ${openaire_db_name}.publication p LATERAL VIEW explode(p.instance.instancetype) instances as instancetype where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.publication_concepts purge; + CREATE TABLE ${stats_db_name}.publication_concepts STORED AS PARQUET AS SELECT substr(p.id, 4) as id, case when contexts.context.id RLIKE '^[^::]+::[^::]+::.+$' then contexts.context.id @@ -55,6 +60,8 @@ from ${openaire_db_name}.publication p LATERAL VIEW explode(p.context) contexts as context where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.publication_datasources purge; + CREATE TABLE ${stats_db_name}.publication_datasources STORED AS PARQUET as SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource FROM ( @@ -66,29 +73,39 @@ FROM ( from ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference = false and d.datainfo.invisible=false) d on p.datasource = d.id; +DROP TABLE IF EXISTS ${stats_db_name}.publication_languages purge; + CREATE TABLE ${stats_db_name}.publication_languages STORED AS PARQUET AS select substr(p.id, 4) as id, p.language.classname as language FROM ${openaire_db_name}.publication p where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.publication_oids purge; + CREATE TABLE ${stats_db_name}.publication_oids STORED AS PARQUET AS SELECT substr(p.id, 4) AS id, oids.ids AS oid FROM ${openaire_db_name}.publication p LATERAL VIEW explode(p.originalid) oids AS ids where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.publication_pids purge; + CREATE TABLE ${stats_db_name}.publication_pids STORED AS PARQUET AS SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value as pid FROM ${openaire_db_name}.publication p LATERAL VIEW explode(p.pid) pids AS ppid where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.publication_topics purge; + CREATE TABLE ${stats_db_name}.publication_topics STORED AS PARQUET as select substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS TYPE, subjects.subject.value AS topic FROM ${openaire_db_name}.publication p LATERAL VIEW explode(p.subject) subjects AS subject where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.publication_citations purge; + CREATE TABLE ${stats_db_name}.publication_citations STORED AS PARQUET AS SELECT substr(p.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS cites FROM ${openaire_db_name}.publication p diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql index d5d242230..b52abd865 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql @@ -1,79 +1,3 @@ ---drop database if exists TARGET cascade; ---create database if not exists TARGET; --- ---create view if not exists TARGET.category as select * from SOURCE.category; ---create view if not exists TARGET.concept as select * from SOURCE.concept; ---create view if not exists TARGET.context as select * from SOURCE.context; ---create view if not exists TARGET.country as select * from SOURCE.country; ---create view if not exists TARGET.countrygdp as select * from SOURCE.countrygdp; ---create view if not exists TARGET.creation_date as select * from SOURCE.creation_date; ---create view if not exists TARGET.funder as select * from SOURCE.funder; ---create view if not exists TARGET.fundref as select * from SOURCE.fundref; ---create view if not exists TARGET.rndexpenditure as select * from SOURCE.rndexpediture; ---create view if not exists TARGET.rndgdpexpenditure as select * from SOURCE.rndgdpexpenditure; ---create view if not exists TARGET.doctoratestudents as select * from SOURCE.doctoratestudents; ---create view if not exists TARGET.totalresearchers as select * from SOURCE.totalresearchers; ---create view if not exists TARGET.totalresearchersft as select * from SOURCE.totalresearchersft; ---create view if not exists TARGET.hrrst as select * from SOURCE.hrrst; --- ---create table TARGET.result stored as parquet as --- select distinct * from ( --- select * from SOURCE.result r where exists (select 1 from SOURCE.result_projects rp join SOURCE.project p on rp.project=p.id where rp.id=r.id) --- union all --- select * from SOURCE.result r where exists (select 1 from SOURCE.result_concepts rc where rc.id=r.id) --- union all --- select * from SOURCE.result r where exists (select 1 from SOURCE.result_organization ro where ro.id=r.id and ro.organization in ( --- 'openorgs____::b84450f9864182c67b8611b5593f4250', --"Athena Research and Innovation Center In Information Communication & Knowledge Technologies', --ARC" --- 'openorgs____::d41cf6bd4ab1b1362a44397e0b95c975', --National Research Council --- 'openorgs____::d2a09b9d5eabb10c95f9470e172d05d2', --??? Not exists ?? --- 'openorgs____::d169c7407dd417152596908d48c11460', --Masaryk University --- 'openorgs____::1ec924b1759bb16d0a02f2dad8689b21', --University of Belgrade --- 'openorgs____::0ae431b820e4c33db8967fbb2b919150', --University of Helsinki --- 'openorgs____::759d59f05d77188faee99b7493b46805', --University of Minho --- 'openorgs____::cad284878801b9465fa51a95b1d779db', --Universidad Politécnica de Madrid --- 'openorgs____::eadc8da90a546e98c03f896661a2e4d4', --University of Göttingen --- 'openorgs____::c0286313e36479eff8676dba9b724b40', --National and Kapodistrian University of Athens --- -- 'openorgs____::c80a8243a5e5c620d7931c88d93bf17a', --Université Paris Diderot --- 'openorgs____::c08634f0a6b0081c3dc6e6c93a4314f3', --Bielefeld University --- 'openorgs____::6fc85e4a8f7ecaf4b0c738d010e967ea', --University of Southern Denmark --- 'openorgs____::3d6122f87f9a97a99d8f6e3d73313720', --Humboldt-Universität zu Berlin --- 'openorgs____::16720ada63d0fa8ca41601feae7d1aa5', --TU Darmstadt --- 'openorgs____::ccc0a066b56d2cfaf90c2ae369df16f5', --KU Leuven --- 'openorgs____::4c6f119632adf789746f0a057ed73e90', --University of the Western Cape --- 'openorgs____::ec3665affa01aeafa28b7852c4176dbd', --Rudjer Boskovic Institute --- 'openorgs____::5f31346d444a7f06a28c880fb170b0f6', --Ghent University --- 'openorgs____::2dbe47117fd5409f9c61620813456632', --University of Luxembourg --- 'openorgs____::6445d7758d3a40c4d997953b6632a368', --National Institute of Informatics (NII) --- 'openorgs____::b77c01aa15de3675da34277d48de2ec1', -- Valencia Catholic University Saint Vincent Martyr --- 'openorgs____::7fe2f66cdc43983c6b24816bfe9cf6a0', -- Unviersity of Warsaw --- 'openorgs____::15e7921fc50d9aa1229a82a84429419e', -- University Of Thessaly --- 'openorgs____::11f7919dadc8f8a7251af54bba60c956', -- Technical University of Crete --- 'openorgs____::84f0c5f5dbb6daf42748485924efde4b', -- University of Piraeus --- 'openorgs____::4ac562f0376fce3539504567649cb373', -- University of Patras --- 'openorgs____::3e8d1f8c3f6cd7f418b09f1f58b4873b', -- Aristotle University of Thessaloniki --- 'openorgs____::3fcef6e1c469c10f2a84b281372c9814', -- World Bank --- 'openorgs____::1698a2eb1885ef8adb5a4a969e745ad3', -- École des Ponts ParisTech --- 'openorgs____::e15adb13c4dadd49de4d35c39b5da93a', -- Nanyang Technological University --- 'openorgs____::4b34103bde246228fcd837f5f1bf4212', -- Autonomous University of Barcelona --- 'openorgs____::72ec75fcfc4e0df1a76dc4c49007fceb', -- McMaster University --- 'openorgs____::51c7fc556e46381734a25a6fbc3fd398', -- University of Modena and Reggio Emilia --- 'openorgs____::235d7f9ad18ecd7e6dc62ea4990cb9db', -- Bilkent University --- 'openorgs____::31f2fa9e05b49d4cf40a19c3fed8eb06', -- Saints Cyril and Methodius University of Skopje --- 'openorgs____::db7686f30f22cbe73a4fde872ce812a6', -- University of Milan --- 'openorgs____::b8b8ca674452579f3f593d9f5e557483', -- University College Cork --- 'openorgs____::38d7097854736583dde879d12dacafca' -- Brown University --- 'openorgs____::57784c9e047e826fefdb1ef816120d92', --Arts et Métiers ParisTech --- 'openorgs____::2530baca8a15936ba2e3297f2bce2e7e', -- University of Cape Town --- 'openorgs____::d11f981828c485cd23d93f7f24f24db1', -- Technological University Dublin --- 'openorgs____::5e6bf8962665cdd040341171e5c631d8', -- Delft University of Technology --- 'openorgs____::846cb428d3f52a445f7275561a7beb5d', -- University of Manitoba --- 'openorgs____::eb391317ed0dc684aa81ac16265de041', -- Universitat Rovira i Virgili --- 'openorgs____::66aa9fc2fceb271423dfabcc38752dc0', -- Lund University --- 'openorgs____::3cff625a4370d51e08624cc586138b2f' -- IMT Atlantique --- ) )) foo; --- ---ANALYZE TABLE TARGET.result COMPUTE STATISTICS; - create view if not exists TARGET.category as select * from SOURCE.category; create view if not exists TARGET.concept as select * from SOURCE.concept; create view if not exists TARGET.context as select * from SOURCE.context; @@ -91,76 +15,52 @@ create view if not exists TARGET.hrrst as select * from SOURCE.hrrst; create view if not exists TARGET.graduatedoctorates as select * from SOURCE.graduatedoctorates; create table TARGET.result_citations stored as parquet as select * from SOURCE.result_citations orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_citations COMPUTE STATISTICS; create table TARGET.result_references_oc stored as parquet as select * from SOURCE.result_references_oc orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_references_oc COMPUTE STATISTICS; create table TARGET.result_citations_oc stored as parquet as select * from SOURCE.result_citations_oc orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_citations_oc COMPUTE STATISTICS; create table TARGET.result_classifications stored as parquet as select * from SOURCE.result_classifications orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_classifications COMPUTE STATISTICS; create table TARGET.result_apc stored as parquet as select * from SOURCE.result_apc orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_apc COMPUTE STATISTICS; create table TARGET.result_concepts stored as parquet as select * from SOURCE.result_concepts orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_concepts COMPUTE STATISTICS; create table TARGET.result_datasources stored as parquet as select * from SOURCE.result_datasources orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_datasources COMPUTE STATISTICS; create table TARGET.result_fundercount stored as parquet as select * from SOURCE.result_fundercount orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_fundercount COMPUTE STATISTICS; create table TARGET.result_gold stored as parquet as select * from SOURCE.result_gold orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_gold COMPUTE STATISTICS; create table TARGET.result_greenoa stored as parquet as select * from SOURCE.result_greenoa orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_greenoa COMPUTE STATISTICS; create table TARGET.result_languages stored as parquet as select * from SOURCE.result_languages orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_languages COMPUTE STATISTICS; create table TARGET.result_licenses stored as parquet as select * from SOURCE.result_licenses orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_licenses COMPUTE STATISTICS; create table TARGET.licenses_normalized STORED AS PARQUET as select * from SOURCE.licenses_normalized; ---ANALYZE TABLE TARGET.licenses_normalized COMPUTE STATISTICS; create table TARGET.result_oids stored as parquet as select * from SOURCE.result_oids orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_oids COMPUTE STATISTICS; create table TARGET.result_organization stored as parquet as select * from SOURCE.result_organization orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_organization COMPUTE STATISTICS; create table TARGET.result_peerreviewed stored as parquet as select * from SOURCE.result_peerreviewed orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_peerreviewed COMPUTE STATISTICS; create table TARGET.result_pids stored as parquet as select * from SOURCE.result_pids orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_pids COMPUTE STATISTICS; create table TARGET.result_projectcount stored as parquet as select * from SOURCE.result_projectcount orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_projectcount COMPUTE STATISTICS; create table TARGET.result_projects stored as parquet as select * from SOURCE.result_projects orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_projects COMPUTE STATISTICS; create table TARGET.result_refereed stored as parquet as select * from SOURCE.result_refereed orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_refereed COMPUTE STATISTICS; create table TARGET.result_sources stored as parquet as select * from SOURCE.result_sources orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_sources COMPUTE STATISTICS; create table TARGET.result_topics stored as parquet as select * from SOURCE.result_topics orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_topics COMPUTE STATISTICS; create table TARGET.result_fos stored as parquet as select * from SOURCE.result_fos orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_fos COMPUTE STATISTICS; create table TARGET.result_accessroute stored as parquet as select * from SOURCE.result_accessroute orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_accessroute COMPUTE STATISTICS; create table TARGET.result_orcid stored as parquet as select * from SOURCE.result_orcid orig where exists (select 1 from TARGET.result r where r.id=orig.id); @@ -169,7 +69,6 @@ create view TARGET.foo2 as select * from SOURCE.result_result rr where rr.target create table TARGET.result_result STORED AS PARQUET as select distinct * from (select * from TARGET.foo1 union all select * from TARGET.foo2) foufou; drop view TARGET.foo1; drop view TARGET.foo2; ---ANALYZE TABLE TARGET.result_result COMPUTE STATISTICS; -- datasources create view if not exists TARGET.datasource as select * from SOURCE.datasource; @@ -178,7 +77,6 @@ create view if not exists TARGET.datasource_organizations as select * from SOURC create view if not exists TARGET.datasource_sources as select * from SOURCE.datasource_sources; create table TARGET.datasource_results stored as parquet as select id as result, datasource as id from TARGET.result_datasources; ---ANALYZE TABLE TARGET.datasource_results COMPUTE STATISTICS; -- organizations create view if not exists TARGET.organization as select * from SOURCE.organization; @@ -196,28 +94,27 @@ create view if not exists TARGET.project_classification as select * from SOURCE. create view if not exists TARGET.project_organization_contribution as select * from SOURCE.project_organization_contribution; create table TARGET.project_results stored as parquet as select id as result, project as id from TARGET.result_projects; ---ANALYZE TABLE TARGET.project_results COMPUTE STATISTICS; -- indicators -- Sprint 1 ---- create table TARGET.indi_pub_green_oa stored as parquet as select * from SOURCE.indi_pub_green_oa orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_green_oa COMPUTE STATISTICS; + create table TARGET.indi_pub_grey_lit stored as parquet as select * from SOURCE.indi_pub_grey_lit orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_grey_lit COMPUTE STATISTICS; + create table TARGET.indi_pub_doi_from_crossref stored as parquet as select * from SOURCE.indi_pub_doi_from_crossref orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_doi_from_crossref COMPUTE STATISTICS; + -- Sprint 2 ---- create table TARGET.indi_result_has_cc_licence stored as parquet as select * from SOURCE.indi_result_has_cc_licence orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_result_has_cc_licence COMPUTE STATISTICS; + create table TARGET.indi_result_has_cc_licence_url stored as parquet as select * from SOURCE.indi_result_has_cc_licence_url orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_result_has_cc_licence_url COMPUTE STATISTICS; + create table TARGET.indi_pub_has_abstract stored as parquet as select * from SOURCE.indi_pub_has_abstract orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_has_abstract COMPUTE STATISTICS; + create table TARGET.indi_result_with_orcid stored as parquet as select * from SOURCE.indi_result_with_orcid orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_result_with_orcid COMPUTE STATISTICS; + ---- Sprint 3 ---- create table TARGET.indi_funded_result_with_fundref stored as parquet as select * from SOURCE.indi_funded_result_with_fundref orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_funded_result_with_fundref COMPUTE STATISTICS; + create view TARGET.indi_result_org_collab as select * from SOURCE.indi_result_org_collab; create view TARGET.indi_result_org_country_collab as select * from SOURCE.indi_result_org_country_collab; create view TARGET.indi_project_collab_org as select * from SOURCE.indi_project_collab_org; @@ -226,32 +123,32 @@ create view TARGET.indi_funder_country_collab as select * from SOURCE.indi_funde create view TARGET.indi_result_country_collab as select * from SOURCE.indi_result_country_collab; ---- Sprint 4 ---- create table TARGET.indi_pub_diamond stored as parquet as select * from SOURCE.indi_pub_diamond orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_diamond COMPUTE STATISTICS; + create table TARGET.indi_pub_in_transformative stored as parquet as select * from SOURCE.indi_pub_in_transformative orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_in_transformative COMPUTE STATISTICS; + create table TARGET.indi_pub_closed_other_open stored as parquet as select * from SOURCE.indi_pub_closed_other_open orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_closed_other_open COMPUTE STATISTICS; + ---- Sprint 5 ---- create table TARGET.indi_result_no_of_copies stored as parquet as select * from SOURCE.indi_result_no_of_copies orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_result_no_of_copies COMPUTE STATISTICS; + ---- Sprint 6 ---- create table TARGET.indi_pub_hybrid_oa_with_cc stored as parquet as select * from SOURCE.indi_pub_hybrid_oa_with_cc orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_hybrid_oa_with_cc COMPUTE STATISTICS; + create table TARGET.indi_pub_bronze_oa stored as parquet as select * from SOURCE.indi_pub_bronze_oa orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_bronze_oa COMPUTE STATISTICS; + create table TARGET.indi_pub_downloads stored as parquet as select * from SOURCE.indi_pub_downloads orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); ---ANALYZE TABLE TARGET.indi_pub_downloads COMPUTE STATISTICS; + create table TARGET.indi_pub_downloads_datasource stored as parquet as select * from SOURCE.indi_pub_downloads_datasource orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); ---ANALYZE TABLE TARGET.indi_pub_downloads_datasource COMPUTE STATISTICS; + create table TARGET.indi_pub_downloads_year stored as parquet as select * from SOURCE.indi_pub_downloads_year orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); ---ANALYZE TABLE TARGET.indi_pub_downloads_year COMPUTE STATISTICS; + create table TARGET.indi_pub_downloads_datasource_year stored as parquet as select * from SOURCE.indi_pub_downloads_datasource_year orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); ---ANALYZE TABLE TARGET.indi_pub_downloads_datasource_year COMPUTE STATISTICS; + ---- Sprint 7 ---- create table TARGET.indi_pub_gold_oa stored as parquet as select * from SOURCE.indi_pub_gold_oa orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_gold_oa COMPUTE STATISTICS; + create table TARGET.indi_pub_hybrid stored as parquet as select * from SOURCE.indi_pub_hybrid orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_hybrid COMPUTE STATISTICS; + create view TARGET.indi_org_fairness as select * from SOURCE.indi_org_fairness; create view TARGET.indi_org_fairness_pub_pr as select * from SOURCE.indi_org_fairness_pub_pr; create view TARGET.indi_org_fairness_pub_year as select * from SOURCE.indi_org_fairness_pub_year; @@ -262,17 +159,17 @@ create view TARGET.indi_org_findable as select * from SOURCE.indi_org_findable; create view TARGET.indi_org_openess as select * from SOURCE.indi_org_openess; create view TARGET.indi_org_openess_year as select * from SOURCE.indi_org_openess_year; create table TARGET.indi_pub_has_preprint stored as parquet as select * from SOURCE.indi_pub_has_preprint orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_has_preprint COMPUTE STATISTICS; + create table TARGET.indi_pub_in_subscribed stored as parquet as select * from SOURCE.indi_pub_in_subscribed orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_in_subscribed COMPUTE STATISTICS; + create table TARGET.indi_result_with_pid stored as parquet as select * from SOURCE.indi_result_with_pid orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_result_with_pid COMPUTE STATISTICS; + create table TARGET.indi_impact_measures stored as parquet as select * from SOURCE.indi_impact_measures orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_impact_measures COMPUTE STATISTICS; + create table TARGET.indi_pub_interdisciplinarity stored as parquet as select * from SOURCE.indi_pub_interdisciplinarity orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_interdisciplinarity COMPUTE STATISTICS; + create table TARGET.result_apc_affiliations stored as parquet as select * from SOURCE.result_apc_affiliations orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_apc_affiliations COMPUTE STATISTICS; + create table TARGET.indi_is_project_result_after stored as parquet as select * from SOURCE.indi_is_project_result_after orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); create table TARGET.indi_is_funder_plan_s stored as parquet as select * from SOURCE.indi_is_funder_plan_s orig where exists (select 1 from TARGET.result r where r.id=orig.id); create view TARGET.indi_funder_fairness as select * from SOURCE.indi_funder_fairness; diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql index df4795e3e..2b6a68514 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql @@ -81,8 +81,6 @@ create table TARGET.result stored as parquet as 'openorgs____::8839b55dae0c84d56fd533f52d5d483a' -- Leibniz Institute of Ecological Urban and Regional Development ) )) foo; ---ANALYZE TABLE TARGET.result COMPUTE STATISTICS; - create view if not exists TARGET.category as select * from SOURCE.category; create view if not exists TARGET.concept as select * from SOURCE.concept; create view if not exists TARGET.context as select * from SOURCE.context; @@ -97,86 +95,63 @@ create view if not exists TARGET.doctoratestudents as select * from SOURCE.docto create view if not exists TARGET.totalresearchers as select * from SOURCE.totalresearchers; create view if not exists TARGET.totalresearchersft as select * from SOURCE.totalresearchersft; create view if not exists TARGET.hrrst as select * from SOURCE.hrrst; ---create view if not exists TARGET.graduatedoctorates as select * from SOURCE.graduatedoctorates; +create view if not exists TARGET.graduatedoctorates as select * from SOURCE.graduatedoctorates; create table TARGET.result_citations stored as parquet as select * from SOURCE.result_citations orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_citations COMPUTE STATISTICS; create table TARGET.result_references_oc stored as parquet as select * from SOURCE.result_references_oc orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_references_oc COMPUTE STATISTICS; create table TARGET.result_citations_oc stored as parquet as select * from SOURCE.result_citations_oc orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_citations_oc COMPUTE STATISTICS; create table TARGET.result_classifications stored as parquet as select * from SOURCE.result_classifications orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_classifications COMPUTE STATISTICS; create table TARGET.result_apc stored as parquet as select * from SOURCE.result_apc orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_apc COMPUTE STATISTICS; create table TARGET.result_concepts stored as parquet as select * from SOURCE.result_concepts orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_concepts COMPUTE STATISTICS; create table TARGET.result_datasources stored as parquet as select * from SOURCE.result_datasources orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_datasources COMPUTE STATISTICS; create table TARGET.result_fundercount stored as parquet as select * from SOURCE.result_fundercount orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_fundercount COMPUTE STATISTICS; create table TARGET.result_gold stored as parquet as select * from SOURCE.result_gold orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_gold COMPUTE STATISTICS; create table TARGET.result_greenoa stored as parquet as select * from SOURCE.result_greenoa orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_greenoa COMPUTE STATISTICS; create table TARGET.result_languages stored as parquet as select * from SOURCE.result_languages orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_languages COMPUTE STATISTICS; create table TARGET.result_licenses stored as parquet as select * from SOURCE.result_licenses orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_licenses COMPUTE STATISTICS; create table TARGET.licenses_normalized STORED AS PARQUET as select * from SOURCE.licenses_normalized; ---ANALYZE TABLE TARGET.licenses_normalized COMPUTE STATISTICS; create table TARGET.result_oids stored as parquet as select * from SOURCE.result_oids orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_oids COMPUTE STATISTICS; create table TARGET.result_organization stored as parquet as select * from SOURCE.result_organization orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_organization COMPUTE STATISTICS; create table TARGET.result_peerreviewed stored as parquet as select * from SOURCE.result_peerreviewed orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_peerreviewed COMPUTE STATISTICS; create table TARGET.result_pids stored as parquet as select * from SOURCE.result_pids orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_pids COMPUTE STATISTICS; create table TARGET.result_projectcount stored as parquet as select * from SOURCE.result_projectcount orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_projectcount COMPUTE STATISTICS; create table TARGET.result_projects stored as parquet as select * from SOURCE.result_projects orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_projects COMPUTE STATISTICS; create table TARGET.result_refereed stored as parquet as select * from SOURCE.result_refereed orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_refereed COMPUTE STATISTICS; create table TARGET.result_sources stored as parquet as select * from SOURCE.result_sources orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_sources COMPUTE STATISTICS; create table TARGET.result_topics stored as parquet as select * from SOURCE.result_topics orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_topics COMPUTE STATISTICS; create table TARGET.result_fos stored as parquet as select * from SOURCE.result_fos orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_fos COMPUTE STATISTICS; create table TARGET.result_accessroute stored as parquet as select * from SOURCE.result_accessroute orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_accessroute COMPUTE STATISTICS; + +create table TARGET.result_orcid stored as parquet as select * from SOURCE.result_orcid orig where exists (select 1 from TARGET.result r where r.id=orig.id); create view TARGET.foo1 as select * from SOURCE.result_result rr where rr.source in (select id from TARGET.result); create view TARGET.foo2 as select * from SOURCE.result_result rr where rr.target in (select id from TARGET.result); create table TARGET.result_result STORED AS PARQUET as select distinct * from (select * from TARGET.foo1 union all select * from TARGET.foo2) foufou; drop view TARGET.foo1; drop view TARGET.foo2; ---ANALYZE TABLE TARGET.result_result COMPUTE STATISTICS; -- datasources create view if not exists TARGET.datasource as select * from SOURCE.datasource; @@ -185,7 +160,6 @@ create view if not exists TARGET.datasource_organizations as select * from SOURC create view if not exists TARGET.datasource_sources as select * from SOURCE.datasource_sources; create table TARGET.datasource_results stored as parquet as select id as result, datasource as id from TARGET.result_datasources; ---ANALYZE TABLE TARGET.datasource_results COMPUTE STATISTICS; -- organizations create view if not exists TARGET.organization as select * from SOURCE.organization; @@ -203,28 +177,26 @@ create view if not exists TARGET.project_classification as select * from SOURCE. create view if not exists TARGET.project_organization_contribution as select * from SOURCE.project_organization_contribution; create table TARGET.project_results stored as parquet as select id as result, project as id from TARGET.result_projects; ---ANALYZE TABLE TARGET.project_results COMPUTE STATISTICS; -- indicators -- Sprint 1 ---- create table TARGET.indi_pub_green_oa stored as parquet as select * from SOURCE.indi_pub_green_oa orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_green_oa COMPUTE STATISTICS; + create table TARGET.indi_pub_grey_lit stored as parquet as select * from SOURCE.indi_pub_grey_lit orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_grey_lit COMPUTE STATISTICS; + create table TARGET.indi_pub_doi_from_crossref stored as parquet as select * from SOURCE.indi_pub_doi_from_crossref orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_doi_from_crossref COMPUTE STATISTICS; + -- Sprint 2 ---- create table TARGET.indi_result_has_cc_licence stored as parquet as select * from SOURCE.indi_result_has_cc_licence orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_result_has_cc_licence COMPUTE STATISTICS; + create table TARGET.indi_result_has_cc_licence_url stored as parquet as select * from SOURCE.indi_result_has_cc_licence_url orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_result_has_cc_licence_url COMPUTE STATISTICS; + create table TARGET.indi_pub_has_abstract stored as parquet as select * from SOURCE.indi_pub_has_abstract orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_has_abstract COMPUTE STATISTICS; + create table TARGET.indi_result_with_orcid stored as parquet as select * from SOURCE.indi_result_with_orcid orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_result_with_orcid COMPUTE STATISTICS; + ---- Sprint 3 ---- create table TARGET.indi_funded_result_with_fundref stored as parquet as select * from SOURCE.indi_funded_result_with_fundref orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_funded_result_with_fundref COMPUTE STATISTICS; create view TARGET.indi_result_org_collab as select * from SOURCE.indi_result_org_collab; create view TARGET.indi_result_org_country_collab as select * from SOURCE.indi_result_org_country_collab; create view TARGET.indi_project_collab_org as select * from SOURCE.indi_project_collab_org; @@ -233,32 +205,29 @@ create view TARGET.indi_funder_country_collab as select * from SOURCE.indi_funde create view TARGET.indi_result_country_collab as select * from SOURCE.indi_result_country_collab; ---- Sprint 4 ---- create table TARGET.indi_pub_diamond stored as parquet as select * from SOURCE.indi_pub_diamond orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_diamond COMPUTE STATISTICS; + create table TARGET.indi_pub_in_transformative stored as parquet as select * from SOURCE.indi_pub_in_transformative orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_in_transformative COMPUTE STATISTICS; + create table TARGET.indi_pub_closed_other_open stored as parquet as select * from SOURCE.indi_pub_closed_other_open orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_closed_other_open COMPUTE STATISTICS; ---- Sprint 5 ---- create table TARGET.indi_result_no_of_copies stored as parquet as select * from SOURCE.indi_result_no_of_copies orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_result_no_of_copies COMPUTE STATISTICS; ---- Sprint 6 ---- create table TARGET.indi_pub_hybrid_oa_with_cc stored as parquet as select * from SOURCE.indi_pub_hybrid_oa_with_cc orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_hybrid_oa_with_cc COMPUTE STATISTICS; + create table TARGET.indi_pub_bronze_oa stored as parquet as select * from SOURCE.indi_pub_bronze_oa orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_bronze_oa COMPUTE STATISTICS; + create table TARGET.indi_pub_downloads stored as parquet as select * from SOURCE.indi_pub_downloads orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); ---ANALYZE TABLE TARGET.indi_pub_downloads COMPUTE STATISTICS; + create table TARGET.indi_pub_downloads_datasource stored as parquet as select * from SOURCE.indi_pub_downloads_datasource orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); ---ANALYZE TABLE TARGET.indi_pub_downloads_datasource COMPUTE STATISTICS; + create table TARGET.indi_pub_downloads_year stored as parquet as select * from SOURCE.indi_pub_downloads_year orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); ---ANALYZE TABLE TARGET.indi_pub_downloads_year COMPUTE STATISTICS; + create table TARGET.indi_pub_downloads_datasource_year stored as parquet as select * from SOURCE.indi_pub_downloads_datasource_year orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); ---ANALYZE TABLE TARGET.indi_pub_downloads_datasource_year COMPUTE STATISTICS; + ---- Sprint 7 ---- create table TARGET.indi_pub_gold_oa stored as parquet as select * from SOURCE.indi_pub_gold_oa orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_gold_oa COMPUTE STATISTICS; + create table TARGET.indi_pub_hybrid stored as parquet as select * from SOURCE.indi_pub_hybrid orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_hybrid COMPUTE STATISTICS; create view TARGET.indi_org_fairness as select * from SOURCE.indi_org_fairness; create view TARGET.indi_org_fairness_pub_pr as select * from SOURCE.indi_org_fairness_pub_pr; create view TARGET.indi_org_fairness_pub_year as select * from SOURCE.indi_org_fairness_pub_year; @@ -269,19 +238,19 @@ create view TARGET.indi_org_findable as select * from SOURCE.indi_org_findable; create view TARGET.indi_org_openess as select * from SOURCE.indi_org_openess; create view TARGET.indi_org_openess_year as select * from SOURCE.indi_org_openess_year; create table TARGET.indi_pub_has_preprint stored as parquet as select * from SOURCE.indi_pub_has_preprint orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_has_preprint COMPUTE STATISTICS; + create table TARGET.indi_pub_in_subscribed stored as parquet as select * from SOURCE.indi_pub_in_subscribed orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_in_subscribed COMPUTE STATISTICS; + create table TARGET.indi_result_with_pid stored as parquet as select * from SOURCE.indi_result_with_pid orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_result_with_pid COMPUTE STATISTICS; + create table TARGET.indi_impact_measures stored as parquet as select * from SOURCE.indi_impact_measures orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_impact_measures COMPUTE STATISTICS; + create table TARGET.indi_pub_interdisciplinarity stored as parquet as select * from SOURCE.indi_pub_interdisciplinarity orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.indi_pub_interdisciplinarity COMPUTE STATISTICS; + create table TARGET.result_apc_affiliations stored as parquet as select * from SOURCE.result_apc_affiliations orig where exists (select 1 from TARGET.result r where r.id=orig.id); ---ANALYZE TABLE TARGET.result_apc_affiliations COMPUTE STATISTICS; -create table TARGET.indi_is_project_result_after stored as parquet as select * from SOURCE.indi_is_project_result_after orig where exists (select 1 from TARGET.result r where r.id=orig.id); -create table TARGET.indi_is_funder_plan_s stored as parquet as select * from SOURCE.indi_is_funder_plan_s orig where exists (select 1 from TARGET.result r where r.id=orig.id); + +create table TARGET.indi_is_project_result_after stored as parquet as select * from SOURCE.indi_is_project_result_after orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); +create view TARGET.indi_is_funder_plan_s as select * from SOURCE.indi_is_funder_plan_s; create view TARGET.indi_funder_fairness as select * from SOURCE.indi_funder_fairness; create view TARGET.indi_funder_openess as select * from SOURCE.indi_funder_openess; create view TARGET.indi_funder_findable as select * from SOURCE.indi_funder_findable; diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs.sql index 9a9407c2d..4469782f0 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs.sql @@ -11,5 +11,4 @@ create table TARGET.result stored as parquet as join SOURCE.context cont on cont.id=cat.context -- join SOURCE.result where rc.id=r.id and conc.category like CONTEXT) -) foo; ---ANALYZE TABLE TARGET.result COMPUTE STATISTICS; \ No newline at end of file +) foo; \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs_tail.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs_tail.sql index bad18efde..a28206d56 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs_tail.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs_tail.sql @@ -11,5 +11,4 @@ create table TARGET.result stored as parquet as join SOURCE.context cont on cont.id=cat.context -- join SOURCE.result where rc.id=r.id and conc.category not in (CONTEXTS)) -) foo; ---ANALYZE TABLE TARGET.result COMPUTE STATISTICS; \ No newline at end of file +) foo; \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_funded.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_funded.sql index b8d3c0242..ce6475c22 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_funded.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_funded.sql @@ -4,6 +4,4 @@ create database if not exists TARGET; create table TARGET.result stored as parquet as select distinct * from ( select * from SOURCE.result r where exists (select 1 from SOURCE.result_projects rp join SOURCE.project p on rp.project=p.id where rp.id=r.id) - ) foo; - ---ANALYZE TABLE TARGET.result COMPUTE STATISTICS; \ No newline at end of file + ) foo; \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql index 7bfba92a8..d2f08b391 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql @@ -59,6 +59,4 @@ create table TARGET.result stored as parquet as 'openorgs____::5d55fb216b14691cf68218daf5d78cd9', -- Munster Technological University 'openorgs____::0fccc7640f0cb44d5cd1b06b312a06b9', -- Cardiff University 'openorgs____::8839b55dae0c84d56fd533f52d5d483a' -- Leibniz Institute of Ecological Urban and Regional Development - ))) foo; - ---ANALYZE TABLE TARGET.result COMPUTE STATISTICS; \ No newline at end of file + ))) foo; \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step21-createObservatoryDB.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step21-createObservatoryDB.sql index b7e421813..2e6f0711c 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step21-createObservatoryDB.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step21-createObservatoryDB.sql @@ -8,7 +8,6 @@ from ${stats_db_name}.result r group by rl.id ) rln on rln.id=r.id; ---ANALYZE TABLE ${observatory_db_name}.result_cc_licence COMPUTE STATISTICS; create table ${observatory_db_name}.result_affiliated_country stored as parquet as select @@ -39,7 +38,6 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, c.code, c.name; ---ANALYZE TABLE ${observatory_db_name}.result_affiliated_country COMPUTE STATISTICS; create table ${observatory_db_name}.result_affiliated_year stored as parquet as select @@ -70,7 +68,6 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, r.year; ---ANALYZE TABLE ${observatory_db_name}.result_affiliated_year COMPUTE STATISTICS; create table ${observatory_db_name}.result_affiliated_year_country stored as parquet as select @@ -101,7 +98,6 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, r.year, c.code, c.name; ---ANALYZE TABLE ${observatory_db_name}.result_affiliated_year_country COMPUTE STATISTICS; create table ${observatory_db_name}.result_affiliated_datasource stored as parquet as select @@ -134,8 +130,6 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, d.name; ---ANALYZE TABLE ${observatory_db_name}.result_affiliated_datasource COMPUTE STATISTICS; - create table ${observatory_db_name}.result_affiliated_datasource_country stored as parquet as select count(distinct r.id) as total, @@ -167,8 +161,6 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, d.name, c.code, c.name; ---ANALYZE TABLE ${observatory_db_name}.result_affiliated_datasource_country COMPUTE STATISTICS; - create table ${observatory_db_name}.result_affiliated_organization stored as parquet as select count(distinct r.id) as total, @@ -198,8 +190,6 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, o.name; ---ANALYZE TABLE ${observatory_db_name}.result_affiliated_organization COMPUTE STATISTICS; - create table ${observatory_db_name}.result_affiliated_organization_country stored as parquet as select count(distinct r.id) as total, @@ -229,8 +219,6 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, o.name, c.code, c.name; ---ANALYZE TABLE ${observatory_db_name}.result_affiliated_organization_country COMPUTE STATISTICS; - create table ${observatory_db_name}.result_affiliated_funder stored as parquet as select count(distinct r.id) as total, @@ -262,8 +250,6 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, p.funder; ---ANALYZE TABLE ${observatory_db_name}.result_affiliated_funder COMPUTE STATISTICS; - create table ${observatory_db_name}.result_affiliated_funder_country stored as parquet as select count(distinct r.id) as total, @@ -295,8 +281,6 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, p.funder, c.code, c.name; ---ANALYZE TABLE ${observatory_db_name}.result_affiliated_funder_country COMPUTE STATISTICS; - create table ${observatory_db_name}.result_deposited_country stored as parquet as select count(distinct r.id) as total, @@ -328,8 +312,6 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, c.code, c.name; ---ANALYZE TABLE ${observatory_db_name}.result_deposited_country COMPUTE STATISTICS; - create table ${observatory_db_name}.result_deposited_year stored as parquet as select count(distinct r.id) as total, @@ -361,7 +343,6 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, r.year; ---ANALYZE TABLE ${observatory_db_name}.result_deposited_year COMPUTE STATISTICS; create table ${observatory_db_name}.result_deposited_year_country stored as parquet as select @@ -394,8 +375,6 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, r.year, c.code, c.name; ---ANALYZE TABLE ${observatory_db_name}.result_deposited_year_country COMPUTE STATISTICS; - create table ${observatory_db_name}.result_deposited_datasource stored as parquet as select count(distinct r.id) as total, @@ -427,8 +406,6 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, d.name; ---ANALYZE TABLE ${observatory_db_name}.result_deposited_datasource COMPUTE STATISTICS; - create table ${observatory_db_name}.result_deposited_datasource_country stored as parquet as select count(distinct r.id) as total, @@ -460,8 +437,6 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, d.name, c.code, c.name; ---ANALYZE TABLE ${observatory_db_name}.result_deposited_datasource_country COMPUTE STATISTICS; - create table ${observatory_db_name}.result_deposited_organization stored as parquet as select count(distinct r.id) as total, @@ -493,8 +468,6 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, o.name; ---ANALYZE TABLE ${observatory_db_name}.result_deposited_organization COMPUTE STATISTICS; - create table ${observatory_db_name}.result_deposited_organization_country stored as parquet as select count(distinct r.id) as total, @@ -526,8 +499,6 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, o.name, c.code, c.name; ---ANALYZE TABLE ${observatory_db_name}.result_deposited_organization_country COMPUTE STATISTICS; - create table ${observatory_db_name}.result_deposited_funder stored as parquet as select count(distinct r.id) as total, @@ -561,8 +532,6 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, p.funder; ---ANALYZE TABLE ${observatory_db_name}.result_deposited_funder COMPUTE STATISTICS; - create table ${observatory_db_name}.result_deposited_funder_country stored as parquet as select count(distinct r.id) as total, @@ -595,5 +564,3 @@ from ${stats_db_name}.result r group by r.green, r.gold, case when rl.type is not null then true else false end, case when pids.pid is not null then true else false end, case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, p.funder, c.code, c.name; - ---ANALYZE TABLE ${observatory_db_name}.result_deposited_funder_country COMPUTE STATISTICS; \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step7.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step7.sql index 1514ecf52..eb16a161e 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step7.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step7.sql @@ -123,6 +123,8 @@ UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_topics; +DROP TABLE IF EXISTS ${stats_db_name}.result_fos purge; + create table ${stats_db_name}.result_fos stored as parquet as with lvl1 as (select id, topic from ${stats_db_name}.result_topics where topic like '__ %' and type='Fields of Science and Technology classification'), @@ -133,6 +135,8 @@ from lvl1 join lvl2 on lvl1.id=lvl2.id and substr(lvl2.topic, 1, 2)=substr(lvl1.topic, 1, 2) join lvl3 on lvl3.id=lvl1.id and substr(lvl3.topic, 1, 4)=substr(lvl2.topic, 1, 4); +DROP TABLE IF EXISTS ${stats_db_name}.result_organization purge; + CREATE TABLE ${stats_db_name}.result_organization STORED AS PARQUET AS SELECT substr(r.target, 4) AS id, substr(r.source, 4) AS organization FROM ${openaire_db_name}.relation r @@ -140,6 +144,8 @@ WHERE r.reltype = 'resultOrganization' and r.target like '50|%' and r.datainfo.deletedbyinference = false and r.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.result_projects purge; + CREATE TABLE ${stats_db_name}.result_projects STORED AS PARQUET AS select pr.result AS id, pr.id AS project, datediff(p.enddate, p.startdate) AS daysfromend, pr.provenance as provenance FROM ${stats_db_name}.result r diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step8.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step8.sql index 248716b36..07204db0c 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step8.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step8.sql @@ -5,6 +5,8 @@ -- Datasource table/view and Datasource related tables/views ------------------------------------------------------------ ------------------------------------------------------------ +DROP TABLE IF EXISTS ${stats_db_name}.datasource_tmp purge; + CREATE TABLE ${stats_db_name}.datasource_tmp ( `id` string, @@ -48,6 +50,7 @@ WHERE d1.datainfo.deletedbyinference = FALSE and d1.datainfo.invisible=false; -- Updating temporary table with everything that is not based on results -> This is done with the following "dual" table. -- Creating a temporary dual table that will be removed after the following insert + CREATE TABLE ${stats_db_name}.dual ( dummy CHAR(1)); INSERT INTO ${stats_db_name}.dual VALUES ('X'); @@ -74,16 +77,22 @@ DROP TABLE ${stats_db_name}.dual; UPDATE ${stats_db_name}.datasource_tmp SET name='Other' WHERE name = 'Unknown Repository'; UPDATE ${stats_db_name}.datasource_tmp SET yearofvalidation=null WHERE yearofvalidation = '-1'; +DROP TABLE IF EXISTS ${stats_db_name}.datasource_languages purge; + CREATE TABLE ${stats_db_name}.datasource_languages STORED AS PARQUET AS SELECT substr(d.id, 4) AS id, langs.languages AS language FROM ${openaire_db_name}.datasource d LATERAL VIEW explode(d.odlanguages.value) langs AS languages where d.datainfo.deletedbyinference=false and d.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.datasource_oids purge; + CREATE TABLE ${stats_db_name}.datasource_oids STORED AS PARQUET AS SELECT substr(d.id, 4) AS id, oids.ids AS oid FROM ${openaire_db_name}.datasource d LATERAL VIEW explode(d.originalid) oids AS ids where d.datainfo.deletedbyinference=false and d.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.datasource_organizations purge; + CREATE TABLE ${stats_db_name}.datasource_organizations STORED AS PARQUET AS SELECT substr(r.target, 4) AS id, substr(r.source, 4) AS organization FROM ${openaire_db_name}.relation r @@ -91,6 +100,8 @@ WHERE r.reltype = 'datasourceOrganization' and r.datainfo.deletedbyinference = f -- datasource sources: -- where the datasource info have been collected from. +DROP TABLE IF EXISTS ${stats_db_name}.datasource_sources purge; + create table if not exists ${stats_db_name}.datasource_sources STORED AS PARQUET AS select substr(d.id, 4) as id, substr(cf.key, 4) as datasource from ${openaire_db_name}.datasource d lateral view explode(d.collectedfrom) cfrom as cf diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step9.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step9.sql index 3da36dfe5..19d301e27 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step9.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step9.sql @@ -3,6 +3,8 @@ -- Organization table/view and Organization related tables/views ---------------------------------------------------------------- ---------------------------------------------------------------- +DROP TABLE IF EXISTS ${stats_db_name}.organization purge; + CREATE TABLE IF NOT EXISTS ${stats_db_name}.organization STORED AS PARQUET AS SELECT substr(o.id, 4) as id, o.legalname.value as name, diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml index aa991730b..cbf97944d 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml @@ -114,6 +114,7 @@ ${wf:conf('resumeFrom') eq 'step21-createObservatoryDB'} ${wf:conf('resumeFrom') eq 'step21-createObservatoryDB-post'} ${wf:conf('resumeFrom') eq 'step22-copyDataToImpalaCluster'} + ${wf:conf('resumeFrom') eq 'step22a-createPDFsAggregated'} ${wf:conf('resumeFrom') eq 'step23-finalizeImpalaCluster'} ${wf:conf('resumeFrom') eq 'Step24-updateCache'} @@ -448,6 +449,22 @@ ${hadoop_user_name} copyDataToImpalaCluster.sh + + + + + + + ${jobTracker} + ${nameNode} + createPDFsAggregated.sh + + + ${stats_db_name} + ${monitor_db_name} + ${hadoop_user_name} + createPDFsAggregated.sh + From 11a1207f9c8476db4fc926f707915c3209720106 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Wed, 22 Nov 2023 12:22:14 +0100 Subject: [PATCH 104/148] [graph cleaning] applying coar based vocabularies in bulk --- .../dhp/oa/merge/GroupEntitiesSparkJob.java | 40 +++++- .../oaf/utils/GraphCleaningFunctions.java | 95 +++++++++++++ .../dhp/schema/oaf/utils/OafMapperUtils.java | 21 ++- .../group_graph_entities_parameters.json | 6 + .../raw/AbstractMdRecordToOafMapper.java | 63 +-------- .../group/GroupEntitiesSparkJobTest.java | 131 ++++++++++++++++-- .../raw/GenerateEntitiesApplicationTest.java | 5 +- .../dnetlib/dhp/oa/graph/raw/MappersTest.java | 28 ++-- .../dhp/oa/graph/group/dataset/dataset.json | 6 +- .../graph/group/publication/publication.json | 6 +- pom.xml | 2 +- 11 files changed, 294 insertions(+), 109 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java index f5c8eea19..0225a5063 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/GroupEntitiesSparkJob.java @@ -21,10 +21,15 @@ import org.slf4j.LoggerFactory; import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.common.HdfsSupport; +import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup; import eu.dnetlib.dhp.schema.common.EntityType; import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.oaf.OafEntity; +import eu.dnetlib.dhp.schema.oaf.utils.GraphCleaningFunctions; import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils; +import eu.dnetlib.dhp.utils.ISLookupClientFactory; +import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; +import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; import scala.Tuple2; /** @@ -35,6 +40,12 @@ public class GroupEntitiesSparkJob { private static final Encoder OAFENTITY_KRYO_ENC = Encoders.kryo(OafEntity.class); + private ArgumentApplicationParser parser; + + public GroupEntitiesSparkJob(ArgumentApplicationParser parser) { + this.parser = parser; + } + public static void main(String[] args) throws Exception { String jsonConfiguration = IOUtils @@ -51,6 +62,17 @@ public class GroupEntitiesSparkJob { .orElse(Boolean.TRUE); log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + final String isLookupUrl = parser.get("isLookupUrl"); + log.info("isLookupUrl: {}", isLookupUrl); + + final ISLookUpService isLookupService = ISLookupClientFactory.getLookUpService(isLookupUrl); + + new GroupEntitiesSparkJob(parser).run(isSparkSessionManaged, isLookupService); + } + + public void run(Boolean isSparkSessionManaged, ISLookUpService isLookUpService) + throws ISLookUpException { + String graphInputPath = parser.get("graphInputPath"); log.info("graphInputPath: {}", graphInputPath); @@ -60,19 +82,21 @@ public class GroupEntitiesSparkJob { String outputPath = parser.get("outputPath"); log.info("outputPath: {}", outputPath); - boolean filterInvisible = Boolean.valueOf(parser.get("filterInvisible")); + boolean filterInvisible = Boolean.parseBoolean(parser.get("filterInvisible")); log.info("filterInvisible: {}", filterInvisible); SparkConf conf = new SparkConf(); conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); conf.registerKryoClasses(ModelSupport.getOafModelClasses()); + final VocabularyGroup vocs = VocabularyGroup.loadVocsFromIS(isLookUpService); + runWithSparkSession( conf, isSparkSessionManaged, spark -> { HdfsSupport.remove(checkpointPath, spark.sparkContext().hadoopConfiguration()); - groupEntities(spark, graphInputPath, checkpointPath, outputPath, filterInvisible); + groupEntities(spark, graphInputPath, checkpointPath, outputPath, filterInvisible, vocs); }); } @@ -81,7 +105,7 @@ public class GroupEntitiesSparkJob { String inputPath, String checkpointPath, String outputPath, - boolean filterInvisible) { + boolean filterInvisible, VocabularyGroup vocs) { Dataset allEntities = spark.emptyDataset(OAFENTITY_KRYO_ENC); @@ -106,10 +130,14 @@ public class GroupEntitiesSparkJob { } Dataset groupedEntities = allEntities - .groupByKey((MapFunction) OafEntity::getId, Encoders.STRING()) - .reduceGroups((ReduceFunction) (b, a) -> OafMapperUtils.mergeEntities(b, a)) .map( - (MapFunction, Tuple2>) t -> new Tuple2( + (MapFunction) entity -> GraphCleaningFunctions + .applyCoarVocabularies(entity, vocs), + OAFENTITY_KRYO_ENC) + .groupByKey((MapFunction) OafEntity::getId, Encoders.STRING()) + .reduceGroups((ReduceFunction) OafMapperUtils::mergeEntities) + .map( + (MapFunction, Tuple2>) t -> new Tuple2<>( t._2().getClass().getName(), t._2()), Encoders.tuple(Encoders.STRING(), OAFENTITY_KRYO_ENC)); diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java index 324e3dd58..b23dade98 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java @@ -1,6 +1,8 @@ package eu.dnetlib.dhp.schema.oaf.utils; +import static eu.dnetlib.dhp.schema.common.ModelConstants.*; +import static eu.dnetlib.dhp.schema.common.ModelConstants.OPENAIRE_META_RESOURCE_TYPE; import static eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils.getProvenance; import java.time.LocalDate; @@ -784,4 +786,97 @@ public class GraphCleaningFunctions extends CleaningFunctions { return s; } + public static OafEntity applyCoarVocabularies(OafEntity entity, VocabularyGroup vocs) { + + if (entity instanceof Result) { + final Result result = (Result) entity; + + Optional + .ofNullable(result.getInstance()) + .ifPresent( + instances -> instances + .forEach( + instance -> { + if (Objects.isNull(instance.getInstanceTypeMapping())) { + List mapping = Lists.newArrayList(); + mapping + .add( + OafMapperUtils + .instanceTypeMapping( + instance.getInstancetype().getClassname(), + OPENAIRE_COAR_RESOURCE_TYPES_3_1)); + instance.setInstanceTypeMapping(mapping); + } + Optional optionalItm = instance + .getInstanceTypeMapping() + .stream() + .filter(GraphCleaningFunctions::originalResourceType) + .findFirst(); + if (optionalItm.isPresent()) { + InstanceTypeMapping coarItm = optionalItm.get(); + Optional + .ofNullable( + vocs + .lookupTermBySynonym( + OPENAIRE_COAR_RESOURCE_TYPES_3_1, coarItm.getOriginalType())) + .ifPresent(type -> { + coarItm.setTypeCode(type.getClassid()); + coarItm.setTypeLabel(type.getClassname()); + }); + final List mappings = Lists.newArrayList(); + if (vocs.vocabularyExists(OPENAIRE_USER_RESOURCE_TYPES)) { + Optional + .ofNullable( + vocs + .lookupTermBySynonym( + OPENAIRE_USER_RESOURCE_TYPES, coarItm.getTypeCode())) + .ifPresent( + type -> mappings + .add( + OafMapperUtils + .instanceTypeMapping(coarItm.getTypeCode(), type))); + } + if (!mappings.isEmpty()) { + instance.getInstanceTypeMapping().addAll(mappings); + } + } + })); + result.setMetaResourceType(getMetaResourceType(result.getInstance(), vocs)); + } + + return entity; + } + + private static boolean originalResourceType(InstanceTypeMapping itm) { + return StringUtils.isNotBlank(itm.getOriginalType()) && + OPENAIRE_COAR_RESOURCE_TYPES_3_1.equals(itm.getVocabularyName()) && + StringUtils.isBlank(itm.getTypeCode()) && + StringUtils.isBlank(itm.getTypeLabel()); + } + + private static Qualifier getMetaResourceType(final List instances, final VocabularyGroup vocs) { + + if (vocs.vocabularyExists(OPENAIRE_META_RESOURCE_TYPE)) { + Optional instanceTypeMapping = instances + .stream() + .flatMap( + i -> Optional.ofNullable(i.getInstanceTypeMapping()).map(Collection::stream).orElse(Stream.empty())) + .filter(t -> OPENAIRE_COAR_RESOURCE_TYPES_3_1.equals(t.getVocabularyName())) + .findFirst(); + + if (!instanceTypeMapping.isPresent()) { + return null; + } else { + final String typeCode = instanceTypeMapping.get().getTypeCode(); + return Optional + .ofNullable(vocs.lookupTermBySynonym(OPENAIRE_META_RESOURCE_TYPE, typeCode)) + .orElseThrow( + () -> new IllegalStateException("unable to find a synonym for '" + typeCode + "' in " + + OPENAIRE_META_RESOURCE_TYPE)); + } + } else { + throw new IllegalStateException("vocabulary '" + OPENAIRE_META_RESOURCE_TYPE + "' not available"); + } + } + } diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/OafMapperUtils.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/OafMapperUtils.java index 1c557c805..4cecd0895 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/OafMapperUtils.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/OafMapperUtils.java @@ -140,15 +140,28 @@ public class OafMapperUtils { .collect(Collectors.toList()); } - public static InstanceTypeMapping instanceTypeMapping(String originalType, Qualifier term) { + public static InstanceTypeMapping instanceTypeMapping(String originalType, String code, String label, + String vocabularyName) { final InstanceTypeMapping m = new InstanceTypeMapping(); - m.setVocabularyName(term.getSchemeid()); + m.setVocabularyName(vocabularyName); m.setOriginalType(originalType); - m.setTypeCode(term.getClassid()); - m.setTypeLabel(term.getClassname()); + m.setTypeCode(code); + m.setTypeLabel(label); return m; } + public static InstanceTypeMapping instanceTypeMapping(String originalType, Qualifier term) { + return instanceTypeMapping(originalType, term.getClassid(), term.getClassname(), term.getSchemeid()); + } + + public static InstanceTypeMapping instanceTypeMapping(String originalType) { + return instanceTypeMapping(originalType, null, null, null); + } + + public static InstanceTypeMapping instanceTypeMapping(String originalType, String vocabularyName) { + return instanceTypeMapping(originalType, null, null, vocabularyName); + } + public static Qualifier unknown(final String schemeid, final String schemename) { return qualifier(UNKNOWN, "Unknown", schemeid, schemename); } diff --git a/dhp-common/src/main/resources/eu/dnetlib/dhp/oa/merge/group_graph_entities_parameters.json b/dhp-common/src/main/resources/eu/dnetlib/dhp/oa/merge/group_graph_entities_parameters.json index 58e3ca711..512878457 100644 --- a/dhp-common/src/main/resources/eu/dnetlib/dhp/oa/merge/group_graph_entities_parameters.json +++ b/dhp-common/src/main/resources/eu/dnetlib/dhp/oa/merge/group_graph_entities_parameters.json @@ -28,5 +28,11 @@ "paramLongName": "filterInvisible", "paramDescription": "if true filters out invisible entities", "paramRequired": true + }, + { + "paramName": "isu", + "paramLongName": "isLookupUrl", + "paramDescription": "url to the ISLookup Service", + "paramRequired": true } ] \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java index 49133cedb..a71908b02 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/raw/AbstractMdRecordToOafMapper.java @@ -125,9 +125,7 @@ public abstract class AbstractMdRecordToOafMapper { final String type = getResultType(doc, instances); - final Qualifier metaResourceType = getMetaResourceType(instances); - - return createOafs(doc, type, metaResourceType, instances, collectedFrom, entityInfo, lastUpdateTimestamp); + return createOafs(doc, type, instances, collectedFrom, entityInfo, lastUpdateTimestamp); } catch (DocumentException e) { log.error("Error with record:\n" + xml); return Lists.newArrayList(); @@ -153,30 +151,6 @@ public abstract class AbstractMdRecordToOafMapper { return type; } - protected Qualifier getMetaResourceType(final List instances) { - - if (vocs.vocabularyExists(OPENAIRE_META_RESOURCE_TYPE)) { - Optional instanceTypeMapping = instances - .stream() - .flatMap(i -> i.getInstanceTypeMapping().stream()) - .filter(t -> OPENAIRE_COAR_RESOURCE_TYPES_3_1.equals(t.getVocabularyName())) - .findFirst(); - - if (!instanceTypeMapping.isPresent()) { - return null; - } else { - final String typeCode = instanceTypeMapping.get().getTypeCode(); - return Optional - .ofNullable(vocs.lookupTermBySynonym(OPENAIRE_META_RESOURCE_TYPE, typeCode)) - .orElseThrow( - () -> new IllegalStateException("unable to find a synonym for '" + typeCode + "' in " + - OPENAIRE_META_RESOURCE_TYPE)); - } - } else { - throw new IllegalStateException("vocabulary '" + OPENAIRE_META_RESOURCE_TYPE + "' not available"); - } - } - private KeyValue getProvenanceDatasource(final Document doc, final String xpathId, final String xpathName) { final String dsId = doc.valueOf(xpathId); final String dsName = doc.valueOf(xpathName); @@ -191,14 +165,13 @@ public abstract class AbstractMdRecordToOafMapper { protected List createOafs( final Document doc, final String type, - final Qualifier metaResourceType, final List instances, final KeyValue collectedFrom, final DataInfo info, final long lastUpdateTimestamp) { final OafEntity entity = createEntity( - doc, type, metaResourceType, instances, collectedFrom, info, lastUpdateTimestamp); + doc, type, instances, collectedFrom, info, lastUpdateTimestamp); final Set originalId = Sets.newHashSet(entity.getOriginalId()); originalId.add(entity.getId()); @@ -231,7 +204,6 @@ public abstract class AbstractMdRecordToOafMapper { private OafEntity createEntity(final Document doc, final String type, - final Qualifier metaResourceType, final List instances, final KeyValue collectedFrom, final DataInfo info, @@ -239,12 +211,12 @@ public abstract class AbstractMdRecordToOafMapper { switch (type.toLowerCase()) { case "publication": final Publication p = new Publication(); - populateResultFields(p, metaResourceType, doc, instances, collectedFrom, info, lastUpdateTimestamp); + populateResultFields(p, doc, instances, collectedFrom, info, lastUpdateTimestamp); p.setJournal(prepareJournal(doc, info)); return p; case "dataset": final Dataset d = new Dataset(); - populateResultFields(d, metaResourceType, doc, instances, collectedFrom, info, lastUpdateTimestamp); + populateResultFields(d, doc, instances, collectedFrom, info, lastUpdateTimestamp); d.setStoragedate(prepareDatasetStorageDate(doc, info)); d.setDevice(prepareDatasetDevice(doc, info)); d.setSize(prepareDatasetSize(doc, info)); @@ -255,7 +227,7 @@ public abstract class AbstractMdRecordToOafMapper { return d; case "software": final Software s = new Software(); - populateResultFields(s, metaResourceType, doc, instances, collectedFrom, info, lastUpdateTimestamp); + populateResultFields(s, doc, instances, collectedFrom, info, lastUpdateTimestamp); s.setDocumentationUrl(prepareSoftwareDocumentationUrls(doc, info)); s.setLicense(prepareSoftwareLicenses(doc, info)); s.setCodeRepositoryUrl(prepareSoftwareCodeRepositoryUrl(doc, info)); @@ -265,7 +237,7 @@ public abstract class AbstractMdRecordToOafMapper { case "otherresearchproducts": default: final OtherResearchProduct o = new OtherResearchProduct(); - populateResultFields(o, metaResourceType, doc, instances, collectedFrom, info, lastUpdateTimestamp); + populateResultFields(o, doc, instances, collectedFrom, info, lastUpdateTimestamp); o.setContactperson(prepareOtherResearchProductContactPersons(doc, info)); o.setContactgroup(prepareOtherResearchProductContactGroups(doc, info)); o.setTool(prepareOtherResearchProductTools(doc, info)); @@ -402,13 +374,11 @@ public abstract class AbstractMdRecordToOafMapper { private void populateResultFields( final Result r, - final Qualifier metaResourceType, final Document doc, final List instances, final KeyValue collectedFrom, final DataInfo info, final long lastUpdateTimestamp) { - r.setMetaResourceType(metaResourceType); r.setDataInfo(info); r.setLastupdatetimestamp(lastUpdateTimestamp); r.setId(createOpenaireId(50, doc.valueOf("//dri:objIdentifier"), false)); @@ -555,26 +525,7 @@ public abstract class AbstractMdRecordToOafMapper { .ofNullable(findOriginalType(doc)) .map(originalType -> { final List mappings = Lists.newArrayList(); - - if (vocs.vocabularyExists(OPENAIRE_COAR_RESOURCE_TYPES_3_1)) { - - // TODO verify what the vocabs return when a synonym is not defined - Optional - .ofNullable(vocs.lookupTermBySynonym(OPENAIRE_COAR_RESOURCE_TYPES_3_1, originalType)) - .ifPresent(coarTerm -> { - mappings.add(OafMapperUtils.instanceTypeMapping(originalType, coarTerm)); - if (vocs.vocabularyExists(OPENAIRE_USER_RESOURCE_TYPES)) { - - // TODO verify what the vocabs return when a synonym is not defined - Optional - .ofNullable( - vocs.lookupTermBySynonym(OPENAIRE_USER_RESOURCE_TYPES, coarTerm.getClassid())) - .ifPresent( - type -> mappings.add(OafMapperUtils.instanceTypeMapping(originalType, type))); - } - }); - } - + mappings.add(OafMapperUtils.instanceTypeMapping(originalType, OPENAIRE_COAR_RESOURCE_TYPES_3_1)); return mappings; }) .orElse(new ArrayList<>()); diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/group/GroupEntitiesSparkJobTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/group/GroupEntitiesSparkJobTest.java index 0887adf45..242aed88e 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/group/GroupEntitiesSparkJobTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/group/GroupEntitiesSparkJobTest.java @@ -1,15 +1,23 @@ package eu.dnetlib.dhp.oa.graph.group; +import static eu.dnetlib.dhp.schema.common.ModelConstants.*; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.lenient; import java.io.IOException; import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import org.apache.commons.cli.ParseException; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.function.FilterFunction; import org.apache.spark.api.java.function.MapFunction; @@ -17,20 +25,36 @@ import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Encoders; import org.apache.spark.sql.SparkSession; import org.junit.jupiter.api.*; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.common.HdfsSupport; +import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup; import eu.dnetlib.dhp.oa.merge.GroupEntitiesSparkJob; +import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.common.ModelSupport; +import eu.dnetlib.dhp.schema.oaf.Instance; +import eu.dnetlib.dhp.schema.oaf.InstanceTypeMapping; import eu.dnetlib.dhp.schema.oaf.OafEntity; import eu.dnetlib.dhp.schema.oaf.Result; import eu.dnetlib.dhp.utils.DHPUtils; +import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; +import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; +@ExtendWith(MockitoExtension.class) @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class GroupEntitiesSparkJobTest { + @Mock + private ISLookUpService isLookUpService; + + private VocabularyGroup vocabularies; + private static SparkSession spark; private static ObjectMapper mapper = new ObjectMapper() @@ -45,10 +69,10 @@ public class GroupEntitiesSparkJobTest { @BeforeAll public static void beforeAll() throws IOException { - workingDir = Files.createTempDirectory(GroupEntitiesSparkJob.class.getSimpleName()); + workingDir = Files.createTempDirectory(GroupEntitiesSparkJobTest.class.getSimpleName()); SparkConf conf = new SparkConf(); - conf.setAppName(GroupEntitiesSparkJob.class.getSimpleName()); + conf.setAppName(GroupEntitiesSparkJobTest.class.getSimpleName()); conf.setMaster("local"); conf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer"); conf.registerKryoClasses(ModelSupport.getOafModelClasses()); @@ -56,10 +80,17 @@ public class GroupEntitiesSparkJobTest { } @BeforeEach - public void beforeEach() throws IOException, URISyntaxException { + public void beforeEach() throws IOException, URISyntaxException, ISLookUpException { dataInputPath = Paths.get(ClassLoader.getSystemResource("eu/dnetlib/dhp/oa/graph/group").toURI()); checkpointPath = workingDir.resolve("grouped_entity"); outputPath = workingDir.resolve("dispatched_entity"); + + lenient().when(isLookUpService.quickSearchProfile(VocabularyGroup.VOCABULARIES_XQUERY)).thenReturn(vocs()); + lenient() + .when(isLookUpService.quickSearchProfile(VocabularyGroup.VOCABULARY_SYNONYMS_XQUERY)) + .thenReturn(synonyms()); + + vocabularies = VocabularyGroup.loadVocsFromIS(isLookUpService); } @AfterAll @@ -71,18 +102,17 @@ public class GroupEntitiesSparkJobTest { @Test @Order(1) void testGroupEntities() throws Exception { - GroupEntitiesSparkJob.main(new String[] { - "-isSparkSessionManaged", - Boolean.FALSE.toString(), - "-graphInputPath", - dataInputPath.toString(), - "-checkpointPath", - checkpointPath.toString(), - "-outputPath", - outputPath.toString(), - "-filterInvisible", - Boolean.FALSE.toString() - }); + new GroupEntitiesSparkJob( + args( + "/eu/dnetlib/dhp/oa/merge/group_graph_entities_parameters.json", + new String[] { + "--isSparkSessionManaged", Boolean.FALSE.toString(), + "--graphInputPath", dataInputPath.toString(), + "--checkpointPath", checkpointPath.toString(), + "--outputPath", outputPath.toString(), + "--filterInvisible", Boolean.FALSE.toString(), + "--isLookupUrl", "lookupurl" + })).run(false, isLookUpService); Dataset checkpointTable = spark .read() @@ -109,6 +139,14 @@ public class GroupEntitiesSparkJobTest { .map((MapFunction) s -> mapper.readValue(s, Result.class), Encoders.bean(Result.class)); assertEquals(3, output.count()); + + List resultTypes = output + .map((MapFunction) value -> value.getResulttype().getClassid(), Encoders.STRING()) + .distinct() + .collectAsList(); + + assertEquals(2, resultTypes.size()); + assertEquals( 2, output @@ -121,5 +159,68 @@ public class GroupEntitiesSparkJobTest { .map((MapFunction) r -> r.getResulttype().getClassid(), Encoders.STRING()) .filter((FilterFunction) s -> s.equals("dataset")) .count()); + + Result result = output + .filter("id = '50|doi_________::09821844208a5cd6300b2bfb13bca1b9'") + .first(); + + result.getInstance().forEach(instance -> { + Optional coarType = instance + .getInstanceTypeMapping() + .stream() + .filter(itm -> OPENAIRE_COAR_RESOURCE_TYPES_3_1.equals(itm.getVocabularyName())) + .filter(itm -> "journal-article".equals(itm.getOriginalType())) + .findFirst(); + + assertTrue(coarType.isPresent()); + assertEquals("http://purl.org/coar/resource_type/c_2df8fbb1", coarType.get().getTypeCode()); + assertEquals("research article", coarType.get().getTypeLabel()); + }); + + final Dataset filtered = output.filter("id = '50|DansKnawCris::203a27996ddc0fd1948258e5b7dec61c'"); + assertEquals(1, filtered.count()); + result = filtered.first(); + + result + .getInstance() + .stream() + .flatMap(instance -> instance.getInstanceTypeMapping().stream()) + .filter(itm -> OPENAIRE_COAR_RESOURCE_TYPES_3_1.equals(itm.getVocabularyName())) + .filter(itm -> "Patent".equals(itm.getOriginalType())) + .forEach(itm -> { + assertEquals("http://purl.org/coar/resource_type/c_15cd", itm.getTypeCode()); + assertEquals("patent", itm.getTypeLabel()); + }); } + + private List vocs() throws IOException { + return IOUtils + .readLines( + Objects + .requireNonNull( + getClass().getResourceAsStream("/eu/dnetlib/dhp/oa/graph/clean/terms.txt"))); + } + + private List synonyms() throws IOException { + return IOUtils + .readLines( + Objects + .requireNonNull( + getClass().getResourceAsStream("/eu/dnetlib/dhp/oa/graph/clean/synonyms.txt"))); + } + + private ArgumentApplicationParser args(String paramSpecs, String[] args) throws IOException, ParseException { + ArgumentApplicationParser parser = new ArgumentApplicationParser(classPathResourceAsString(paramSpecs)); + parser.parseArgument(args); + return parser; + } + + private static String classPathResourceAsString(String path) throws IOException { + return IOUtils + .toString( + Objects + .requireNonNull( + GroupEntitiesSparkJobTest.class.getResourceAsStream(path))); + } + } diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplicationTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplicationTest.java index 53b3f8432..6d6b2ffbd 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplicationTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/GenerateEntitiesApplicationTest.java @@ -92,13 +92,14 @@ class GenerateEntitiesApplicationTest { private List vocs() throws IOException { return IOUtils .readLines( - GraphCleaningFunctionsTest.class.getResourceAsStream("/eu/dnetlib/dhp/oa/graph/clean/terms.txt")); + GenerateEntitiesApplicationTest.class.getResourceAsStream("/eu/dnetlib/dhp/oa/graph/clean/terms.txt")); } private List synonyms() throws IOException { return IOUtils .readLines( - GraphCleaningFunctionsTest.class.getResourceAsStream("/eu/dnetlib/dhp/oa/graph/clean/synonyms.txt")); + GenerateEntitiesApplicationTest.class + .getResourceAsStream("/eu/dnetlib/dhp/oa/graph/clean/synonyms.txt")); } } diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java index e6997ac1d..1f2defa01 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/raw/MappersTest.java @@ -122,7 +122,7 @@ class MappersTest { assertTrue(instance.getPid().isEmpty()); assertNotNull(instance.getInstanceTypeMapping()); - assertEquals(2, instance.getInstanceTypeMapping().size()); + assertEquals(1, instance.getInstanceTypeMapping().size()); Optional coarType = instance .getInstanceTypeMapping() @@ -131,8 +131,8 @@ class MappersTest { .findFirst(); assertTrue(coarType.isPresent()); - assertEquals("http://purl.org/coar/resource_type/c_5794", coarType.get().getTypeCode()); - assertEquals("conference paper", coarType.get().getTypeLabel()); + assertNull(coarType.get().getTypeCode()); + assertNull(coarType.get().getTypeLabel()); Optional userType = instance .getInstanceTypeMapping() @@ -140,9 +140,7 @@ class MappersTest { .filter(itm -> ModelConstants.OPENAIRE_USER_RESOURCE_TYPES.equals(itm.getVocabularyName())) .findFirst(); - assertTrue(userType.isPresent()); - assertEquals("Article", userType.get().getTypeCode()); - assertEquals("Article", userType.get().getTypeLabel()); + assertFalse(userType.isPresent()); assertFalse(instance.getAlternateIdentifier().isEmpty()); assertEquals("doi", instance.getAlternateIdentifier().get(0).getQualifier().getClassid()); @@ -710,14 +708,10 @@ class MappersTest { assertEquals("0001", p_cleaned.getInstance().get(0).getRefereed().getClassid()); assertEquals("peerReviewed", p_cleaned.getInstance().get(0).getRefereed().getClassname()); - assertNotNull(p_cleaned.getMetaResourceType()); - assertEquals("Research Literature", p_cleaned.getMetaResourceType().getClassid()); - assertEquals("Research Literature", p_cleaned.getMetaResourceType().getClassname()); - assertEquals(ModelConstants.OPENAIRE_META_RESOURCE_TYPE, p_cleaned.getMetaResourceType().getSchemeid()); - assertEquals(ModelConstants.OPENAIRE_META_RESOURCE_TYPE, p_cleaned.getMetaResourceType().getSchemename()); + assertNull(p_cleaned.getMetaResourceType()); assertNotNull(p_cleaned.getInstance().get(0).getInstanceTypeMapping()); - assertEquals(2, p_cleaned.getInstance().get(0).getInstanceTypeMapping().size()); + assertEquals(1, p_cleaned.getInstance().get(0).getInstanceTypeMapping().size()); assertTrue( p_cleaned @@ -728,8 +722,7 @@ class MappersTest { .anyMatch( t -> "journal-article".equals(t.getOriginalType()) && ModelConstants.OPENAIRE_COAR_RESOURCE_TYPES_3_1.equals(t.getVocabularyName()) && - "http://purl.org/coar/resource_type/c_2df8fbb1".equals(t.getTypeCode()) && - "research article".equals(t.getTypeLabel()))); + Objects.isNull(t.getTypeCode()) && Objects.isNull(t.getTypeLabel()))); assertTrue( p_cleaned @@ -737,11 +730,8 @@ class MappersTest { .get(0) .getInstanceTypeMapping() .stream() - .anyMatch( - t -> "journal-article".equals(t.getOriginalType()) && - ModelConstants.OPENAIRE_USER_RESOURCE_TYPES.equals(t.getVocabularyName()) && - "Article".equals(t.getTypeCode()) && - "Article".equals(t.getTypeLabel()))); + .noneMatch( + t -> ModelConstants.OPENAIRE_USER_RESOURCE_TYPES.equals(t.getVocabularyName()))); } @Test diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/group/dataset/dataset.json b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/group/dataset/dataset.json index e30be47e9..efbabb3c8 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/group/dataset/dataset.json +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/group/dataset/dataset.json @@ -1,3 +1,3 @@ -{"author":[{"affiliation":[],"fullname":"Greenough, B","name":"B","pid":[],"rank":1,"surname":"Greenough"}],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":[{"key":"10|opendoar____::358aee4cc897452c00244351e4d91f69","value":"Zenodo"}],"context":[],"contributor":[],"country":[],"coverage":[],"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofcollection":"2021-09-25T10:55:00.639Z","dateoftransformation":"2021-09-25T11:00:04.201Z","description":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Heritage Education"}],"externalReference":[],"extraInfo":[],"format":[],"fulltext":[],"geolocation":[],"id":"50|doi_________::09821844208a5cd6300b2bfb13bca1b9","instance":[{"accessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"urn","classname":"urn","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"urn:nbn:nl:ui:13-59-cjhf"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"Digital Object Identifier","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.17632/96bpgw5j9d.1"}],"collectedfrom":{"key":"10|opendoar____::358aee4cc897452c00244351e4d91f69","value":"Zenodo"},"hostedby":{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"},"instancetype":{"classid":"0021","classname":"Dataset","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"alternateIdentifier":[],"refereed":{"classid":"0000","classname":"Unknown","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"url":["","http://dx.doi.org/10.17632/96bpgw5j9d.1"]}],"language":{"classid":"und","classname":"Undetermined","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastupdatetimestamp":1635434801681,"oaiprovenance":{"originDescription":{"altered":true,"baseURL":"http%3A%2F%2Fservices.nod.dans.knaw.nl%2Foa-cerif","datestamp":"2021-08-16T15:29:45Z","harvestDate":"2021-09-25T10:55:00.639Z","identifier":"oai:services.nod.dans.knaw.nl:Products/dans:oai:easy.dans.knaw.nl:easy-dataset:211323","metadataNamespace":""}},"originalId":["50|DansKnawCris::09821844208a5cd6300b2bfb13bca1b9","oai:services.nod.dans.knaw.nl:Products/dans:oai:easy.dans.knaw.nl:easy-dataset:211323"],"pid":[],"relevantdate":[],"resourcetype":{"classid":"0021","classname":"0021","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"source":[],"subject":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Interdisciplinary sciences"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Interdisciplinary sciences"}],"title":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"Heritage Education"}]} -{"author":[{"affiliation":[],"fullname":"Keijers, D.M.G.","name":"D.M.G.","pid":[],"rank":1,"surname":"Keijers"}],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":[{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"}],"context":[],"contributor":[],"country":[],"coverage":[],"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofcollection":"2021-09-25T10:41:59.767Z","dateoftransformation":"2021-09-25T11:00:19.238Z","description":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"onderzoeksrapport"}],"externalReference":[],"extraInfo":[],"format":[],"fulltext":[],"geolocation":[],"id":"50|DansKnawCris::0dd644304b7116e8e58da3a5e3adc37a","instance":[{"accessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"urn","classname":"urn","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"urn:nbn:nl:ui:13-das-fkq"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"Digital Object Identifier","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.17026/dans-xsw-qtnx"}],"collectedfrom":{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"},"hostedby":{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"},"instancetype":{"classid":"0021","classname":"Dataset","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"alternateIdentifier":[],"refereed":{"classid":"0000","classname":"Unknown","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"url":["","http://dx.doi.org/10.17026/dans-xsw-qtnx"]}],"language":{"classid":"dut/nld","classname":"Dutch; Flemish","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastupdatetimestamp":1635434847381,"oaiprovenance":{"originDescription":{"altered":true,"baseURL":"http%3A%2F%2Fservices.nod.dans.knaw.nl%2Foa-cerif","datestamp":"2021-08-16T13:53:29Z","harvestDate":"2021-09-25T10:41:59.767Z","identifier":"oai:services.nod.dans.knaw.nl:Products/dans:oai:easy.dans.knaw.nl:easy-dataset:20759","metadataNamespace":""}},"originalId":["oai:services.nod.dans.knaw.nl:Products/dans:oai:easy.dans.knaw.nl:easy-dataset:20759","50|DansKnawCris::0dd644304b7116e8e58da3a5e3adc37a"],"pid":[],"relevantdate":[],"resourcetype":{"classid":"0021","classname":"0021","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"source":[],"subject":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"PROSPECTIE"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Archaeology"}],"title":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"Plangebied Lange Ekker te Vessem, gemeente Eersel"}]} -{"author":[],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":[{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"}],"context":[],"contributor":[],"country":[],"coverage":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofcollection":"2021-09-25T10:43:13.768Z","dateoftransformation":"2021-09-25T11:01:22.863Z","description":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"This find is registered at Portable Antiquities of the Netherlands with number PAN-00054604"}],"externalReference":[],"extraInfo":[],"format":[],"fulltext":[],"geolocation":[],"id":"50|DansKnawCris::203a27996ddc0fd1948258e5b7dec61c","instance":[{"accessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"urn","classname":"urn","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"urn:nbn:nl:ui:13-a7-hwgy"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"Digital Object Identifier","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.17026/dans-x3z-fsq5"}],"collectedfrom":{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"},"hostedby":{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"},"instancetype":{"classid":"0021","classname":"Dataset","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"alternateIdentifier":[],"refereed":{"classid":"0000","classname":"Unknown","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"url":["","http://dx.doi.org/10.17026/dans-x3z-fsq5"]}],"language":{"classid":"eng","classname":"English","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastupdatetimestamp":1635434508886,"oaiprovenance":{"originDescription":{"altered":true,"baseURL":"http%3A%2F%2Fservices.nod.dans.knaw.nl%2Foa-cerif","datestamp":"2021-08-16T14:01:37Z","harvestDate":"2021-09-25T10:43:13.768Z","identifier":"oai:services.nod.dans.knaw.nl:Products/dans:oai:easy.dans.knaw.nl:easy-dataset:129566","metadataNamespace":""}},"originalId":["oai:services.nod.dans.knaw.nl:Products/dans:oai:easy.dans.knaw.nl:easy-dataset:129566","50|DansKnawCris::203a27996ddc0fd1948258e5b7dec61c"],"pid":[],"relevantdate":[],"resourcetype":{"classid":"0021","classname":"0021","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"source":[],"subject":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"early medieval enamelled disc brooch variant A9"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Broader Match: disc brooches"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Broader Match: schijffibula - geemailleerd"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"metal"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"copper alloy"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Temporal coverage: Early Middle Ages C"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Temporal coverage: Early Middle Ages D"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Temporal coverage: 800 until 1000"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Archaeology"}],"title":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"PAN-00054604 - early medieval enamelled disc brooch variant A9"}]} \ No newline at end of file +{"author":[{"affiliation":[],"fullname":"Greenough, B","name":"B","pid":[],"rank":1,"surname":"Greenough"}],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":[{"key":"10|opendoar____::358aee4cc897452c00244351e4d91f69","value":"Zenodo"}],"context":[],"contributor":[],"country":[],"coverage":[],"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofcollection":"2021-09-25T10:55:00.639Z","dateoftransformation":"2021-09-25T11:00:04.201Z","description":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Heritage Education"}],"externalReference":[],"extraInfo":[],"format":[],"fulltext":[],"geolocation":[],"id":"50|doi_________::09821844208a5cd6300b2bfb13bca1b9","instance":[{"instanceTypeMapping":[{"originalType":"journal-article","vocabularyName":"openaire::coar_resource_types_3_1"}],"accessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"urn","classname":"urn","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"urn:nbn:nl:ui:13-59-cjhf"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"Digital Object Identifier","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.17632/96bpgw5j9d.1"}],"collectedfrom":{"key":"10|opendoar____::358aee4cc897452c00244351e4d91f69","value":"Zenodo"},"hostedby":{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"},"instancetype":{"classid":"0021","classname":"Dataset","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"alternateIdentifier":[],"refereed":{"classid":"0000","classname":"Unknown","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"url":["","http://dx.doi.org/10.17632/96bpgw5j9d.1"]}],"language":{"classid":"und","classname":"Undetermined","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastupdatetimestamp":1635434801681,"oaiprovenance":{"originDescription":{"altered":true,"baseURL":"http%3A%2F%2Fservices.nod.dans.knaw.nl%2Foa-cerif","datestamp":"2021-08-16T15:29:45Z","harvestDate":"2021-09-25T10:55:00.639Z","identifier":"oai:services.nod.dans.knaw.nl:Products/dans:oai:easy.dans.knaw.nl:easy-dataset:211323","metadataNamespace":""}},"originalId":["50|DansKnawCris::09821844208a5cd6300b2bfb13bca1b9","oai:services.nod.dans.knaw.nl:Products/dans:oai:easy.dans.knaw.nl:easy-dataset:211323"],"pid":[],"relevantdate":[],"resourcetype":{"classid":"0021","classname":"0021","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"source":[],"subject":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Interdisciplinary sciences"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Interdisciplinary sciences"}],"title":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"Heritage Education"}]} +{"author":[{"affiliation":[],"fullname":"Keijers, D.M.G.","name":"D.M.G.","pid":[],"rank":1,"surname":"Keijers"}],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":[{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"}],"context":[],"contributor":[],"country":[],"coverage":[],"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofcollection":"2021-09-25T10:41:59.767Z","dateoftransformation":"2021-09-25T11:00:19.238Z","description":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"onderzoeksrapport"}],"externalReference":[],"extraInfo":[],"format":[],"fulltext":[],"geolocation":[],"id":"50|DansKnawCris::0dd644304b7116e8e58da3a5e3adc37a","instance":[{"instanceTypeMapping":[{"originalType":"journal-article","vocabularyName":"openaire::coar_resource_types_3_1"}],"accessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"urn","classname":"urn","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"urn:nbn:nl:ui:13-das-fkq"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"Digital Object Identifier","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.17026/dans-xsw-qtnx"}],"collectedfrom":{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"},"hostedby":{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"},"instancetype":{"classid":"0021","classname":"Dataset","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"alternateIdentifier":[],"refereed":{"classid":"0000","classname":"Unknown","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"url":["","http://dx.doi.org/10.17026/dans-xsw-qtnx"]}],"language":{"classid":"dut/nld","classname":"Dutch; Flemish","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastupdatetimestamp":1635434847381,"oaiprovenance":{"originDescription":{"altered":true,"baseURL":"http%3A%2F%2Fservices.nod.dans.knaw.nl%2Foa-cerif","datestamp":"2021-08-16T13:53:29Z","harvestDate":"2021-09-25T10:41:59.767Z","identifier":"oai:services.nod.dans.knaw.nl:Products/dans:oai:easy.dans.knaw.nl:easy-dataset:20759","metadataNamespace":""}},"originalId":["oai:services.nod.dans.knaw.nl:Products/dans:oai:easy.dans.knaw.nl:easy-dataset:20759","50|DansKnawCris::0dd644304b7116e8e58da3a5e3adc37a"],"pid":[],"relevantdate":[],"resourcetype":{"classid":"0021","classname":"0021","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"source":[],"subject":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"PROSPECTIE"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Archaeology"}],"title":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"Plangebied Lange Ekker te Vessem, gemeente Eersel"}]} +{"author":[],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":[{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"}],"context":[],"contributor":[],"country":[],"coverage":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofcollection":"2021-09-25T10:43:13.768Z","dateoftransformation":"2021-09-25T11:01:22.863Z","description":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"This find is registered at Portable Antiquities of the Netherlands with number PAN-00054604"}],"externalReference":[],"extraInfo":[],"format":[],"fulltext":[],"geolocation":[],"id":"50|DansKnawCris::203a27996ddc0fd1948258e5b7dec61c","instance":[{"instanceTypeMapping":[{"originalType":"journal-article","vocabularyName":"openaire::coar_resource_types_3_1"}],"accessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"urn","classname":"urn","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"urn:nbn:nl:ui:13-a7-hwgy"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"Digital Object Identifier","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.17026/dans-x3z-fsq5"}],"collectedfrom":{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"},"hostedby":{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"},"instancetype":{"classid":"0021","classname":"Dataset","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"alternateIdentifier":[],"refereed":{"classid":"0000","classname":"Unknown","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"url":["","http://dx.doi.org/10.17026/dans-x3z-fsq5"]}],"language":{"classid":"eng","classname":"English","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastupdatetimestamp":1635434508886,"oaiprovenance":{"originDescription":{"altered":true,"baseURL":"http%3A%2F%2Fservices.nod.dans.knaw.nl%2Foa-cerif","datestamp":"2021-08-16T14:01:37Z","harvestDate":"2021-09-25T10:43:13.768Z","identifier":"oai:services.nod.dans.knaw.nl:Products/dans:oai:easy.dans.knaw.nl:easy-dataset:129566","metadataNamespace":""}},"originalId":["oai:services.nod.dans.knaw.nl:Products/dans:oai:easy.dans.knaw.nl:easy-dataset:129566","50|DansKnawCris::203a27996ddc0fd1948258e5b7dec61c"],"pid":[],"relevantdate":[],"resourcetype":{"classid":"0021","classname":"0021","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"dataset","classname":"dataset","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"source":[],"subject":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"early medieval enamelled disc brooch variant A9"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Broader Match: disc brooches"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Broader Match: schijffibula - geemailleerd"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"metal"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"copper alloy"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Temporal coverage: Early Middle Ages C"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Temporal coverage: Early Middle Ages D"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Temporal coverage: 800 until 1000"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Archaeology"}],"title":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"PAN-00054604 - early medieval enamelled disc brooch variant A9"}]} \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/group/publication/publication.json b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/group/publication/publication.json index 29ce76df3..90cf4936b 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/group/publication/publication.json +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/group/publication/publication.json @@ -1,3 +1,3 @@ -{"author":[{"affiliation":[],"fullname":"Greenough, B","name":"B","pid":[],"rank":1,"surname":"Greenough"}],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":[{"key":"10|openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"}],"context":[],"contributor":[],"country":[],"coverage":[],"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofcollection":"2021-09-25T10:55:00.639Z","dateoftransformation":"2021-09-25T11:00:04.201Z","description":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Heritage Education"}],"externalReference":[],"extraInfo":[],"format":[],"fulltext":[],"geolocation":[],"id":"50|doi_________::09821844208a5cd6300b2bfb13bca1b9","instance":[{"accessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"urn","classname":"urn","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"urn:nbn:nl:ui:13-59-cjhf"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"Digital Object Identifier","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.17632/96bpgw5j9d.1"}],"collectedfrom":{"key":"10|openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"},"hostedby":{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"},"instancetype":{"classid":"0021","classname":"Dataset","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"alternateIdentifier":[],"refereed":{"classid":"0000","classname":"Unknown","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"url":["","http://dx.doi.org/10.17632/96bpgw5j9d.1"]}],"language":{"classid":"und","classname":"Undetermined","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastupdatetimestamp":1635434801681,"oaiprovenance":{"originDescription":{"altered":true,"baseURL":"http%3A%2F%2Fservices.nod.dans.knaw.nl%2Foa-cerif","datestamp":"2021-08-16T15:29:45Z","harvestDate":"2021-09-25T10:55:00.639Z","identifier":"oai:services.nod.dans.knaw.nl:Products/dans:oai:easy.dans.knaw.nl:easy-dataset:211323","metadataNamespace":""}},"originalId":["50|DansKnawCris::09821844208a5cd6300b2bfb13bca1b9","oai:services.nod.dans.knaw.nl:Products/dans:oai:easy.dans.knaw.nl:easy-dataset:211323"],"pid":[],"relevantdate":[],"resourcetype":{"classid":"0021","classname":"0021","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"source":[],"subject":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Interdisciplinary sciences"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Interdisciplinary sciences"}],"title":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"Heritage Education"}]} -{"author":[{"affiliation":[],"fullname":"Keijers, D.M.G.","name":"D.M.G.","pid":[],"rank":1,"surname":"Keijers"}],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":[{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"}],"context":[],"contributor":[],"country":[],"coverage":[],"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofcollection":"2021-09-25T10:41:59.767Z","dateoftransformation":"2021-09-25T11:00:19.238Z","description":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"onderzoeksrapport"}],"externalReference":[],"extraInfo":[],"format":[],"fulltext":[],"geolocation":[],"id":"50|DansKnawCris::0dd644304b7116e8e58da3a5e3adc37a","instance":[{"accessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"urn","classname":"urn","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"urn:nbn:nl:ui:13-das-fkq"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"Digital Object Identifier","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.17026/dans-xsw-qtnx"}],"collectedfrom":{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"},"hostedby":{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"},"instancetype":{"classid":"0021","classname":"Dataset","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"alternateIdentifier":[],"refereed":{"classid":"0000","classname":"Unknown","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"url":["","http://dx.doi.org/10.17026/dans-xsw-qtnx"]}],"language":{"classid":"dut/nld","classname":"Dutch; Flemish","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastupdatetimestamp":1635434847381,"oaiprovenance":{"originDescription":{"altered":true,"baseURL":"http%3A%2F%2Fservices.nod.dans.knaw.nl%2Foa-cerif","datestamp":"2021-08-16T13:53:29Z","harvestDate":"2021-09-25T10:41:59.767Z","identifier":"oai:services.nod.dans.knaw.nl:Products/dans:oai:easy.dans.knaw.nl:easy-dataset:20759","metadataNamespace":""}},"originalId":["oai:services.nod.dans.knaw.nl:Products/dans:oai:easy.dans.knaw.nl:easy-dataset:20759","50|DansKnawCris::0dd644304b7116e8e58da3a5e3adc37a"],"pid":[],"relevantdate":[],"resourcetype":{"classid":"0021","classname":"0021","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"source":[],"subject":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"PROSPECTIE"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Archaeology"}],"title":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"Plangebied Lange Ekker te Vessem, gemeente Eersel"}]} -{"author":[],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":[{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"}],"context":[],"contributor":[],"country":[],"coverage":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofcollection":"2021-09-25T10:43:13.768Z","dateoftransformation":"2021-09-25T11:01:22.863Z","description":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"This find is registered at Portable Antiquities of the Netherlands with number PAN-00054604"}],"externalReference":[],"extraInfo":[],"format":[],"fulltext":[],"geolocation":[],"id":"50|DansKnawCris::203a27996ddc0fd1948258e5b7dec61c","instance":[{"accessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"urn","classname":"urn","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"urn:nbn:nl:ui:13-a7-hwgy"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"Digital Object Identifier","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.17026/dans-x3z-fsq5"}],"collectedfrom":{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"},"hostedby":{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"},"instancetype":{"classid":"0021","classname":"Dataset","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"alternateIdentifier":[],"refereed":{"classid":"0000","classname":"Unknown","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"url":["","http://dx.doi.org/10.17026/dans-x3z-fsq5"]}],"language":{"classid":"eng","classname":"English","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastupdatetimestamp":1635434508886,"oaiprovenance":{"originDescription":{"altered":true,"baseURL":"http%3A%2F%2Fservices.nod.dans.knaw.nl%2Foa-cerif","datestamp":"2021-08-16T14:01:37Z","harvestDate":"2021-09-25T10:43:13.768Z","identifier":"oai:services.nod.dans.knaw.nl:Products/dans:oai:easy.dans.knaw.nl:easy-dataset:129566","metadataNamespace":""}},"originalId":["oai:services.nod.dans.knaw.nl:Products/dans:oai:easy.dans.knaw.nl:easy-dataset:129566","50|DansKnawCris::203a27996ddc0fd1948258e5b7dec61c"],"pid":[],"relevantdate":[],"resourcetype":{"classid":"0021","classname":"0021","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"source":[],"subject":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"early medieval enamelled disc brooch variant A9"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Broader Match: disc brooches"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Broader Match: schijffibula - geemailleerd"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"metal"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"copper alloy"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Temporal coverage: Early Middle Ages C"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Temporal coverage: Early Middle Ages D"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Temporal coverage: 800 until 1000"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Archaeology"}],"title":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"PAN-00054604 - early medieval enamelled disc brooch variant A9"}]} \ No newline at end of file +{"author":[{"affiliation":[],"fullname":"Greenough, B","name":"B","pid":[],"rank":1,"surname":"Greenough"}],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":[{"key":"10|openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"}],"context":[],"contributor":[],"country":[],"coverage":[],"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofcollection":"2021-09-25T10:55:00.639Z","dateoftransformation":"2021-09-25T11:00:04.201Z","description":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"Heritage Education"}],"externalReference":[],"extraInfo":[],"format":[],"fulltext":[],"geolocation":[],"id":"50|doi_________::09821844208a5cd6300b2bfb13bca1b9","instance":[{"instanceTypeMapping":[{"originalType":"journal-article","vocabularyName":"openaire::coar_resource_types_3_1"}],"accessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"urn","classname":"urn","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"urn:nbn:nl:ui:13-59-cjhf"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"Digital Object Identifier","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.17632/96bpgw5j9d.1"}],"collectedfrom":{"key":"10|openaire____::9e3be59865b2c1c335d32dae2fe7b254","value":"Datacite"},"hostedby":{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"},"instancetype":{"classid":"0021","classname":"Dataset","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"alternateIdentifier":[],"refereed":{"classid":"0000","classname":"Unknown","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"url":["","http://dx.doi.org/10.17632/96bpgw5j9d.1"]}],"language":{"classid":"und","classname":"Undetermined","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastupdatetimestamp":1635434801681,"oaiprovenance":{"originDescription":{"altered":true,"baseURL":"http%3A%2F%2Fservices.nod.dans.knaw.nl%2Foa-cerif","datestamp":"2021-08-16T15:29:45Z","harvestDate":"2021-09-25T10:55:00.639Z","identifier":"oai:services.nod.dans.knaw.nl:Products/dans:oai:easy.dans.knaw.nl:easy-dataset:211323","metadataNamespace":""}},"originalId":["50|DansKnawCris::09821844208a5cd6300b2bfb13bca1b9","oai:services.nod.dans.knaw.nl:Products/dans:oai:easy.dans.knaw.nl:easy-dataset:211323"],"pid":[],"relevantdate":[],"resourcetype":{"classid":"0021","classname":"0021","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"source":[],"subject":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Interdisciplinary sciences"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Interdisciplinary sciences"}],"title":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"Heritage Education"}]} +{"author":[{"affiliation":[],"fullname":"Keijers, D.M.G.","name":"D.M.G.","pid":[],"rank":1,"surname":"Keijers"}],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":[{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"}],"context":[],"contributor":[],"country":[],"coverage":[],"dataInfo":{"deletedbyinference":true,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofcollection":"2021-09-25T10:41:59.767Z","dateoftransformation":"2021-09-25T11:00:19.238Z","description":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"onderzoeksrapport"}],"externalReference":[],"extraInfo":[],"format":[],"fulltext":[],"geolocation":[],"id":"50|DansKnawCris::0dd644304b7116e8e58da3a5e3adc37a","instance":[{"instanceTypeMapping":[{"originalType":"journal-article","vocabularyName":"openaire::coar_resource_types_3_1"}],"accessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"urn","classname":"urn","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"urn:nbn:nl:ui:13-das-fkq"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"Digital Object Identifier","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.17026/dans-xsw-qtnx"}],"collectedfrom":{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"},"hostedby":{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"},"instancetype":{"classid":"0021","classname":"Dataset","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"alternateIdentifier":[],"refereed":{"classid":"0000","classname":"Unknown","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"url":["","http://dx.doi.org/10.17026/dans-xsw-qtnx"]}],"language":{"classid":"dut/nld","classname":"Dutch; Flemish","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastupdatetimestamp":1635434847381,"oaiprovenance":{"originDescription":{"altered":true,"baseURL":"http%3A%2F%2Fservices.nod.dans.knaw.nl%2Foa-cerif","datestamp":"2021-08-16T13:53:29Z","harvestDate":"2021-09-25T10:41:59.767Z","identifier":"oai:services.nod.dans.knaw.nl:Products/dans:oai:easy.dans.knaw.nl:easy-dataset:20759","metadataNamespace":""}},"originalId":["oai:services.nod.dans.knaw.nl:Products/dans:oai:easy.dans.knaw.nl:easy-dataset:20759","50|DansKnawCris::0dd644304b7116e8e58da3a5e3adc37a"],"pid":[],"relevantdate":[],"resourcetype":{"classid":"0021","classname":"0021","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"source":[],"subject":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"PROSPECTIE"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Archaeology"}],"title":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"Plangebied Lange Ekker te Vessem, gemeente Eersel"}]} +{"author":[],"bestaccessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"collectedfrom":[{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"}],"context":[],"contributor":[],"country":[],"coverage":[],"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"dateofcollection":"2021-09-25T10:43:13.768Z","dateoftransformation":"2021-09-25T11:01:22.863Z","description":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"value":"This find is registered at Portable Antiquities of the Netherlands with number PAN-00054604"}],"externalReference":[],"extraInfo":[],"format":[],"fulltext":[],"geolocation":[],"id":"50|DansKnawCris::203a27996ddc0fd1948258e5b7dec61c","instance":[{"accessright":{"classid":"UNKNOWN","classname":"not available","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"pid":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"urn","classname":"urn","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"urn:nbn:nl:ui:13-a7-hwgy"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"doi","classname":"Digital Object Identifier","schemeid":"dnet:pid_types","schemename":"dnet:pid_types"},"value":"10.17026/dans-x3z-fsq5"}],"collectedfrom":{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"},"hostedby":{"key":"10|openaire____::c6df70599aa984f16ee52b4b86d2e89f","value":"DANS (Data Archiving and Networked Services)"},"instancetype":{"classid":"0019","classname":"Patent","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"alternateIdentifier":[],"refereed":{"classid":"0000","classname":"Unknown","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"url":["","http://dx.doi.org/10.17026/dans-x3z-fsq5"]}],"language":{"classid":"eng","classname":"English","schemeid":"dnet:languages","schemename":"dnet:languages"},"lastupdatetimestamp":1635434508886,"oaiprovenance":{"originDescription":{"altered":true,"baseURL":"http%3A%2F%2Fservices.nod.dans.knaw.nl%2Foa-cerif","datestamp":"2021-08-16T14:01:37Z","harvestDate":"2021-09-25T10:43:13.768Z","identifier":"oai:services.nod.dans.knaw.nl:Products/dans:oai:easy.dans.knaw.nl:easy-dataset:129566","metadataNamespace":""}},"originalId":["oai:services.nod.dans.knaw.nl:Products/dans:oai:easy.dans.knaw.nl:easy-dataset:129566","50|DansKnawCris::203a27996ddc0fd1948258e5b7dec61c"],"pid":[],"relevantdate":[],"resourcetype":{"classid":"0021","classname":"0021","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"resulttype":{"classid":"publication","classname":"publication","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"source":[],"subject":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"early medieval enamelled disc brooch variant A9"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Broader Match: disc brooches"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Broader Match: schijffibula - geemailleerd"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"metal"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"copper alloy"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Temporal coverage: Early Middle Ages C"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Temporal coverage: Early Middle Ages D"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Temporal coverage: 800 until 1000"},{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"value":"Archaeology"}],"title":[{"dataInfo":{"deletedbyinference":false,"inferenceprovenance":"","inferred":false,"invisible":false,"provenanceaction":{"classid":"sysimport:crosswalk:datasetarchive","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"},"trust":"0.9"},"qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"value":"PAN-00054604 - early medieval enamelled disc brooch variant A9"}]} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 0f1ec78af..3fd351c1d 100644 --- a/pom.xml +++ b/pom.xml @@ -888,7 +888,7 @@ 3.3.3 3.4.2 [2.12,3.0) - [4.17.2-SNAPSHOT] + [4.17.2] [4.0.3] [6.0.5] [3.1.6] From 359e81b7a60d987db509b7c53121863380c64aff Mon Sep 17 00:00:00 2001 From: dimitrispie Date: Thu, 23 Nov 2023 10:48:55 +0200 Subject: [PATCH 105/148] Update StatsAtomicActionsJob.java Bug fix for duplicate bronze checks --- .../actionmanager/stats_actionsets/StatsAtomicActionsJob.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsAtomicActionsJob.java b/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsAtomicActionsJob.java index e984d68fc..a64a63c24 100644 --- a/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsAtomicActionsJob.java +++ b/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsAtomicActionsJob.java @@ -138,10 +138,10 @@ public class StatsAtomicActionsJob implements Serializable { r.setPubliclyFunded(usm.isPublicly_funded()); if (usm.isIs_bronze_oa()) r.setOpenAccessColor(OpenAccessColor.bronze); - else if (usm.isIs_gold()) - r.setOpenAccessColor(OpenAccessColor.bronze); else if (usm.isIs_gold()) r.setOpenAccessColor(OpenAccessColor.gold); + else if (usm.isIs_hybrid()) + r.setOpenAccessColor(OpenAccessColor.hybrid); return r; }, Encoders.bean(Result.class)); } From 1ba582de3c9625417032294cd9212400d4e12663 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Thu, 23 Nov 2023 16:27:19 +0100 Subject: [PATCH 106/148] [graph cleaning] added cleaning for result.publisher and result.instance.license --- .../oaf/utils/GraphCleaningFunctions.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java index 7a8acbd36..d39798506 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java @@ -14,6 +14,7 @@ import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; +import eu.dnetlib.dhp.common.vocabulary.VocabularyTerm; import org.apache.commons.lang3.StringUtils; import com.github.sisyphsu.dateparser.DateParserUtils; @@ -28,6 +29,10 @@ import me.xuender.unidecode.Unidecode; public class GraphCleaningFunctions extends CleaningFunctions { + public static final String DNET_PUBLISHERS = "dnet:publishers"; + + public static final String DNET_LICENSES = "dnet:licenses"; + public static final String ORCID_CLEANING_REGEX = ".*([0-9]{4}).*[-–—−=].*([0-9]{4}).*[-–—−=].*([0-9]{4}).*[-–—−=].*([0-9x]{4})"; public static final int ORCID_LEN = 19; public static final String CLEANING_REGEX = "(?:\\n|\\r|\\t)"; @@ -407,6 +412,13 @@ public class GraphCleaningFunctions extends CleaningFunctions { .getPublisher() .getValue() .replaceAll(NAME_CLEANING_REGEX, " ")); + + if (vocs.vocabularyExists(DNET_PUBLISHERS)) { + vocs.find(DNET_PUBLISHERS) + .map(voc -> voc.getTermBySynonym(r.getPublisher().getValue())) + .map(VocabularyTerm::getName) + .ifPresent(publisher -> r.getPublisher().setValue(publisher)); + } } } if (Objects.isNull(r.getLanguage()) || StringUtils.isBlank(r.getLanguage().getClassid())) { @@ -567,6 +579,13 @@ public class GraphCleaningFunctions extends CleaningFunctions { i.setRefereed(qualifier("0000", "Unknown", ModelConstants.DNET_REVIEW_LEVELS)); } + if (Objects.nonNull(i.getLicense()) && Objects.nonNull(i.getLicense().getValue())) { + vocs.find(DNET_LICENSES) + .map(voc -> voc.getTermBySynonym(i.getLicense().getValue())) + .map(VocabularyTerm::getId) + .ifPresent(license -> i.getLicense().setValue(license)); + } + // from the script from Dimitris if ("0000".equals(i.getRefereed().getClassid())) { final boolean isFromCrossref = Optional From 1763d377ad4ccdbc4dce827f2474cfc3f626024c Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Thu, 23 Nov 2023 16:33:24 +0100 Subject: [PATCH 107/148] code formatting --- .../eu/dnetlib/pace/util/DiffPatchMatch.java | 1 - .../PrepareAffiliationRelationsTest.java | 4 +- .../PrepareResultCommunitySet.java | 152 ++++++------- .../ResultProjectList.java | 28 +-- .../SparkResultToCommunityFromProject.java | 206 +++++++++--------- .../ResultToCommunityJobTest.java | 158 +++++++------- 6 files changed, 274 insertions(+), 275 deletions(-) diff --git a/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java b/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java index 12c96500e..cfd9acd70 100644 --- a/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java +++ b/dhp-pace-core/src/main/java/eu/dnetlib/pace/util/DiffPatchMatch.java @@ -18,7 +18,6 @@ package eu.dnetlib.pace.util; * See the License for the specific language governing permissions and * limitations under the License. */ - /* * Diff Match and Patch * Copyright 2018 The diff-match-patch Authors. diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelationsTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelationsTest.java index e2639996c..b87738879 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelationsTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelationsTest.java @@ -79,8 +79,8 @@ public class PrepareAffiliationRelationsTest { .getPath(); String pubmedAffiliationRelationsPath = getClass() - .getResource("/eu/dnetlib/dhp/actionmanager/bipaffiliations/doi_to_ror.json") - .getPath(); + .getResource("/eu/dnetlib/dhp/actionmanager/bipaffiliations/doi_to_ror.json") + .getPath(); String outputPath = workingDir.toString() + "/actionSet"; diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java index 82625a122..7bee1ea0c 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java @@ -31,94 +31,94 @@ import scala.Tuple2; public class PrepareResultCommunitySet { - private static final Logger log = LoggerFactory.getLogger(PrepareResultCommunitySet.class); + private static final Logger log = LoggerFactory.getLogger(PrepareResultCommunitySet.class); - public static void main(String[] args) throws Exception { - String jsonConfiguration = IOUtils - .toString( - PrepareResultCommunitySet.class - .getResourceAsStream( - "/eu/dnetlib/dhp/resulttocommunityfromproject/input_preparecommunitytoresult_parameters.json")); + public static void main(String[] args) throws Exception { + String jsonConfiguration = IOUtils + .toString( + PrepareResultCommunitySet.class + .getResourceAsStream( + "/eu/dnetlib/dhp/resulttocommunityfromproject/input_preparecommunitytoresult_parameters.json")); - final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); - parser.parseArgument(args); + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + parser.parseArgument(args); - Boolean isSparkSessionManaged = isSparkSessionManaged(parser); - log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + Boolean isSparkSessionManaged = isSparkSessionManaged(parser); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); - String inputPath = parser.get("sourcePath"); - log.info("inputPath: {}", inputPath); + String inputPath = parser.get("sourcePath"); + log.info("inputPath: {}", inputPath); - final String outputPath = parser.get("outputPath"); - log.info("outputPath: {}", outputPath); + final String outputPath = parser.get("outputPath"); + log.info("outputPath: {}", outputPath); - final boolean production = Boolean.valueOf(parser.get("production")); - log.info("production: {}", production); + final boolean production = Boolean.valueOf(parser.get("production")); + log.info("production: {}", production); - final CommunityEntityMap projectsMap = Utils.getCommunityProjects(production); - // log.info("projectsMap: {}", new Gson().toJson(projectsMap)); + final CommunityEntityMap projectsMap = Utils.getCommunityProjects(production); + // log.info("projectsMap: {}", new Gson().toJson(projectsMap)); - SparkConf conf = new SparkConf(); + SparkConf conf = new SparkConf(); - runWithSparkSession( - conf, - isSparkSessionManaged, - spark -> { - removeOutputDir(spark, outputPath); - prepareInfo(spark, inputPath, outputPath, projectsMap); - }); - } + runWithSparkSession( + conf, + isSparkSessionManaged, + spark -> { + removeOutputDir(spark, outputPath); + prepareInfo(spark, inputPath, outputPath, projectsMap); + }); + } - private static void prepareInfo( - SparkSession spark, - String inputPath, - String outputPath, - CommunityEntityMap projectMap) { + private static void prepareInfo( + SparkSession spark, + String inputPath, + String outputPath, + CommunityEntityMap projectMap) { - final StructType structureSchema = new StructType() - .add( - "dataInfo", new StructType() - .add("deletedbyinference", DataTypes.BooleanType) - .add("invisible", DataTypes.BooleanType)) - .add("source", DataTypes.StringType) - .add("target", DataTypes.StringType) - .add("relClass", DataTypes.StringType); + final StructType structureSchema = new StructType() + .add( + "dataInfo", new StructType() + .add("deletedbyinference", DataTypes.BooleanType) + .add("invisible", DataTypes.BooleanType)) + .add("source", DataTypes.StringType) + .add("target", DataTypes.StringType) + .add("relClass", DataTypes.StringType); - spark - .read() - .schema(structureSchema) - .json(inputPath) - .filter( - "dataInfo.deletedbyinference != true " + - "and relClass == '" + ModelConstants.IS_PRODUCED_BY + "'") - .select( - new Column("source").as("resultId"), - new Column("target").as("projectId")) - .groupByKey((MapFunction) r -> (String) r.getAs("resultId"), Encoders.STRING()) - .mapGroups((MapGroupsFunction) (k, v) -> { - ResultProjectList rpl = new ResultProjectList(); - rpl.setResultId(k); - ArrayList cl = new ArrayList<>(); - cl.addAll(projectMap.get(v.next().getAs("projectId"))); - v.forEachRemaining(r -> { - projectMap - .get(r.getAs("projectId")) - .forEach(c -> { - if (!cl.contains(c)) - cl.add(c); - }); + spark + .read() + .schema(structureSchema) + .json(inputPath) + .filter( + "dataInfo.deletedbyinference != true " + + "and relClass == '" + ModelConstants.IS_PRODUCED_BY + "'") + .select( + new Column("source").as("resultId"), + new Column("target").as("projectId")) + .groupByKey((MapFunction) r -> (String) r.getAs("resultId"), Encoders.STRING()) + .mapGroups((MapGroupsFunction) (k, v) -> { + ResultProjectList rpl = new ResultProjectList(); + rpl.setResultId(k); + ArrayList cl = new ArrayList<>(); + cl.addAll(projectMap.get(v.next().getAs("projectId"))); + v.forEachRemaining(r -> { + projectMap + .get(r.getAs("projectId")) + .forEach(c -> { + if (!cl.contains(c)) + cl.add(c); + }); - }); - if (cl.size() == 0) - return null; - rpl.setCommunityList(cl); - return rpl; - }, Encoders.bean(ResultProjectList.class)) - .filter(Objects::nonNull) - .write() - .mode(SaveMode.Overwrite) - .option("compression", "gzip") - .json(outputPath); - } + }); + if (cl.size() == 0) + return null; + rpl.setCommunityList(cl); + return rpl; + }, Encoders.bean(ResultProjectList.class)) + .filter(Objects::nonNull) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(outputPath); + } } diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultProjectList.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultProjectList.java index ce96bbfac..44798a1f3 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultProjectList.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultProjectList.java @@ -5,22 +5,22 @@ import java.io.Serializable; import java.util.ArrayList; public class ResultProjectList implements Serializable { - private String resultId; - private ArrayList communityList; + private String resultId; + private ArrayList communityList; - public String getResultId() { - return resultId; - } + public String getResultId() { + return resultId; + } - public void setResultId(String resultId) { - this.resultId = resultId; - } + public void setResultId(String resultId) { + this.resultId = resultId; + } - public ArrayList getCommunityList() { - return communityList; - } + public ArrayList getCommunityList() { + return communityList; + } - public void setCommunityList(ArrayList communityList) { - this.communityList = communityList; - } + public void setCommunityList(ArrayList communityList) { + this.communityList = communityList; + } } diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java index 13202d79d..6e298cf94 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/SparkResultToCommunityFromProject.java @@ -37,127 +37,127 @@ import scala.Tuple2; * @Date 11/10/23 */ public class SparkResultToCommunityFromProject implements Serializable { - private static final Logger log = LoggerFactory.getLogger(SparkResultToCommunityFromProject.class); + private static final Logger log = LoggerFactory.getLogger(SparkResultToCommunityFromProject.class); - public static void main(String[] args) throws Exception { - String jsonConfiguration = IOUtils - .toString( - SparkResultToCommunityFromProject.class - .getResourceAsStream( - "/eu/dnetlib/dhp/resulttocommunityfromproject/input_communitytoresult_parameters.json")); + public static void main(String[] args) throws Exception { + String jsonConfiguration = IOUtils + .toString( + SparkResultToCommunityFromProject.class + .getResourceAsStream( + "/eu/dnetlib/dhp/resulttocommunityfromproject/input_communitytoresult_parameters.json")); - final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); + final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); - parser.parseArgument(args); + parser.parseArgument(args); - Boolean isSparkSessionManaged = isSparkSessionManaged(parser); - log.info("isSparkSessionManaged: {}", isSparkSessionManaged); + Boolean isSparkSessionManaged = isSparkSessionManaged(parser); + log.info("isSparkSessionManaged: {}", isSparkSessionManaged); - String inputPath = parser.get("sourcePath"); - log.info("inputPath: {}", inputPath); + String inputPath = parser.get("sourcePath"); + log.info("inputPath: {}", inputPath); - final String outputPath = parser.get("outputPath"); - log.info("outputPath: {}", outputPath); + final String outputPath = parser.get("outputPath"); + log.info("outputPath: {}", outputPath); - final String possibleupdatespath = parser.get("preparedInfoPath"); - log.info("preparedInfoPath: {}", possibleupdatespath); + final String possibleupdatespath = parser.get("preparedInfoPath"); + log.info("preparedInfoPath: {}", possibleupdatespath); - SparkConf conf = new SparkConf(); + SparkConf conf = new SparkConf(); - runWithSparkSession( - conf, - isSparkSessionManaged, - spark -> { + runWithSparkSession( + conf, + isSparkSessionManaged, + spark -> { - execPropagation(spark, inputPath, outputPath, possibleupdatespath); + execPropagation(spark, inputPath, outputPath, possibleupdatespath); - }); - } + }); + } - private static void execPropagation( - SparkSession spark, - String inputPath, - String outputPath, + private static void execPropagation( + SparkSession spark, + String inputPath, + String outputPath, - String possibleUpdatesPath) { + String possibleUpdatesPath) { - Dataset possibleUpdates = readPath(spark, possibleUpdatesPath, ResultProjectList.class); + Dataset possibleUpdates = readPath(spark, possibleUpdatesPath, ResultProjectList.class); - ModelSupport.entityTypes - .keySet() - .parallelStream() - .forEach(e -> { - if (ModelSupport.isResult(e)) { - removeOutputDir(spark, outputPath + e.name()); - Class resultClazz = ModelSupport.entityTypes.get(e); - Dataset result = readPath(spark, inputPath + e.name(), resultClazz); + ModelSupport.entityTypes + .keySet() + .parallelStream() + .forEach(e -> { + if (ModelSupport.isResult(e)) { + removeOutputDir(spark, outputPath + e.name()); + Class resultClazz = ModelSupport.entityTypes.get(e); + Dataset result = readPath(spark, inputPath + e.name(), resultClazz); - result - .joinWith( - possibleUpdates, - result.col("id").equalTo(possibleUpdates.col("resultId")), - "left_outer") - .map(resultCommunityFn(), Encoders.bean(resultClazz)) - .write() - .mode(SaveMode.Overwrite) - .option("compression", "gzip") - .json(outputPath + e.name()); - } - }); + result + .joinWith( + possibleUpdates, + result.col("id").equalTo(possibleUpdates.col("resultId")), + "left_outer") + .map(resultCommunityFn(), Encoders.bean(resultClazz)) + .write() + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(outputPath + e.name()); + } + }); - } + } - private static MapFunction, R> resultCommunityFn() { - return value -> { - R ret = value._1(); - Optional rcl = Optional.ofNullable(value._2()); - if (rcl.isPresent()) { - // ArrayList communitySet = rcl.get().getCommunityList(); - List contextList = ret - .getContext() - .stream() - .map(Context::getId) - .collect(Collectors.toList()); + private static MapFunction, R> resultCommunityFn() { + return value -> { + R ret = value._1(); + Optional rcl = Optional.ofNullable(value._2()); + if (rcl.isPresent()) { + // ArrayList communitySet = rcl.get().getCommunityList(); + List contextList = ret + .getContext() + .stream() + .map(Context::getId) + .collect(Collectors.toList()); - @SuppressWarnings("unchecked") - R res = (R) ret.getClass().newInstance(); + @SuppressWarnings("unchecked") + R res = (R) ret.getClass().newInstance(); - res.setId(ret.getId()); - List propagatedContexts = new ArrayList<>(); - for (String cId : rcl.get().getCommunityList()) { - if (!contextList.contains(cId)) { - Context newContext = new Context(); - newContext.setId(cId); - newContext - .setDataInfo( - Arrays - .asList( - getDataInfo( - PROPAGATION_DATA_INFO_TYPE, - PROPAGATION_RESULT_COMMUNITY_PROJECT_CLASS_ID, - PROPAGATION_RESULT_COMMUNITY_PROJECT_CLASS_NAME, - ModelConstants.DNET_PROVENANCE_ACTIONS))); - propagatedContexts.add(newContext); - } else { - ret - .getContext() - .stream() - .filter(c -> c.getId().equals(cId)) - .findFirst() - .get() - .getDataInfo() - .add( - getDataInfo( - PROPAGATION_DATA_INFO_TYPE, - PROPAGATION_RESULT_COMMUNITY_PROJECT_CLASS_ID, - PROPAGATION_RESULT_COMMUNITY_PROJECT_CLASS_NAME, - ModelConstants.DNET_PROVENANCE_ACTIONS)); - } - } - res.setContext(propagatedContexts); - ret.mergeFrom(res); - } - return ret; - }; - } + res.setId(ret.getId()); + List propagatedContexts = new ArrayList<>(); + for (String cId : rcl.get().getCommunityList()) { + if (!contextList.contains(cId)) { + Context newContext = new Context(); + newContext.setId(cId); + newContext + .setDataInfo( + Arrays + .asList( + getDataInfo( + PROPAGATION_DATA_INFO_TYPE, + PROPAGATION_RESULT_COMMUNITY_PROJECT_CLASS_ID, + PROPAGATION_RESULT_COMMUNITY_PROJECT_CLASS_NAME, + ModelConstants.DNET_PROVENANCE_ACTIONS))); + propagatedContexts.add(newContext); + } else { + ret + .getContext() + .stream() + .filter(c -> c.getId().equals(cId)) + .findFirst() + .get() + .getDataInfo() + .add( + getDataInfo( + PROPAGATION_DATA_INFO_TYPE, + PROPAGATION_RESULT_COMMUNITY_PROJECT_CLASS_ID, + PROPAGATION_RESULT_COMMUNITY_PROJECT_CLASS_NAME, + ModelConstants.DNET_PROVENANCE_ACTIONS)); + } + } + res.setContext(propagatedContexts); + ret.mergeFrom(res); + } + return ret; + }; + } } diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultToCommunityJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultToCommunityJobTest.java index 40b9745c5..5642501b3 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultToCommunityJobTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/resulttocommunityfromproject/ResultToCommunityJobTest.java @@ -31,103 +31,103 @@ import eu.dnetlib.dhp.schema.oaf.Dataset; public class ResultToCommunityJobTest { - private static final Logger log = LoggerFactory.getLogger(ResultToCommunityJobTest.class); + private static final Logger log = LoggerFactory.getLogger(ResultToCommunityJobTest.class); - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private static SparkSession spark; + private static SparkSession spark; - private static Path workingDir; + private static Path workingDir; - @BeforeAll - public static void beforeAll() throws IOException { - workingDir = Files.createTempDirectory(ResultToCommunityJobTest.class.getSimpleName()); - log.info("using work dir {}", workingDir); + @BeforeAll + public static void beforeAll() throws IOException { + workingDir = Files.createTempDirectory(ResultToCommunityJobTest.class.getSimpleName()); + log.info("using work dir {}", workingDir); - SparkConf conf = new SparkConf(); - conf.setAppName(ResultToCommunityJobTest.class.getSimpleName()); + SparkConf conf = new SparkConf(); + conf.setAppName(ResultToCommunityJobTest.class.getSimpleName()); - conf.setMaster("local[*]"); - conf.set("spark.driver.host", "localhost"); - conf.set("hive.metastore.local", "true"); - conf.set("spark.ui.enabled", "false"); - conf.set("spark.sql.warehouse.dir", workingDir.toString()); - conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString()); + conf.setMaster("local[*]"); + conf.set("spark.driver.host", "localhost"); + conf.set("hive.metastore.local", "true"); + conf.set("spark.ui.enabled", "false"); + conf.set("spark.sql.warehouse.dir", workingDir.toString()); + conf.set("hive.metastore.warehouse.dir", workingDir.resolve("warehouse").toString()); - spark = SparkSession - .builder() - .appName(OrcidPropagationJobTest.class.getSimpleName()) - .config(conf) - .getOrCreate(); - } + spark = SparkSession + .builder() + .appName(OrcidPropagationJobTest.class.getSimpleName()) + .config(conf) + .getOrCreate(); + } - @AfterAll - public static void afterAll() throws IOException { - FileUtils.deleteDirectory(workingDir.toFile()); - spark.stop(); - } + @AfterAll + public static void afterAll() throws IOException { + FileUtils.deleteDirectory(workingDir.toFile()); + spark.stop(); + } - @Test - void testSparkResultToCommunityFromProjectJob() throws Exception { - final String preparedInfoPath = getClass() - .getResource("/eu/dnetlib/dhp/resulttocommunityfromproject/preparedInfo") - .getPath(); - SparkResultToCommunityFromProject - .main( - new String[] { + @Test + void testSparkResultToCommunityFromProjectJob() throws Exception { + final String preparedInfoPath = getClass() + .getResource("/eu/dnetlib/dhp/resulttocommunityfromproject/preparedInfo") + .getPath(); + SparkResultToCommunityFromProject + .main( + new String[] { - "-isSparkSessionManaged", Boolean.FALSE.toString(), - "-sourcePath", getClass() - .getResource("/eu/dnetlib/dhp/resulttocommunityfromproject/sample/") - .getPath(), + "-isSparkSessionManaged", Boolean.FALSE.toString(), + "-sourcePath", getClass() + .getResource("/eu/dnetlib/dhp/resulttocommunityfromproject/sample/") + .getPath(), - "-outputPath", workingDir.toString() + "/", - "-preparedInfoPath", preparedInfoPath - }); + "-outputPath", workingDir.toString() + "/", + "-preparedInfoPath", preparedInfoPath + }); - final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); - JavaRDD tmp = sc - .textFile(workingDir.toString() + "/dataset") - .map(item -> OBJECT_MAPPER.readValue(item, Dataset.class)); + JavaRDD tmp = sc + .textFile(workingDir.toString() + "/dataset") + .map(item -> OBJECT_MAPPER.readValue(item, Dataset.class)); - Assertions.assertEquals(10, tmp.count()); - /** - * {"resultId":"50|57a035e5b1ae::d5be548ca7ae489d762f893be67af52f","communityList":["aurora"]} - * {"resultId":"50|57a035e5b1ae::a77232ffca9115fcad51c3503dbc7e3e","communityList":["aurora"]} - * {"resultId":"50|57a035e5b1ae::803aaad4decab7e27cd4b52a1931b3a1","communityList":["sdsn-gr"]} - * {"resultId":"50|57a035e5b1ae::a02e9e4087bca50687731ae5c765b5e1","communityList":["netherlands"]} - */ - List context = tmp - .filter(r -> r.getId().equals("50|57a035e5b1ae::d5be548ca7ae489d762f893be67af52f")) - .first() - .getContext(); - Assertions.assertTrue(context.stream().anyMatch(c -> containsResultCommunityProject(c))); + Assertions.assertEquals(10, tmp.count()); + /** + * {"resultId":"50|57a035e5b1ae::d5be548ca7ae489d762f893be67af52f","communityList":["aurora"]} + * {"resultId":"50|57a035e5b1ae::a77232ffca9115fcad51c3503dbc7e3e","communityList":["aurora"]} + * {"resultId":"50|57a035e5b1ae::803aaad4decab7e27cd4b52a1931b3a1","communityList":["sdsn-gr"]} + * {"resultId":"50|57a035e5b1ae::a02e9e4087bca50687731ae5c765b5e1","communityList":["netherlands"]} + */ + List context = tmp + .filter(r -> r.getId().equals("50|57a035e5b1ae::d5be548ca7ae489d762f893be67af52f")) + .first() + .getContext(); + Assertions.assertTrue(context.stream().anyMatch(c -> containsResultCommunityProject(c))); - context = tmp - .filter(r -> r.getId().equals("50|57a035e5b1ae::a77232ffca9115fcad51c3503dbc7e3e")) - .first() - .getContext(); - Assertions.assertTrue(context.stream().anyMatch(c -> containsResultCommunityProject(c))); + context = tmp + .filter(r -> r.getId().equals("50|57a035e5b1ae::a77232ffca9115fcad51c3503dbc7e3e")) + .first() + .getContext(); + Assertions.assertTrue(context.stream().anyMatch(c -> containsResultCommunityProject(c))); - Assertions - .assertEquals( - 0, tmp.filter(r -> r.getId().equals("50|57a035e5b1ae::803aaad4decab7e27cd4b52a1931b3a1")).count()); + Assertions + .assertEquals( + 0, tmp.filter(r -> r.getId().equals("50|57a035e5b1ae::803aaad4decab7e27cd4b52a1931b3a1")).count()); - Assertions - .assertEquals( - 0, tmp.filter(r -> r.getId().equals("50|57a035e5b1ae::a02e9e4087bca50687731ae5c765b5e1")).count()); + Assertions + .assertEquals( + 0, tmp.filter(r -> r.getId().equals("50|57a035e5b1ae::a02e9e4087bca50687731ae5c765b5e1")).count()); - Assertions - .assertEquals( - 2, tmp.filter(r -> r.getContext().stream().anyMatch(c -> c.getId().equals("aurora"))).count()); + Assertions + .assertEquals( + 2, tmp.filter(r -> r.getContext().stream().anyMatch(c -> c.getId().equals("aurora"))).count()); - } + } - private static boolean containsResultCommunityProject(Context c) { - return c - .getDataInfo() - .stream() - .anyMatch(di -> di.getProvenanceaction().getClassid().equals("result:community:project")); - } + private static boolean containsResultCommunityProject(Context c) { + return c + .getDataInfo() + .stream() + .anyMatch(di -> di.getProvenanceaction().getClassid().equals("result:community:project")); + } } From 34a4b3cbdfe3bc4d07b565b95089c89ddbd25f9c Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Fri, 24 Nov 2023 12:39:58 +0100 Subject: [PATCH 108/148] Implemented ORCID Enrichment --- .../eu/dnetlib/dhp/oa/merge/AuthorMerger.java | 292 +++++++++++++++++- .../eu/dnetlib/oa/merge/AuthorMergerTest.java | 125 ++++++++ .../collection/orcid/DownloadORCIDTest.java | 39 --- .../orcid/enrich_graph_orcid_parameters.json | 26 ++ .../enrich/orcid/oozie_app/config-default.xml | 34 ++ .../dhp/enrich/orcid/oozie_app/workflow.xml | 52 ++++ .../dhp/enrich/orcid/AuthorEnricher.scala | 37 +++ .../SparkEnrichGraphWithOrcidAuthors.scala | 119 +++++++ .../dhp/enrich/orcid/EnrichOrcidTest.scala | 12 + 9 files changed, 696 insertions(+), 40 deletions(-) create mode 100644 dhp-common/src/test/java/eu/dnetlib/oa/merge/AuthorMergerTest.java create mode 100644 dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/enrich/orcid/enrich_graph_orcid_parameters.json create mode 100644 dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/enrich/orcid/oozie_app/config-default.xml create mode 100644 dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/enrich/orcid/oozie_app/workflow.xml create mode 100644 dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/AuthorEnricher.scala create mode 100644 dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/SparkEnrichGraphWithOrcidAuthors.scala create mode 100644 dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/enrich/orcid/EnrichOrcidTest.scala diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java index aea046203..6c3058303 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java @@ -1,11 +1,18 @@ package eu.dnetlib.dhp.oa.merge; +import java.io.FileWriter; +import java.io.IOException; import java.text.Normalizer; import java.util.*; +import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.MutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.jetbrains.annotations.NotNull; import com.wcohen.ss.JaroWinkler; @@ -14,6 +21,28 @@ import eu.dnetlib.dhp.schema.oaf.StructuredProperty; import eu.dnetlib.pace.model.Person; import scala.Tuple2; +class SimilarityCellInfo implements Comparable { + + public int authorPosition = 0; + public int orcidPosition = 0; + + public double maxColumnSimilarity = 0.0; + + public SimilarityCellInfo() { + } + + public void setValues(final int authPos, final int orcidPos, final double similarity) { + this.authorPosition = authPos; + this.orcidPosition = orcidPos; + this.maxColumnSimilarity = similarity; + } + + @Override + public int compareTo(@NotNull SimilarityCellInfo o) { + return Double.compare(maxColumnSimilarity, o.maxColumnSimilarity); + } +} + public class AuthorMerger { private static final Double THRESHOLD = 0.95; @@ -119,6 +148,267 @@ public class AuthorMerger { }); } + public static String normalizeFullName(final String fullname) { + return nfd(fullname) + .toLowerCase() + // do not compact the regexes in a single expression, would cause StackOverflowError + // in case + // of large input strings + .replaceAll("(\\W)+", " ") + .replaceAll("(\\p{InCombiningDiacriticalMarks})+", " ") + .replaceAll("(\\p{Punct})+", " ") + .replaceAll("(\\d)+", " ") + .replaceAll("(\\n)+", " ") + .trim(); +// return Arrays.stream(fullname.split("[\\s | , | ;]+")).map(String::toLowerCase).sorted().collect(Collectors.joining()); + } + + private static String generateAuthorkey(final Author a) { + if (a.getSurname() == null) + return "NOSURNAME"; + + return normalize(a.getSurname()); + } + +// +// public static List enrichOrcid2(List baseAuthor, List orcidAuthor) { +// if (baseAuthor == null || baseAuthor.isEmpty()) +// return orcidAuthor; +// +// if (orcidAuthor == null || orcidAuthor.isEmpty()) +// return baseAuthor; +// +// if (baseAuthor.size() == 1 && orcidAuthor.size() > 10) +// return baseAuthor; +// +// +// Map> pubClusters = baseAuthor.stream().collect(Collectors.toMap(AuthorMerger::generateAuthorkey, Arrays::asList, (a, b) -> { +// a.addAll(b); +// return a; +// })); +// +// Map> orcidClusters = baseAuthor.stream().collect(Collectors.toMap(AuthorMerger::generateAuthorkey, Arrays::asList, (a, b) -> { +// a.addAll(b); +// return a; +// })); +// +// System.out.println(pubClusters.keySet().size()); +// System.out.println(orcidClusters.keySet().size()); +// +// +// +// +// return null; +// +// +// } + + static int hammingDist(String str1, String str2) { + if (str1.length() != str2.length()) + return Math.max(str1.length(), str2.length()); + int i = 0, count = 0; + while (i < str1.length()) { + if (str1.charAt(i) != str2.charAt(i)) + count++; + i++; + } + return count; + } + + private static String authorFieldToBeCompared(Author author) { + if (StringUtils.isNotBlank(author.getSurname())) { + return author.getSurname(); + + } + if (StringUtils.isNotBlank(author.getFullname())) { + return author.getFullname(); + } + return null; + } + + public static boolean checkSimilarity3(final Author left, final Author right) { + + if (StringUtils.isNotBlank(left.getSurname()) && StringUtils.isNotBlank(left.getName()) + && + StringUtils.isNotBlank(right.getSurname()) && StringUtils.isNotBlank(right.getName()) + + ) + return left.getSurname().equalsIgnoreCase(right.getSurname()) + && left.getName().substring(0, 1).equalsIgnoreCase(right.getName().substring(0, 1)); + + final Person pl = parse(left); + final Person pr = parse(right); + + // If one of them didn't have a surname the match is false + if (!(pl.getSurname() != null && pl.getSurname().stream().anyMatch(StringUtils::isNotBlank) && + pr.getSurname() != null && pr.getSurname().stream().anyMatch(StringUtils::isNotBlank))) + return false; + + // The Authors have one surname in common + if (pl.getSurname().stream().anyMatch(sl -> pr.getSurname().stream().anyMatch(sr -> sr.equalsIgnoreCase(sl)))) { + + // If one of them has only a surname and is the same we can say that they are the same author + if ((pl.getName() == null || pl.getName().stream().allMatch(StringUtils::isBlank)) || + (pr.getName() == null || pr.getName().stream().allMatch(StringUtils::isBlank))) + return true; + // The authors have the same initials of Name in common + if (pl + .getName() + .stream() + .anyMatch( + nl -> pr + .getName() + .stream() + .anyMatch(nr -> nr.substring(0, 1).equalsIgnoreCase(nl.substring(0, 1))))) + return true; + } + return false; + } + + public static boolean checkSimilarity2(final Author left, final Author right) { + final Person pl = parse(left); + final Person pr = parse(right); + + // If one of them didn't have a surname the match is false + if (!(pl.getSurname() != null && pl.getSurname().stream().anyMatch(StringUtils::isNotBlank) && + pr.getSurname() != null && pr.getSurname().stream().anyMatch(StringUtils::isNotBlank))) + return false; + + // The Authors have one surname in common + if (pl.getSurname().stream().anyMatch(sl -> pr.getSurname().stream().anyMatch(sr -> sr.equalsIgnoreCase(sl)))) { + + // If one of them has only a surname and is the same we can say that they are the same author + if ((pl.getName() == null || pl.getName().stream().allMatch(StringUtils::isBlank)) || + (pr.getName() == null || pr.getName().stream().allMatch(StringUtils::isBlank))) + return true; + // The authors have the same initials of Name in common + if (pl + .getName() + .stream() + .anyMatch( + nl -> pr + .getName() + .stream() + .anyMatch(nr -> nr.substring(0, 1).equalsIgnoreCase(nl.substring(0, 1))))) + return true; + } + return false; + } + + public static boolean checkSimilarity(final Author left, final Author right) { + + if (left.getSurname() == null && left.getFullname() == null) + return false; + if (right.getSurname() == null && right.getFullname() == null) + return false; + + // The Authors have the same surname, or we are tolerant from 1 different char(lets say 1 Typo) + if (StringUtils.isNotBlank(left.getSurname()) && StringUtils.isNotBlank(right.getSurname())) { + if (left.getSurname().equalsIgnoreCase(right.getSurname()) + || hammingDist(left.getSurname().toLowerCase(), right.getSurname().toLowerCase()) < 2) { + // IN case on of the two Authors has no given Name the match is true + if (StringUtils.isBlank(left.getName()) || StringUtils.isBlank(right.getName())) + return true; + // If the surname is correct, and they have the same name or the name starts with the same Letter we can + // say is the same author + if (left.getName().equalsIgnoreCase(right.getName()) + || left.getName().substring(0, 1).equalsIgnoreCase(right.getName().substring(0, 1))) + return true; + } + // Different SURNAME + else { + return false; + } + } else { + // This is the case where the two authors have or the surname or the fullname + // get the first not null of the surname or fullname of both + final String l = authorFieldToBeCompared(left); + final String r = authorFieldToBeCompared(right); + if (l == null || r == null) + return false; + // The same length means they are the same field + if (l.length() == r.length()) { + return normalize(l).equals(normalize(r)); + } + // In this case probably l contains the surname and r contains the fullname + if (l.length() < r.length()) + return normalize(r).contains(normalize(l)); + // In this case probably l contains the fullname and r contains the surname + return normalize(l).contains(normalize(r)); + } + return false; + } + + public static List enrichOrcid2(List baseAuthor, List orcidAuthor) { + + final Integer match_itm = 0; + if (baseAuthor == null || baseAuthor.isEmpty()) + return orcidAuthor; + + if (orcidAuthor == null || orcidAuthor.isEmpty()) + return baseAuthor; + + if (baseAuthor.size() == 1 && orcidAuthor.size() > 10) + return baseAuthor; + + final List oAuthor = new ArrayList<>(); + oAuthor.addAll(orcidAuthor); + + baseAuthor.forEach(ba -> { + Optional aMatch = oAuthor.stream().filter(oa -> checkSimilarity2(ba, oa)).findFirst(); + if (aMatch.isPresent()) { + final Author sameAuthor = aMatch.get(); + addPid(ba, sameAuthor.getPid()); + oAuthor.remove(sameAuthor); + } + }); + return baseAuthor; + } + + public static List enrichOrcid(List baseAuthor, List orcidAuthor) { + + if (baseAuthor == null || baseAuthor.isEmpty()) + return orcidAuthor; + + if (orcidAuthor == null || orcidAuthor.isEmpty()) + return baseAuthor; + + if (baseAuthor.size() == 1 && orcidAuthor.size() > 10) + return baseAuthor; + + final Double similarityMatrix[][] = new Double[baseAuthor.size()][orcidAuthor.size()]; + + final List maxColums = new ArrayList<>(); + + for (int i = 0; i < orcidAuthor.size(); i++) + maxColums.add(new SimilarityCellInfo()); + + for (int i = 0; i < baseAuthor.size(); i++) { + for (int j = 0; j < orcidAuthor.size(); j++) { + similarityMatrix[i][j] = sim(baseAuthor.get(i), orcidAuthor.get(j)); + if (maxColums.get(j).maxColumnSimilarity < similarityMatrix[i][j]) + maxColums.get(j).setValues(i, j, similarityMatrix[i][j]); + } + } + maxColums + .stream() + .sorted() + .filter(si -> si.maxColumnSimilarity > 0.85) + .forEach(si -> addPid(baseAuthor.get(si.authorPosition), orcidAuthor.get(si.orcidPosition).getPid())); + return baseAuthor; + + } + + private static void addPid(final Author a, final List pids) { + + if (a.getPid() == null) { + a.setPid(new ArrayList<>()); + } + + a.getPid().addAll(pids); + + } + public static String pidToComparableString(StructuredProperty pid) { final String classid = pid.getQualifier().getClassid() != null ? pid.getQualifier().getClassid().toLowerCase() : ""; @@ -171,7 +461,7 @@ public class AuthorMerger { } } - private static String normalize(final String s) { + public static String normalize(final String s) { String[] normalized = nfd(s) .toLowerCase() // do not compact the regexes in a single expression, would cause StackOverflowError diff --git a/dhp-common/src/test/java/eu/dnetlib/oa/merge/AuthorMergerTest.java b/dhp-common/src/test/java/eu/dnetlib/oa/merge/AuthorMergerTest.java new file mode 100644 index 000000000..7f8d673d1 --- /dev/null +++ b/dhp-common/src/test/java/eu/dnetlib/oa/merge/AuthorMergerTest.java @@ -0,0 +1,125 @@ + +package eu.dnetlib.oa.merge; + +import static org.junit.jupiter.api.Assertions.*; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import org.junit.jupiter.api.Test; +import org.junit.platform.commons.util.StringUtils; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +import eu.dnetlib.dhp.oa.merge.AuthorMerger; +import eu.dnetlib.dhp.schema.oaf.Author; + +public class AuthorMergerTest { + + @Test + public void testNormalization() { + + assertEquals("bruzzolasandro", AuthorMerger.normalizeFullName("Sandro, La Bruzzo")); + assertEquals("baglionimiriam", AuthorMerger.normalizeFullName("Miriam Baglioni")); + assertEquals("baglionimiriam", AuthorMerger.normalizeFullName("Miriam ;Baglioni,")); + + } + + public void testEnrcichAuthor() throws Exception { + final ObjectMapper mapper = new ObjectMapper(); + + BufferedReader pr = new BufferedReader(new InputStreamReader( + AuthorMergerTest.class.getResourceAsStream("/eu/dnetlib/dhp/oa/merge/authors_publication.json"))); + BufferedReader or = new BufferedReader(new InputStreamReader( + AuthorMergerTest.class.getResourceAsStream("/eu/dnetlib/dhp/oa/merge/authors_orcid.json"))); + + TypeReference> aclass = new TypeReference>() { + }; + String pubLine; + + int i = 0; + while ((pubLine = pr.readLine()) != null) { + final String pubId = pubLine; + final String MatchPidOrcid = or.readLine(); + final String pubOrcid = or.readLine(); + + final String data = pr.readLine(); + + if (StringUtils.isNotBlank(data)) { + List publicationAuthors = mapper.readValue(data, aclass); + List orcidAuthors = mapper.readValue(or.readLine(), aclass); + System.out.printf("OAF ID = %s \n", pubId); + System.out.printf("ORCID Intersected ID = %s \n", pubOrcid); + System.out.printf("OAF Author Size = %d \n", publicationAuthors.size()); + System.out.printf("Oricd Author Size = %d \n", orcidAuthors.size()); + System.out.printf("Oricd Matched PID = %s \n", MatchPidOrcid); + + long originalAuthorWithPiD = publicationAuthors + .stream() + .filter( + a -> a.getPid() != null && a + .getPid() + .stream() + .anyMatch( + p -> p.getQualifier() != null + && p.getQualifier().getClassid().toLowerCase().contains("orcid"))) + .count(); + long start = System.currentTimeMillis(); + +// final List enrichedList = AuthorMerger.enrichOrcid(publicationAuthors, orcidAuthors); + final List enrichedList = AuthorMerger.enrichOrcid2(publicationAuthors, orcidAuthors); + + long enrichedAuthorWithPid = enrichedList + .stream() + .filter( + a -> a.getPid() != null && a + .getPid() + .stream() + .anyMatch( + p -> p.getQualifier() != null + && p.getQualifier().getClassid().toLowerCase().contains("orcid"))) + .count(); + + long totalTime = (System.currentTimeMillis() - start) / 1000; + System.out + .printf( + "Enriched authors in %d seconds from %d pid to %d pid \n", totalTime, originalAuthorWithPiD, + enrichedAuthorWithPid); + + System.out.println("================="); + + if (++i > 30) + break; + } + + } + + } + + @Test + public void checkSimilarityTest() { + final Author left = new Author(); + left.setSurname("Wu"); + left.setName("M."); + left.setFullname("Wu, M."); + + System.out.println(AuthorMerger.normalizeFullName(left.getFullname())); + + final Author right = new Author(); + right.setName("Xin"); + right.setSurname("Wu"); + right.setFullname("Xin Wu"); +// System.out.println(AuthorMerger.normalize(right.getFullname())); + boolean same = AuthorMerger.checkSimilarity2(left, right); + + assertFalse(same); + + } + +} diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/orcid/DownloadORCIDTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/orcid/DownloadORCIDTest.java index be5555fc0..868f4e92d 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/orcid/DownloadORCIDTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/collection/orcid/DownloadORCIDTest.java @@ -32,45 +32,6 @@ import eu.dnetlib.dhp.parser.utility.VtdException; public class DownloadORCIDTest { private final Logger log = LoggerFactory.getLogger(DownloadORCIDTest.class); -// public void test() throws Exception { -// -// Configuration conf = new Configuration(); -// // Set FileSystem URI -//// conf.set("fs.defaultFS", "file://"); -// // Because of Maven -// conf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName()); -// conf.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName()); -// -// System.setProperty("hadoop.home.dir", "file:///Users/sandro/orcid/"); -// -// final FileSystem fileSystem = FileSystem.get(conf); -// -// new ExtractORCIDDump(fileSystem).run("/Users/sandro/orcid/", "/Users/sandro/orcid/extracted"); -// -//// final GZIPInputStream gzip = new GZIPInputStream(Files.newInputStream(Paths.get("/Users/sandro/orcid/ORCID_2023_10_activities_1.tar.gz"))); -//// try(final TarArchiveInputStream tais = new TarArchiveInputStream(gzip)) { -//// -//// TarArchiveEntry entry; -//// while ((entry = tais.getNextTarEntry()) != null) { -//// -//// if (entry.isFile() && entry.getName().contains("employments")) { -//// -//// System.out.println(entry.getName()); -//// final String [] items = entry.getName().split("/"); -//// -//// final String res = IOUtils.toString(new BufferedReader(new InputStreamReader(tais))); -//// System.out.println("res = " + res); -//// -//// System.out.println(items[items.length-2]); -//// break; -//// } -//// -//// -//// } -//// } -// -// } - @Test public void testSummary() throws Exception { final String xml = IOUtils diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/enrich/orcid/enrich_graph_orcid_parameters.json b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/enrich/orcid/enrich_graph_orcid_parameters.json new file mode 100644 index 000000000..765c0e8ff --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/enrich/orcid/enrich_graph_orcid_parameters.json @@ -0,0 +1,26 @@ +[ + { + "paramName": "mt", + "paramLongName": "master", + "paramDescription": "should be local or yarn", + "paramRequired": true + }, + { + "paramName": "op", + "paramLongName": "orcidPath", + "paramDescription": "the path of the orcid Table generated by the dump", + "paramRequired": true + }, + { + "paramName": "gp", + "paramLongName": "graphPath", + "paramDescription": "the path of the graph we want to apply enrichment", + "paramRequired": true + }, + { + "paramName": "tp", + "paramLongName": "targetPath", + "paramDescription": "the output path of the graph enriched", + "paramRequired": true + } +] \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/enrich/orcid/oozie_app/config-default.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/enrich/orcid/oozie_app/config-default.xml new file mode 100644 index 000000000..8a7bc8942 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/enrich/orcid/oozie_app/config-default.xml @@ -0,0 +1,34 @@ + + + jobTracker + yarnRM + + + nameNode + hdfs://nameservice1 + + + oozie.use.system.libpath + true + + + oozie.action.sharelib.for.spark + spark2 + + + hiveMetastoreUris + thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083 + + + hiveJdbcUrl + jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000 + + + hiveDbName + openaire + + + oozie.launcher.mapreduce.user.classpath.first + true + + \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/enrich/orcid/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/enrich/orcid/oozie_app/workflow.xml new file mode 100644 index 000000000..1284cceda --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/enrich/orcid/oozie_app/workflow.xml @@ -0,0 +1,52 @@ + + + + orcidPath + the path of the orcid Table generated by the dump + + + graphPath + the path of the graph we want to apply enrichment + + + targetPath + the output path of the graph enriched + + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + yarn + cluster + Enrich Graph with ORCID + eu.dnetlib.dhp.enrich.orcid.SparkEnrichGraphWithOrcidAuthors + dhp-graph-mapper-${projectVersion}.jar + + --executor-memory=${sparkExecutorMemory} + --executor-cores=${sparkExecutorCores} + --driver-memory=${sparkDriverMemory} + --conf spark.executor.memoryOverhead=2g + --conf spark.sql.shuffle.partitions=3000 + --conf spark.extraListeners=${spark2ExtraListeners} + --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} + --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} + + --orcidPath${orcidPath} + --targetPath${targetPath} + --graphPath${graphPath}/publication + --masteryarn + + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/AuthorEnricher.scala b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/AuthorEnricher.scala new file mode 100644 index 000000000..a67de4b95 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/AuthorEnricher.scala @@ -0,0 +1,37 @@ +package eu.dnetlib.dhp.enrich.orcid + +import eu.dnetlib.dhp.schema.oaf.{Author, Publication} +import eu.dnetlib.dhp.schema.sx.OafUtils +import org.apache.spark.sql.Row + +import scala.collection.JavaConverters._ + +object AuthorEnricher extends Serializable { + + def createAuthor(givenName: String, familyName: String, orcid: String): Author = { + val a = new Author + a.setName(givenName) + a.setSurname(familyName) + a.setFullname(s"$givenName $familyName") + a.setPid(List(OafUtils.createSP(orcid, "ORCID", "ORCID")).asJava) + a + + } + + def toOAFAuthor(r: Row): java.util.List[Author] = { + r.getList[Row](1) + .asScala + .map(s => createAuthor(s.getAs[String]("givenName"), s.getAs[String]("familyName"), s.getAs[String]("orcid"))) + .toList + .asJava + } + +// def enrichAuthor(p:Publication,r:Row): Unit = { +// val k:Map[String, OAuthor] =r.getList[Row](1).asScala.map(s => (s.getAs[String]("orcid"), OAuthor(s.getAs[String]("givenName") ,s.getAs[String]("familyName") ))).groupBy(_._1).mapValues(_.map(_._2).head) +// println(k) +// +// +// +// } + +} diff --git a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/SparkEnrichGraphWithOrcidAuthors.scala b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/SparkEnrichGraphWithOrcidAuthors.scala new file mode 100644 index 000000000..e190b2b33 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/SparkEnrichGraphWithOrcidAuthors.scala @@ -0,0 +1,119 @@ +package eu.dnetlib.dhp.enrich.orcid + +import com.fasterxml.jackson.databind.ObjectMapper +import eu.dnetlib.dhp.application.AbstractScalaApplication +import eu.dnetlib.dhp.oa.merge.AuthorMerger +import eu.dnetlib.dhp.schema.oaf.{Author, DataInfo, Instance, Publication, StructuredProperty} +import org.apache.spark.sql.{Dataset, Encoder, Encoders, Row, SaveMode, SparkSession} +import org.apache.spark.sql.functions.{col, collect_set, concat, explode, expr, first, flatten, lower, size, struct} +import org.slf4j.{Logger, LoggerFactory} +import org.apache.spark.sql.types._ + +class SparkEnrichGraphWithOrcidAuthors(propertyPath: String, args: Array[String], log: Logger) + extends AbstractScalaApplication(propertyPath, args, log: Logger) { + + /** Here all the spark applications runs this method + * where the whole logic of the spark node is defined + */ + override def run(): Unit = { + val graphPath = parser.get("graphPath") + log.info(s"graphPath is '$graphPath'") + val orcidPath = parser.get("orcidPath") + log.info(s"orcidPath is '$orcidPath'") + val targetPath = parser.get("targetPath") + log.info(s"targetPath is '$targetPath'") + enrichResult(spark, graphPath, orcidPath, targetPath) + } + + def enrichResult(spark: SparkSession, graphPath: String, orcidPath: String, outputPath: String): Unit = { + val orcidPublication = generateOrcidTable(spark, orcidPath) + implicit val publicationEncoder = Encoders.bean(classOf[Publication]) + + val aschema = new StructType() + .add("id", StringType) + .add("dataInfo", Encoders.bean(classOf[DataInfo]).schema) + .add( + "author",Encoders.bean(classOf[Author]).schema + + ) + + val schema = new StructType() + .add("id", StringType) + .add("dataInfo", Encoders.bean(classOf[DataInfo]).schema) + .add( + "instance", + ArrayType(new StructType().add("pid", ArrayType(Encoders.bean(classOf[StructuredProperty]).schema))) + ) + val entities = spark.read + .schema(schema) + .json(graphPath) + .where("datainfo.deletedbyinference = false") + .drop("datainfo") + .withColumn("instances", explode(col("instance"))) + .withColumn("pids", explode(col("instances.pid"))) + .select( + col("pids.qualifier.classid").alias("pid_schema"), + col("pids.value").alias("pid_value"), + col("id").alias("dnet_id") + ) + val orcidDnet = orcidPublication + .join( + entities, + lower(col("schema")).equalTo(lower(col("pid_schema"))) && + lower(col("value")).equalTo(lower(col("pid_value"))), + "inner" + ) + .groupBy(col("dnet_id")) + .agg(collect_set(orcidPublication("author")).alias("orcid_authors")) + .select("dnet_id", "orcid_authors") + .cache() + + val publication = spark.read.schema(publicationEncoder.schema).json(graphPath).as[Publication] + + publication + .joinWith(orcidDnet, publication("id").equalTo(orcidDnet("dnet_id")), "left") + .map { + case (p: Publication, null) => { + p + } + case (p: Publication, r: Row) => + p.setAuthor(AuthorMerger.enrichOrcid2(p.getAuthor, AuthorEnricher.toOAFAuthor(r))) + p + } + .write + .mode(SaveMode.Overwrite) + .option("compression", "gzip") + .json(outputPath) + } + + def generateOrcidTable(spark: SparkSession, inputPath: String): Dataset[Row] = { + val orcidAuthors = + spark.read.load(s"$inputPath/Authors").select("orcid", "familyName", "givenName", "creditName", "otherNames") + val orcidWorks = spark.read + .load(s"$inputPath/Works") + .select(col("orcid"), explode(col("pids")).alias("identifier")) + .where( + "identifier.schema = 'doi' or identifier.schema ='pmid' or identifier.schema ='pmc' or identifier.schema ='arxiv' or identifier.schema ='handle'" + ) + orcidAuthors + .join(orcidWorks, orcidAuthors("orcid").equalTo(orcidWorks("orcid"))) + .select( + col("identifier.schema").alias("schema"), + col("identifier.value").alias("value"), + struct(orcidAuthors("orcid").alias("orcid"), col("givenName"), col("familyName")).alias("author") + ) + } +} + +object SparkEnrichGraphWithOrcidAuthors { + + val log: Logger = LoggerFactory.getLogger(SparkEnrichGraphWithOrcidAuthors.getClass) + + def main(args: Array[String]): Unit = { + + new SparkEnrichGraphWithOrcidAuthors("/eu/dnetlib/dhp/enrich/orcid/enrich_graph_orcid_parameters.json", args, log) + .initialize() + .run() + + } +} diff --git a/dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/enrich/orcid/EnrichOrcidTest.scala b/dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/enrich/orcid/EnrichOrcidTest.scala new file mode 100644 index 000000000..0ddb7c0aa --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/enrich/orcid/EnrichOrcidTest.scala @@ -0,0 +1,12 @@ +package eu.dnetlib.dhp.enrich.orcid + +import org.apache.spark.SparkConf +import org.apache.spark.sql.SparkSession +import org.junit.jupiter.api.Test +import org.slf4j.{Logger, LoggerFactory} + +class EnrichOrcidTest { + + val log: Logger = LoggerFactory.getLogger(getClass) + +} From 48e0427a234b1bdee63f21d6c8879f10ceeba3da Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Mon, 27 Nov 2023 15:10:27 +0100 Subject: [PATCH 109/148] changed the parameter from production to baseURL. Fixed issue in tagging configuration --- .../eu/dnetlib/dhp/api/QueryCommunityAPI.java | 10 +- .../dnetlib/dhp/bulktag/SparkBulkTagJob.java | 1 + .../dhp/bulktag/input_bulkTag_parameters.json | 8 +- ...t_preparecommunitytoresult_parameters.json | 8 +- ...t_preparecommunitytoresult_parameters.json | 6 +- .../dnetlib/dhp/bulktag/BulkTagJobTest.java | 8 +- .../tagging_conf_remove.xml | 182 +++++++++--------- 7 files changed, 112 insertions(+), 111 deletions(-) diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java index dc10fce15..b7c0164c1 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java @@ -34,32 +34,32 @@ public class QueryCommunityAPI { public static String communities(String baseURL) throws IOException { - return get(baseURL + "community/communities"); + return get(baseURL + "communities"); } public static String community(String id, String baseURL ) throws IOException { - return get(baseURL + "community/" + id); + return get(baseURL + id); } public static String communityDatasource(String id, String baseURL ) throws IOException { - return get(baseURL + "community/" + id + "/contentproviders"); + return get(baseURL + id + "/contentproviders"); } public static String communityPropagationOrganization(String id, String baseURL ) throws IOException { - return get(baseURL + "community/" + id + "/propagationOrganizations"); + return get(baseURL + id + "/propagationOrganizations"); } public static String communityProjects(String id, String page, String size, String baseURL ) throws IOException { - return get(baseURL + "community/" + id + "/projects/" + page + "/" + size); + return get(baseURL + id + "/projects/" + page + "/" + size); } diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java index d972cecac..f01063955 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java @@ -80,6 +80,7 @@ public class SparkBulkTagJob { cc = CommunityConfigurationFactory.newInstance(taggingConf); } else { cc = Utils.getCommunityConfiguration(baseURL); + log.info(OBJECT_MAPPER.writeValueAsString(cc)); } runWithSparkSession( diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/input_bulkTag_parameters.json b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/input_bulkTag_parameters.json index dbe2d088f..d88904cd9 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/input_bulkTag_parameters.json +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/input_bulkTag_parameters.json @@ -34,10 +34,10 @@ }, { - "paramName": "p", - "paramLongName": "production", - "paramDescription": "this parameter is intended for testing purposes only. It is a possible tagging configuration obtained via the XQUERY. Intended to be removed", - "paramRequired": true + "paramName": "bu", + "paramLongName": "baseURL", + "paramDescription": "this parameter is to specify the api to be queried (beta or production)", + "paramRequired": false } ] \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/input_preparecommunitytoresult_parameters.json b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/input_preparecommunitytoresult_parameters.json index 8b6291e5d..3601db7ac 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/input_preparecommunitytoresult_parameters.json +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/input_preparecommunitytoresult_parameters.json @@ -24,10 +24,10 @@ "paramRequired": true }, { - "paramName": "p", - "paramLongName": "production", - "paramDescription": "the path used to store temporary output files", - "paramRequired": true + "paramName": "bu", + "paramLongName": "baseURL", + "paramDescription": "the base URL to the community API to use", + "paramRequired": false } ] \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/input_preparecommunitytoresult_parameters.json b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/input_preparecommunitytoresult_parameters.json index 9a50c79fa..cbc01c2d5 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/input_preparecommunitytoresult_parameters.json +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/input_preparecommunitytoresult_parameters.json @@ -19,10 +19,10 @@ "paramRequired": true }, { - "paramName": "p", - "paramLongName": "production", + "paramName": "bu", + "paramLongName": "baseURL", "paramDescription": "the path used to store temporary output files", - "paramRequired": true + "paramRequired": false } ] \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java index ba0c6b252..d193716a8 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java @@ -533,6 +533,7 @@ public class BulkTagJobTest { + "where MyD.inferenceprovenance = 'bulktagging'"; org.apache.spark.sql.Dataset idExplodeCommunity = spark.sql(query); + Assertions.assertEquals(7, idExplodeCommunity.count()); Assertions @@ -1572,11 +1573,10 @@ public class BulkTagJobTest { "-isSparkSessionManaged", Boolean.FALSE.toString(), "-sourcePath", getClass().getResource("/eu/dnetlib/dhp/bulktag/sample/dataset/no_updates/").getPath(), - "-taggingConf", taggingConf, - "-outputPath", workingDir.toString() + "/", - "-production", Boolean.TRUE.toString(), - "-pathMap", pathMap +// "-baseURL", "https://services.openaire.eu/openaire/community/", + "-pathMap", pathMap, + "-taggingConf", taggingConf }); final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/communityconfiguration/tagging_conf_remove.xml b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/communityconfiguration/tagging_conf_remove.xml index edd6c7e0a..21c7d452f 100644 --- a/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/communityconfiguration/tagging_conf_remove.xml +++ b/dhp-workflows/dhp-enrichment/src/test/resources/eu/dnetlib/dhp/bulktag/communityconfiguration/tagging_conf_remove.xml @@ -26,7 +26,7 @@ - re3data_____::a507cdacc5bbcc08761c92185dee5cab + 10|re3data_____::a507cdacc5bbcc08761c92185dee5cab @@ -140,39 +140,39 @@ - re3data_____::9ebe127e5f3a0bf401875690f3bb6b81 + 10|re3data_____::9ebe127e5f3a0bf401875690f3bb6b81 - doajarticles::c6cd4b532e12868c1d760a8d7cda6815 + 10|doajarticles::c6cd4b532e12868c1d760a8d7cda6815 - doajarticles::a6de4499bb87bf3c01add0a9e2c9ed0b + 10|doajarticles::a6de4499bb87bf3c01add0a9e2c9ed0b - doajarticles::6eb31d13b12bc06bbac06aef63cf33c9 + 10|doajarticles::6eb31d13b12bc06bbac06aef63cf33c9 - doajarticles::0da84e9dfdc8419576169e027baa8028 + 10|doajarticles::0da84e9dfdc8419576169e027baa8028 - re3data_____::84e123776089ce3c7a33db98d9cd15a8 + 10|re3data_____::84e123776089ce3c7a33db98d9cd15a8 - openaire____::c5502a43e76feab55dd00cf50f519125 + 10|openaire____::c5502a43e76feab55dd00cf50f519125 - re3data_____::a48f09c562b247a9919acfe195549b47 + 10|re3data_____::a48f09c562b247a9919acfe195549b47 - opendoar____::97275a23ca44226c9964043c8462be96 + 10|opendoar____::97275a23ca44226c9964043c8462be96 @@ -287,55 +287,55 @@ - doajarticles::8cec81178926caaca531afbd8eb5d64c + 10|doajarticles::8cec81178926caaca531afbd8eb5d64c - doajarticles::0f7a7f30b5400615cae1829f3e743982 + 10|doajarticles::0f7a7f30b5400615cae1829f3e743982 - doajarticles::9740f7f5af3e506d2ad2c215cdccd51a + 10|doajarticles::9740f7f5af3e506d2ad2c215cdccd51a - doajarticles::9f3fbaae044fa33cb7069b72935a3254 + 10|doajarticles::9f3fbaae044fa33cb7069b72935a3254 - doajarticles::cb67f33eb9819f5c624ce0313957f6b3 + 10|doajarticles::cb67f33eb9819f5c624ce0313957f6b3 - doajarticles::e21c97cbb7a209afc75703681c462906 + 10|doajarticles::e21c97cbb7a209afc75703681c462906 - doajarticles::554cde3be9e5c4588b4c4f9f503120cb + 10|doajarticles::554cde3be9e5c4588b4c4f9f503120cb - tubitakulakb::11e22f49e65b9fd11d5b144b93861a1b + 10|tubitakulakb::11e22f49e65b9fd11d5b144b93861a1b - doajarticles::57c5d3837da943e93b28ec4db82ec7a5 + 10|doajarticles::57c5d3837da943e93b28ec4db82ec7a5 - doajarticles::a186f5ddb8e8c7ecc992ef51cf3315b1 + 10|doajarticles::a186f5ddb8e8c7ecc992ef51cf3315b1 - doajarticles::e21c97cbb7a209afc75703681c462906 + 10|doajarticles::e21c97cbb7a209afc75703681c462906 - doajarticles::dca64612dfe0963fffc119098a319957 + 10|doajarticles::dca64612dfe0963fffc119098a319957 - doajarticles::dd70e44479f0ade25aa106aef3e87a0a + 10|doajarticles::dd70e44479f0ade25aa106aef3e87a0a @@ -406,27 +406,27 @@ - re3data_____::5b9bf9171d92df854cf3c520692e9122 + 10|re3data_____::5b9bf9171d92df854cf3c520692e9122 - doajarticles::c7d3de67dc77af72f6747157441252ec + 10|doajarticles::c7d3de67dc77af72f6747157441252ec - re3data_____::8515794670370f49c1d176c399c714f5 + 10|re3data_____::8515794670370f49c1d176c399c714f5 - doajarticles::d640648c84b10d425f96f11c3de468f3 + 10|doajarticles::d640648c84b10d425f96f11c3de468f3 - doajarticles::0c0e74daa5d95504eade9c81ebbd5b8a + 10|doajarticles::0c0e74daa5d95504eade9c81ebbd5b8a - rest________::fb1a3d4523c95e63496e3bc7ba36244b + 10|rest________::fb1a3d4523c95e63496e3bc7ba36244b @@ -743,27 +743,27 @@ - opendoar____::1a551829d50f1400b0dab21fdd969c04 + 10|opendoar____::1a551829d50f1400b0dab21fdd969c04 - opendoar____::49af6c4e558a7569d80eee2e035e2bd7 + 10|opendoar____::49af6c4e558a7569d80eee2e035e2bd7 - opendoar____::0266e33d3f546cb5436a10798e657d97 + 10|opendoar____::0266e33d3f546cb5436a10798e657d97 - opendoar____::fd4c2dc64ccb8496e6f1f94c85f30d06 + 10|opendoar____::fd4c2dc64ccb8496e6f1f94c85f30d06 - opendoar____::41bfd20a38bb1b0bec75acf0845530a7 + 10|opendoar____::41bfd20a38bb1b0bec75acf0845530a7 - opendoar____::87ae6fb631f7c8a627e8e28785d9992d + 10|opendoar____::87ae6fb631f7c8a627e8e28785d9992d @@ -983,11 +983,11 @@ - opendoar____::7e7757b1e12abcb736ab9a754ffb617a + 10|opendoar____::7e7757b1e12abcb736ab9a754ffb617a {"criteria":[{"constraint":[{"verb":"contains","field":"contributor","value":"DARIAH"}]}]} - opendoar____::96da2f590cd7246bbde0051047b0d6f7 + 10|opendoar____::96da2f590cd7246bbde0051047b0d6f7 {"criteria":[{"constraint":[{"verb":"contains","field":"contributor","value":"DARIAH"}]}]} @@ -1166,87 +1166,87 @@ - doajarticles::1c5bdf8fca58937894ad1441cca99b76 + 10|doajarticles::1c5bdf8fca58937894ad1441cca99b76 - doajarticles::b37a634324a45c821687e6e80e6f53b4 + 10|doajarticles::b37a634324a45c821687e6e80e6f53b4 - doajarticles::4bf64f2a104040e4e055cd9594b2d77c + 10|doajarticles::4bf64f2a104040e4e055cd9594b2d77c - doajarticles::479ca537c12755d1868bbf02938a900c + 10|doajarticles::479ca537c12755d1868bbf02938a900c - doajarticles::55f31df96a60e2309f45b7c265fcf7a2 + 10|doajarticles::55f31df96a60e2309f45b7c265fcf7a2 - doajarticles::c52a09891a5301f9986ebbfe3761810c + 10|doajarticles::c52a09891a5301f9986ebbfe3761810c - doajarticles::379807bc7f6c71a227ef1651462c414c + 10|doajarticles::379807bc7f6c71a227ef1651462c414c - doajarticles::36069db531a00b85a2e8fb301f4bdc19 + 10|doajarticles::36069db531a00b85a2e8fb301f4bdc19 - doajarticles::b6a898da311ded96fabf49c520b80d5d + 10|doajarticles::b6a898da311ded96fabf49c520b80d5d - doajarticles::d0753d9180b35a271d8b4a31f449749f + 10|doajarticles::d0753d9180b35a271d8b4a31f449749f - doajarticles::172050a92511838393a3fe237ae47e31 + 10|doajarticles::172050a92511838393a3fe237ae47e31 - doajarticles::301ed96c62abb160a3e29796efe5c95c + 10|doajarticles::301ed96c62abb160a3e29796efe5c95c - doajarticles::0f4f805b3d842f2c7f1b077c3426fa59 + 10|doajarticles::0f4f805b3d842f2c7f1b077c3426fa59 - doajarticles::ba73728b84437b8d48ae287b867c7215 + 10|doajarticles::ba73728b84437b8d48ae287b867c7215 - doajarticles::86faef424d804309ccf45f692523aa48 + 10|doajarticles::86faef424d804309ccf45f692523aa48 - doajarticles::73bd758fa41671de70964c3ecba013af + 10|doajarticles::73bd758fa41671de70964c3ecba013af - doajarticles::e661fc0bdb24af42b740a08f0ddc6cf4 + 10|doajarticles::e661fc0bdb24af42b740a08f0ddc6cf4 - doajarticles::a6d3052047d5dbfbd43d95b4afb0f3d7 + 10|doajarticles::a6d3052047d5dbfbd43d95b4afb0f3d7 - doajarticles::ca61df07089acc53a1569bde6673d82a + 10|doajarticles::ca61df07089acc53a1569bde6673d82a - doajarticles::237dd6f1606600459d0297abd8ed9976 + 10|doajarticles::237dd6f1606600459d0297abd8ed9976 - doajarticles::fba6191177ede7c51ea1cdf58eae7f8b + 10|doajarticles::fba6191177ede7c51ea1cdf58eae7f8b @@ -1345,87 +1345,87 @@ - doajarticles::c6f0ed5fa41e98863e7c73501fe4bd6d + 10|doajarticles::c6f0ed5fa41e98863e7c73501fe4bd6d - doajarticles::ae4c7286c79590f19fdca670156ce816 + 10|doajarticles::ae4c7286c79590f19fdca670156ce816 - doajarticles::0f664bce92ce953e0c7a92068c46bfb3 + 10|doajarticles::0f664bce92ce953e0c7a92068c46bfb3 - doajarticles::00017183dc4c858fb77541985323a4ef + 10|doajarticles::00017183dc4c858fb77541985323a4ef - doajarticles::93b306f458cce3d7aaaf58c0a725f4f9 + 10|doajarticles::93b306f458cce3d7aaaf58c0a725f4f9 - doajarticles::9dbf8fbf3e9fe0fe1fc01e55fbd90bfc + 10|doajarticles::9dbf8fbf3e9fe0fe1fc01e55fbd90bfc - doajarticles::a2bda8785c863279bba4b8f34827b4c9 + 10|doajarticles::a2bda8785c863279bba4b8f34827b4c9 - doajarticles::019a1fcb42c3fea1c1b689df76330b58 + 10|doajarticles::019a1fcb42c3fea1c1b689df76330b58 - doajarticles::0daa8281938831e9c82bfed8b55a2975 + 10|doajarticles::0daa8281938831e9c82bfed8b55a2975 - doajarticles::f67ad6d268162079b3abd51a24468744 + 10|doajarticles::f67ad6d268162079b3abd51a24468744 - doajarticles::c6f0ed5fa41e98863e7c73501fe4bd6d + 10|doajarticles::c6f0ed5fa41e98863e7c73501fe4bd6d - doajarticles::ad114356e196a4a3d84dda59c720dacd + 10|doajarticles::ad114356e196a4a3d84dda59c720dacd - doajarticles::01e8a54fdecaaf354c67a2dd74ae7d4f + 10|doajarticles::01e8a54fdecaaf354c67a2dd74ae7d4f - doajarticles::449305f096b10a9464449ff2d0e10e06 + 10|doajarticles::449305f096b10a9464449ff2d0e10e06 - doajarticles::982c0c0ac378256254cce2fa6572bb6c + 10|doajarticles::982c0c0ac378256254cce2fa6572bb6c - doajarticles::49d6ed47138884566ce93cf0ccb12c02 + 10|doajarticles::49d6ed47138884566ce93cf0ccb12c02 - doajarticles::a98e820dbc2e8ee0fc84ab66f263267c + 10|doajarticles::a98e820dbc2e8ee0fc84ab66f263267c - doajarticles::50b1ce37427b36368f8f0f1317e47f83 + 10|doajarticles::50b1ce37427b36368f8f0f1317e47f83 - doajarticles::f0ec29b7450b2ac5d0ad45327eeb531a + 10|doajarticles::f0ec29b7450b2ac5d0ad45327eeb531a - doajarticles::d8d421d3b0349a7aaa93758b27a54e84 + 10|doajarticles::d8d421d3b0349a7aaa93758b27a54e84 - doajarticles::7ffc35ac5133da01d421ccf8af5b70bc + 10|doajarticles::7ffc35ac5133da01d421ccf8af5b70bc @@ -1454,81 +1454,81 @@ - opendoar____::358aee4cc897452c00244351e4d91f69 + 10|opendoar____::358aee4cc897452c00244351e4d91f69 {"criteria":[{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"COVID-19"}]}, {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"SARS-CoV-2"}]}, {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"2019-nCoV"}}]} - re3data_____::7b0ad08687b2c960d5aeef06f811d5e6 + 10|re3data_____::7b0ad08687b2c960d5aeef06f811d5e6 {"criteria":[{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"COVID-19"}]}, {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"SARS-CoV-2"}]}, {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"2019-nCoV"}]}]} - driver______::bee53aa31dc2cbb538c10c2b65fa5824 + 10|driver______::bee53aa31dc2cbb538c10c2b65fa5824 {"criteria":[{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"COVID-19"}]}, {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"SARS-CoV-2"}]}, {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"2019-nCoV"}]}]} - openaire____::437f4b072b1aa198adcbc35910ff3b98 + 10|openaire____::437f4b072b1aa198adcbc35910ff3b98 {"criteria":[{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"COVID-19"}]}, {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"SARS-CoV-2"}]}, {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"2019-nCoV"}]}]} - openaire____::081b82f96300b6a6e3d282bad31cb6e2 + 10|openaire____::081b82f96300b6a6e3d282bad31cb6e2 {"criteria":[{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"COVID-19"}]}, {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"SARS-CoV-2"}]}, {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"2019-nCoV"}]}]} - openaire____::9e3be59865b2c1c335d32dae2fe7b254 + 10|openaire____::9e3be59865b2c1c335d32dae2fe7b254 {"criteria":[{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"COVID-19"}]}, {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"SARS-CoV-2"}]}, {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"2019-nCoV"}]}]} - opendoar____::8b6dd7db9af49e67306feb59a8bdc52c + 10|opendoar____::8b6dd7db9af49e67306feb59a8bdc52c {"criteria":[{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"COVID-19"}]}, {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"SARS-CoV-2"}]}, {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"2019-nCoV"}]}]} - share_______::4719356ec8d7d55d3feb384ce879ad6c + 10|share_______::4719356ec8d7d55d3feb384ce879ad6c {"criteria":[{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"COVID-19"}]}, {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"SARS-CoV-2"}]}, {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"2019-nCoV"}]}]} - share_______::bbd802baad85d1fd440f32a7a3a2c2b1 + 10|share_______::bbd802baad85d1fd440f32a7a3a2c2b1 {"criteria":[{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"COVID-19"}]}, {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"SARS-CoV-2"}]}, {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"2019-nCoV"}]}]} - opendoar____::6f4922f45568161a8cdf4ad2299f6d23 + 10|opendoar____::6f4922f45568161a8cdf4ad2299f6d23 {"criteria":[{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"COVID-19"}]}, {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"SARS-CoV-2"}]}, {"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"2019-nCoV"}]}]} - re3data_____::7980778c78fb4cf0fab13ce2159030dc + 10|re3data_____::7980778c78fb4cf0fab13ce2159030dc {"criteria":[{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"SARS-CoV-2"}]},{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"COVID-19"}]},{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"2019-nCov"}]}]} - re3data_____::978378def740bbf2bfb420de868c460b + 10|re3data_____::978378def740bbf2bfb420de868c460b {"criteria":[{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"SARS-CoV-2"}]},{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"COVID-19"}]},{"constraint":[{"verb":"contains_caseinsensitive","field":"title","value":"2019-nCov"}]}]} From 8eb70e6657870e0d661a5c8619076bc013eb23ee Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Mon, 27 Nov 2023 15:13:15 +0100 Subject: [PATCH 110/148] refactoring --- .../PrepareAffiliationRelationsTest.java | 4 ++-- .../eu/dnetlib/dhp/api/QueryCommunityAPI.java | 21 ++++++++----------- .../PrepareResultCommunitySet.java | 2 +- .../dnetlib/dhp/bulktag/BulkTagJobTest.java | 2 +- 4 files changed, 13 insertions(+), 16 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelationsTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelationsTest.java index e2639996c..b87738879 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelationsTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelationsTest.java @@ -79,8 +79,8 @@ public class PrepareAffiliationRelationsTest { .getPath(); String pubmedAffiliationRelationsPath = getClass() - .getResource("/eu/dnetlib/dhp/actionmanager/bipaffiliations/doi_to_ror.json") - .getPath(); + .getResource("/eu/dnetlib/dhp/actionmanager/bipaffiliations/doi_to_ror.json") + .getPath(); String outputPath = workingDir.toString() + "/actionSet"; diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java index b7c0164c1..cf33c6509 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/api/QueryCommunityAPI.java @@ -15,8 +15,6 @@ import org.jetbrains.annotations.NotNull; */ public class QueryCommunityAPI { - - private static String get(String geturl) throws IOException { URL url = new URL(geturl); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); @@ -34,32 +32,31 @@ public class QueryCommunityAPI { public static String communities(String baseURL) throws IOException { - return get(baseURL + "communities"); + return get(baseURL + "communities"); } - public static String community(String id, String baseURL ) throws IOException { + public static String community(String id, String baseURL) throws IOException { - return get(baseURL + id); + return get(baseURL + id); } - public static String communityDatasource(String id, String baseURL ) throws IOException { - - return get(baseURL + id + "/contentproviders"); + public static String communityDatasource(String id, String baseURL) throws IOException { + return get(baseURL + id + "/contentproviders"); } - public static String communityPropagationOrganization(String id, String baseURL ) throws IOException { + public static String communityPropagationOrganization(String id, String baseURL) throws IOException { - return get(baseURL + id + "/propagationOrganizations"); + return get(baseURL + id + "/propagationOrganizations"); } - public static String communityProjects(String id, String page, String size, String baseURL ) throws IOException { + public static String communityProjects(String id, String page, String size, String baseURL) throws IOException { - return get(baseURL + id + "/projects/" + page + "/" + size); + return get(baseURL + id + "/projects/" + page + "/" + size); } diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java index 2c2eb69dd..7fed2606b 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java @@ -52,7 +52,7 @@ public class PrepareResultCommunitySet { final String outputPath = parser.get("outputPath"); log.info("outputPath: {}", outputPath); - final String baseURL = parser.get("baseURL"); + final String baseURL = parser.get("baseURL"); log.info("baseUEL: {}", baseURL); final CommunityEntityMap projectsMap = Utils.getCommunityProjects(baseURL); diff --git a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java index d193716a8..743e62b5f 100644 --- a/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java +++ b/dhp-workflows/dhp-enrichment/src/test/java/eu/dnetlib/dhp/bulktag/BulkTagJobTest.java @@ -1576,7 +1576,7 @@ public class BulkTagJobTest { "-outputPath", workingDir.toString() + "/", // "-baseURL", "https://services.openaire.eu/openaire/community/", "-pathMap", pathMap, - "-taggingConf", taggingConf + "-taggingConf", taggingConf }); final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); From 6f4d0c05eae2965d5b2afde1d78627cbf2ebbd63 Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Tue, 28 Nov 2023 08:43:56 +0100 Subject: [PATCH 111/148] Implemented Author MErger for ORCID that takes in account the case when name and surname are swapped --- .../eu/dnetlib/dhp/oa/merge/AuthorMerger.java | 114 ++++-------------- .../eu/dnetlib/oa/merge/AuthorMergerTest.java | 33 ++--- .../dhp/oa/merge/authors_orcid_sample.json | 3 + .../oa/merge/authors_publication_sample.json | 2 + .../SparkEnrichGraphWithOrcidAuthors.scala | 15 ++- 5 files changed, 51 insertions(+), 116 deletions(-) create mode 100644 dhp-common/src/test/resources/eu/dnetlib/dhp/oa/merge/authors_orcid_sample.json create mode 100644 dhp-common/src/test/resources/eu/dnetlib/dhp/oa/merge/authors_publication_sample.json diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java index 6c3058303..852ee163d 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java @@ -159,17 +159,11 @@ public class AuthorMerger { .replaceAll("(\\p{Punct})+", " ") .replaceAll("(\\d)+", " ") .replaceAll("(\\n)+", " ") + .trim(); // return Arrays.stream(fullname.split("[\\s | , | ;]+")).map(String::toLowerCase).sorted().collect(Collectors.joining()); } - private static String generateAuthorkey(final Author a) { - if (a.getSurname() == null) - return "NOSURNAME"; - - return normalize(a.getSurname()); - } - // // public static List enrichOrcid2(List baseAuthor, List orcidAuthor) { // if (baseAuthor == null || baseAuthor.isEmpty()) @@ -226,54 +220,26 @@ public class AuthorMerger { return null; } - public static boolean checkSimilarity3(final Author left, final Author right) { - - if (StringUtils.isNotBlank(left.getSurname()) && StringUtils.isNotBlank(left.getName()) - && - StringUtils.isNotBlank(right.getSurname()) && StringUtils.isNotBlank(right.getName()) - - ) - return left.getSurname().equalsIgnoreCase(right.getSurname()) - && left.getName().substring(0, 1).equalsIgnoreCase(right.getName().substring(0, 1)); - - final Person pl = parse(left); - final Person pr = parse(right); - - // If one of them didn't have a surname the match is false - if (!(pl.getSurname() != null && pl.getSurname().stream().anyMatch(StringUtils::isNotBlank) && - pr.getSurname() != null && pr.getSurname().stream().anyMatch(StringUtils::isNotBlank))) - return false; - - // The Authors have one surname in common - if (pl.getSurname().stream().anyMatch(sl -> pr.getSurname().stream().anyMatch(sr -> sr.equalsIgnoreCase(sl)))) { - - // If one of them has only a surname and is the same we can say that they are the same author - if ((pl.getName() == null || pl.getName().stream().allMatch(StringUtils::isBlank)) || - (pr.getName() == null || pr.getName().stream().allMatch(StringUtils::isBlank))) - return true; - // The authors have the same initials of Name in common - if (pl - .getName() - .stream() - .anyMatch( - nl -> pr - .getName() - .stream() - .anyMatch(nr -> nr.substring(0, 1).equalsIgnoreCase(nl.substring(0, 1))))) - return true; - } - return false; - } - public static boolean checkSimilarity2(final Author left, final Author right) { final Person pl = parse(left); final Person pr = parse(right); - // If one of them didn't have a surname the match is false + // If one of them didn't have a surname we verify if they have the fullName not empty + // and verify if the normalized version is equal if (!(pl.getSurname() != null && pl.getSurname().stream().anyMatch(StringUtils::isNotBlank) && - pr.getSurname() != null && pr.getSurname().stream().anyMatch(StringUtils::isNotBlank))) - return false; + pr.getSurname() != null && pr.getSurname().stream().anyMatch(StringUtils::isNotBlank))) { + if (pl.getFullname() != null && !pl.getFullname().isEmpty() && pr.getFullname() != null + && !pr.getFullname().isEmpty()) { + return pl + .getFullname() + .stream() + .anyMatch( + fl -> pr.getFullname().stream().anyMatch(fr -> normalize(fl).equalsIgnoreCase(normalize(fr)))); + } else { + return false; + } + } // The Authors have one surname in common if (pl.getSurname().stream().anyMatch(sl -> pr.getSurname().stream().anyMatch(sr -> sr.equalsIgnoreCase(sl)))) { @@ -292,56 +258,18 @@ public class AuthorMerger { .anyMatch(nr -> nr.substring(0, 1).equalsIgnoreCase(nl.substring(0, 1))))) return true; } - return false; - } - public static boolean checkSimilarity(final Author left, final Author right) { - - if (left.getSurname() == null && left.getFullname() == null) + // Sometimes we noticed that publication have author wrote in inverse order Surname, Name + // We verify if we have an exact match between name and surname + if (pl.getSurname().stream().anyMatch(sl -> pr.getName().stream().anyMatch(nr -> nr.equalsIgnoreCase(sl))) && + pl.getName().stream().anyMatch(nl -> pr.getSurname().stream().anyMatch(sr -> sr.equalsIgnoreCase(nl)))) + return true; + else return false; - if (right.getSurname() == null && right.getFullname() == null) - return false; - - // The Authors have the same surname, or we are tolerant from 1 different char(lets say 1 Typo) - if (StringUtils.isNotBlank(left.getSurname()) && StringUtils.isNotBlank(right.getSurname())) { - if (left.getSurname().equalsIgnoreCase(right.getSurname()) - || hammingDist(left.getSurname().toLowerCase(), right.getSurname().toLowerCase()) < 2) { - // IN case on of the two Authors has no given Name the match is true - if (StringUtils.isBlank(left.getName()) || StringUtils.isBlank(right.getName())) - return true; - // If the surname is correct, and they have the same name or the name starts with the same Letter we can - // say is the same author - if (left.getName().equalsIgnoreCase(right.getName()) - || left.getName().substring(0, 1).equalsIgnoreCase(right.getName().substring(0, 1))) - return true; - } - // Different SURNAME - else { - return false; - } - } else { - // This is the case where the two authors have or the surname or the fullname - // get the first not null of the surname or fullname of both - final String l = authorFieldToBeCompared(left); - final String r = authorFieldToBeCompared(right); - if (l == null || r == null) - return false; - // The same length means they are the same field - if (l.length() == r.length()) { - return normalize(l).equals(normalize(r)); - } - // In this case probably l contains the surname and r contains the fullname - if (l.length() < r.length()) - return normalize(r).contains(normalize(l)); - // In this case probably l contains the fullname and r contains the surname - return normalize(l).contains(normalize(r)); - } - return false; } public static List enrichOrcid2(List baseAuthor, List orcidAuthor) { - final Integer match_itm = 0; if (baseAuthor == null || baseAuthor.isEmpty()) return orcidAuthor; diff --git a/dhp-common/src/test/java/eu/dnetlib/oa/merge/AuthorMergerTest.java b/dhp-common/src/test/java/eu/dnetlib/oa/merge/AuthorMergerTest.java index 7f8d673d1..9eccab5f1 100644 --- a/dhp-common/src/test/java/eu/dnetlib/oa/merge/AuthorMergerTest.java +++ b/dhp-common/src/test/java/eu/dnetlib/oa/merge/AuthorMergerTest.java @@ -9,6 +9,7 @@ import java.io.IOException; import java.io.InputStreamReader; import java.util.Arrays; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; import org.junit.jupiter.api.Test; @@ -22,22 +23,15 @@ import eu.dnetlib.dhp.schema.oaf.Author; public class AuthorMergerTest { + @Test - public void testNormalization() { - - assertEquals("bruzzolasandro", AuthorMerger.normalizeFullName("Sandro, La Bruzzo")); - assertEquals("baglionimiriam", AuthorMerger.normalizeFullName("Miriam Baglioni")); - assertEquals("baglionimiriam", AuthorMerger.normalizeFullName("Miriam ;Baglioni,")); - - } - public void testEnrcichAuthor() throws Exception { final ObjectMapper mapper = new ObjectMapper(); BufferedReader pr = new BufferedReader(new InputStreamReader( - AuthorMergerTest.class.getResourceAsStream("/eu/dnetlib/dhp/oa/merge/authors_publication.json"))); + Objects.requireNonNull(AuthorMergerTest.class.getResourceAsStream("/eu/dnetlib/dhp/oa/merge/authors_publication_sample.json")))); BufferedReader or = new BufferedReader(new InputStreamReader( - AuthorMergerTest.class.getResourceAsStream("/eu/dnetlib/dhp/oa/merge/authors_orcid.json"))); + Objects.requireNonNull(AuthorMergerTest.class.getResourceAsStream("/eu/dnetlib/dhp/oa/merge/authors_orcid_sample.json")))); TypeReference> aclass = new TypeReference>() { }; @@ -93,32 +87,27 @@ public class AuthorMergerTest { enrichedAuthorWithPid); System.out.println("================="); - - if (++i > 30) - break; } - } - } @Test public void checkSimilarityTest() { final Author left = new Author(); - left.setSurname("Wu"); - left.setName("M."); - left.setFullname("Wu, M."); + left.setName("Anand"); + left.setSurname("Rachna"); + left.setFullname("Anand, Rachna"); System.out.println(AuthorMerger.normalizeFullName(left.getFullname())); final Author right = new Author(); - right.setName("Xin"); - right.setSurname("Wu"); - right.setFullname("Xin Wu"); + right.setName("Rachna"); + right.setSurname("Anand"); + right.setFullname("Rachna, Anand"); // System.out.println(AuthorMerger.normalize(right.getFullname())); boolean same = AuthorMerger.checkSimilarity2(left, right); - assertFalse(same); + assertTrue(same); } diff --git a/dhp-common/src/test/resources/eu/dnetlib/dhp/oa/merge/authors_orcid_sample.json b/dhp-common/src/test/resources/eu/dnetlib/dhp/oa/merge/authors_orcid_sample.json new file mode 100644 index 000000000..ec521b3b7 --- /dev/null +++ b/dhp-common/src/test/resources/eu/dnetlib/dhp/oa/merge/authors_orcid_sample.json @@ -0,0 +1,3 @@ +WrappedArray(arXiv1507.08202) +50|arXiv_dedup_::34e03f2336b8b28286550425e65634ea +[{"fullname":"Liron Barak","name":"Liron","surname":"Barak","rank":null,"pid":[{"value":"0000-0002-3436-2726","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Giuseppe Avolio","name":"Giuseppe","surname":"Avolio","rank":null,"pid":[{"value":"0000-0003-2664-3437","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Tobias Golling","name":"Tobias","surname":"Golling","rank":null,"pid":[{"value":"0000-0001-8535-6687","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Ivan Sykora","name":"Ivan","surname":"Sykora","rank":null,"pid":[{"value":"0000-0003-3447-5621","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Joao Carvalho","name":"Joao","surname":"Carvalho","rank":null,"pid":[{"value":"0000-0002-3015-7821","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Jonathan David Long","name":"Jonathan David","surname":"Long","rank":null,"pid":[{"value":"0000-0002-2115-9382","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Heather Russell","name":"Heather","surname":"Russell","rank":null,"pid":[{"value":"0000-0003-4181-0678","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Alison Lister","name":"Alison","surname":"Lister","rank":null,"pid":[{"value":"0000-0002-1552-3651","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Malte Backhaus","name":"Malte","surname":"Backhaus","rank":null,"pid":[{"value":"0000-0002-5888-2304","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Xin Wu","name":"Xin","surname":"Wu","rank":null,"pid":[{"value":"0000-0001-7655-389X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Luke Lambourne","name":"Luke","surname":"Lambourne","rank":null,"pid":[{"value":"0000-0002-7001-7575","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Kazunori Hanagaki","name":"Kazunori","surname":"Hanagaki","rank":null,"pid":[{"value":"0000-0003-0676-0441","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Krzysztof Sliwa","name":"Krzysztof","surname":"Sliwa","rank":null,"pid":[{"value":"0000-0002-1201-4771","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Jiri Chudoba","name":"Jiri","surname":"Chudoba","rank":null,"pid":[{"value":"0000-0002-6425-2579","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Jörn Große-Knetter","name":"Jörn","surname":"Große-Knetter","rank":null,"pid":[{"value":"0000-0003-3085-7067","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Katharine Leney","name":"Katharine","surname":"Leney","rank":null,"pid":[{"value":"0000-0002-1525-2695","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Michel Lefebvre","name":"Michel","surname":"Lefebvre","rank":null,"pid":[{"value":"0000-0002-5560-0586","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Ahmed Bassalat","name":"Ahmed","surname":"Bassalat","rank":null,"pid":[{"value":"0000-0002-0129-1423","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Borut Kersevan","name":"Borut","surname":"Kersevan","rank":null,"pid":[{"value":"0000-0002-4529-452X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Calum Macdonald","name":"Calum","surname":"Macdonald","rank":null,"pid":[{"value":"0000-0001-7857-9188","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Davide Costanzo","name":"Davide","surname":"Costanzo","rank":null,"pid":[{"value":"0000-0003-4920-6264","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Wladyslaw Dabrowski","name":"Wladyslaw","surname":"Dabrowski","rank":null,"pid":[{"value":"0000-0001-9061-9568","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Jean-François Grivaz","name":"Jean-François","surname":"Grivaz","rank":null,"pid":[{"value":"0000-0003-4793-7995","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Elisabetta Pianori","name":"Elisabetta","surname":"Pianori","rank":null,"pid":[{"value":"0000-0001-9233-5892","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Christoph Falk Anders","name":"Christoph Falk","surname":"Anders","rank":null,"pid":[{"value":"0000-0001-6632-6327","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Vasiliki Kouskoura","name":"Vasiliki","surname":"Kouskoura","rank":null,"pid":[{"value":"0000-0002-8987-3208","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Ruth Pöttgen","name":"Ruth","surname":"Pöttgen","rank":null,"pid":[{"value":"0000-0002-3304-0987","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Sergey Burdin","name":"Sergey","surname":"Burdin","rank":null,"pid":[{"value":"0000-0003-4831-4132","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Farid Ould-Saada","name":"Farid","surname":"Ould-Saada","rank":null,"pid":[{"value":"0000-0002-9404-835X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Peter Onyisi","name":"Peter","surname":"Onyisi","rank":null,"pid":[{"value":"0000-0003-4201-7997","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Javier Llorente Merino","name":"Javier","surname":"Llorente Merino","rank":null,"pid":[{"value":"0000-0003-0027-7969","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Michel Vetterli","name":"Michel","surname":"Vetterli","rank":null,"pid":[{"value":"0000-0002-7223-2965","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Nicolas Morange","name":"Nicolas","surname":"Morange","rank":null,"pid":[{"value":"0000-0003-0047-7215","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Gianluca Introzzi","name":"Gianluca","surname":"Introzzi","rank":null,"pid":[{"value":"0000-0002-1314-2580","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Roger Moore","name":"Roger","surname":"Moore","rank":null,"pid":[{"value":"0000-0003-4160-4700","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Pierre-Antoine Delsart","name":"Pierre-Antoine","surname":"Delsart","rank":null,"pid":[{"value":"0000-0002-9556-2924","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Victor Solovyev","name":"Victor","surname":"Solovyev","rank":null,"pid":[{"value":"0000-0002-9402-6329","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Jiangyong jia","name":"Jiangyong","surname":"jia","rank":null,"pid":[{"value":"0000-0002-5725-3397","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Kyle Cranmer","name":"Kyle","surname":"Cranmer","rank":null,"pid":[{"value":"0000-0002-5769-7094","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Frederik Ruehr","name":"Frederik","surname":"Ruehr","rank":null,"pid":[{"value":"0000-0003-4452-620X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Peter van Gemmeren","name":"Peter","surname":"van Gemmeren","rank":null,"pid":[{"value":"0000-0002-7227-4006","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Jacob Kempster","name":"Jacob","surname":"Kempster","rank":null,"pid":[{"value":"0000-0003-4168-3373","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Oleg Brandt","name":"Oleg","surname":"Brandt","rank":null,"pid":[{"value":"0000-0001-5219-1417","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Antonio Policicchio","name":"Antonio","surname":"Policicchio","rank":null,"pid":[{"value":"0000-0002-1290-220X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Fabrice Hubaut","name":"Fabrice","surname":"Hubaut","rank":null,"pid":[{"value":"0000-0002-0113-2465","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Christopher Gorham Lester","name":"Christopher Gorham","surname":"Lester","rank":null,"pid":[{"value":"0000-0001-5770-4883","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Royer Edson Ticse Torres","name":"Royer Edson","surname":"Ticse Torres","rank":null,"pid":[{"value":"0000-0001-8178-5257","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Maria Josefina Alconada Verzini","name":"Maria Josefina","surname":"Alconada Verzini","rank":null,"pid":[{"value":"0000-0003-2212-7830","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"PHILLIP URQUIJO","name":"PHILLIP","surname":"URQUIJO","rank":null,"pid":[{"value":"0000-0002-0887-7953","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Annick Lleres","name":"Annick","surname":"Lleres","rank":null,"pid":[{"value":"0000-0003-1769-8524","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Andrei Snesarev","name":"Andrei","surname":"Snesarev","rank":null,"pid":[{"value":"0000-0002-9067-8362","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Marcin Wolter","name":"Marcin","surname":"Wolter","rank":null,"pid":[{"value":"0000-0001-9184-2921","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Garabed Halladjian","name":"Garabed","surname":"Halladjian","rank":null,"pid":[{"value":"0000-0001-7162-0301","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Anthony Morley","name":"Anthony","surname":"Morley","rank":null,"pid":[{"value":"0000-0003-0373-1346","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Francesco Nuti","name":"Francesco","surname":"Nuti","rank":null,"pid":[{"value":"0000-0003-3491-7637","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Mar Capeans","name":"Mar","surname":"Capeans","rank":null,"pid":[{"value":"0000-0001-7727-9175","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Jose Enrique Garcia Navarro","name":"Jose Enrique","surname":"Garcia Navarro","rank":null,"pid":[{"value":"0000-0002-0279-0523","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Aidan Robson","name":"Aidan","surname":"Robson","rank":null,"pid":[{"value":"0000-0002-1659-8284","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Eduardo Ros","name":"Eduardo","surname":"Ros","rank":null,"pid":[{"value":"0000-0003-2812-9554","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Kevin Varvell","name":"Kevin","surname":"Varvell","rank":null,"pid":[{"value":"0000-0003-1017-1295","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Ina Carli","name":"Ina","surname":"Carli","rank":null,"pid":[{"value":"0000-0002-0411-1141","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Ivo van Vulpen","name":"Ivo","surname":"van Vulpen","rank":null,"pid":[{"value":"0000-0001-7074-5655","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Liaoshan Shi","name":"Liaoshan","surname":"Shi","rank":null,"pid":[{"value":"0000-0001-9532-5075","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Pavel Starovoitov","name":"Pavel","surname":"Starovoitov","rank":null,"pid":[{"value":"0000-0003-1990-0992","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Lorenzo Feligioni","name":"Lorenzo","surname":"Feligioni","rank":null,"pid":[{"value":"0000-0002-1403-0951","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Christoph Wasicki","name":"Christoph","surname":"Wasicki","rank":null,"pid":[{"value":"0000-0001-8041-741X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Nuno Castro","name":"Nuno","surname":"Castro","rank":null,"pid":[{"value":"0000-0001-8491-4376","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Mikhail Levchenko","name":"Mikhail","surname":"Levchenko","rank":null,"pid":[{"value":"0000-0002-5495-0656","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Marek Palka","name":"Marek","surname":"Palka","rank":null,"pid":[{"value":"0000-0002-7185-3540","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Philip Allport","name":"Philip","surname":"Allport","rank":null,"pid":[{"value":"0000-0001-7303-2570","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Tomas Jakoubek","name":"Tomas","surname":"Jakoubek","rank":null,"pid":[{"value":"0000-0001-7038-0369","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Elizabeth Brost","name":"Elizabeth","surname":"Brost","rank":null,"pid":[{"value":"0000-0002-6800-9808","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Maciej Trzebinski","name":"Maciej","surname":"Trzebinski","rank":null,"pid":[{"value":"0000-0002-5151-7101","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Nikola Makovec","name":"Nikola","surname":"Makovec","rank":null,"pid":[{"value":"0000-0001-5124-904X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Bernhard Meirose","name":"Bernhard","surname":"Meirose","rank":null,"pid":[{"value":"0000-0003-0032-7022","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Andrea Dell’Acqua","name":"Andrea","surname":"Dell’Acqua","rank":null,"pid":[{"value":"0000-0003-2453-7745","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Carlos Escobar Ibáñez","name":"Carlos","surname":"Escobar Ibáñez","rank":null,"pid":[{"value":"0000-0003-4442-4537","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Hideyuki Oide","name":"Hideyuki","surname":"Oide","rank":null,"pid":[{"value":"0000-0002-2173-3233","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Valentina Maria Martina Cairo","name":"Valentina Maria Martina","surname":"Cairo","rank":null,"pid":[{"value":"0000-0002-0758-7575","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Johannes Erdmann","name":"Johannes","surname":"Erdmann","rank":null,"pid":[{"value":"0000-0002-8073-2740","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Frederic Deliot","name":"Frederic","surname":"Deliot","rank":null,"pid":[{"value":"0000-0003-0777-6031","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Fabian Hügging","name":"Fabian","surname":"Hügging","rank":null,"pid":[{"value":"0000-0002-7472-3151","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Antoine Marzin","name":"Antoine","surname":"Marzin","rank":null,"pid":[{"value":"0000-0003-4364-4351","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Sebastien Prince","name":"Sebastien","surname":"Prince","rank":null,"pid":[{"value":"0000-0001-9947-3892","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Melissa Ridel","name":"Melissa","surname":"Ridel","rank":null,"pid":[{"value":"0000-0002-2601-7420","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Christian Weiser","name":"Christian","surname":"Weiser","rank":null,"pid":[{"value":"0000-0002-6456-6834","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Farida Fassi","name":"Farida","surname":"Fassi","rank":null,"pid":[{"value":"0000-0002-6423-7213","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Manuella Vincter","name":"Manuella","surname":"Vincter","rank":null,"pid":[{"value":"0000-0002-5338-8972","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Julien Caudron","name":"Julien","surname":"Caudron","rank":null,"pid":[{"value":"0000-0002-3530-6531","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Giada Mancini","name":"Giada","surname":"Mancini","rank":null,"pid":[{"value":"0000-0001-6158-2751","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Vasiliki A. Mitsou","name":"Vasiliki A.","surname":"Mitsou","rank":null,"pid":[{"value":"0000-0002-1533-8886","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"David Wardrope","name":"David","surname":"Wardrope","rank":null,"pid":[{"value":"0000-0002-8208-2964","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Loïc Valéry","name":"Loïc","surname":"Valéry","rank":null,"pid":[{"value":"0000-0002-5510-1111","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Krisztian Peters","name":"Krisztian","surname":"Peters","rank":null,"pid":[{"value":"0000-0002-7654-1677","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Andrea Knue","name":"Andrea","surname":"Knue","rank":null,"pid":[{"value":"0000-0002-1559-9285","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Stathes Paganis","name":"Stathes","surname":"Paganis","rank":null,"pid":[{"value":"0000-0002-1950-8993","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Steffen Stärz","name":"Steffen","surname":"Stärz","rank":null,"pid":[{"value":"0000-0002-2908-3909","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Emma Torro Pastor","name":"Emma","surname":"Torro Pastor","rank":null,"pid":[{"value":"0000-0002-5507-7924","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Andrey Kiryunin","name":"Andrey","surname":"Kiryunin","rank":null,"pid":[{"value":"0000-0001-7490-6890","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Daniela Marcella Rebuzzi","name":"Daniela Marcella","surname":"Rebuzzi","rank":null,"pid":[{"value":"0000-0003-4461-3880","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Frederick Luehring","name":"Frederick","surname":"Luehring","rank":null,"pid":[{"value":"0000-0001-8721-6901","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"alessandro cerri","name":"alessandro","surname":"cerri","rank":null,"pid":[{"value":"0000-0002-1904-6661","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Remi Lafaye","name":"Remi","surname":"Lafaye","rank":null,"pid":[{"value":"0000-0001-7848-6088","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Torsten Akesson","name":"Torsten","surname":"Akesson","rank":null,"pid":[{"value":"0000-0003-4141-5408","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Hongbo Zhu","name":"Hongbo","surname":"Zhu","rank":null,"pid":[{"value":"0000-0001-8066-7048","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Paul Laycock","name":"Paul","surname":"Laycock","rank":null,"pid":[{"value":"0000-0002-8572-5339","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Paolo Giromini","name":"Paolo","surname":"Giromini","rank":null,"pid":[{"value":"0000-0003-0276-287X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Leszek Adamczyk","name":"Leszek","surname":"Adamczyk","rank":null,"pid":[{"value":"0000-0002-5859-2075","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Stephen Lloyd","name":"Stephen","surname":"Lloyd","rank":null,"pid":[{"value":"0000-0002-5073-2264","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Aaron Angerami","name":"Aaron","surname":"Angerami","rank":null,"pid":[{"value":"0000-0001-7834-8750","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Susumu Oda","name":"Susumu","surname":"Oda","rank":null,"pid":[{"value":"0000-0001-5836-768X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Nathalie Besson","name":"Nathalie","surname":"Besson","rank":null,"pid":[{"value":"0000-0001-9248-6252","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"MARCELLO FANTI","name":"MARCELLO","surname":"FANTI","rank":null,"pid":[{"value":"0000-0002-8773-145X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Peter Kodyš","name":"Peter","surname":"Kodyš","rank":null,"pid":[{"value":"0000-0002-8644-2349","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Teng Jian Khoo","name":"Teng Jian","surname":"Khoo","rank":null,"pid":[{"value":"0000-0002-5954-3101","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Javier Montejo Berlingen","name":"Javier","surname":"Montejo Berlingen","rank":null,"pid":[{"value":"0000-0001-9213-904X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Sandro Palestini","name":"Sandro","surname":"Palestini","rank":null,"pid":[{"value":"0000-0002-4110-096X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Mark Oreglia","name":"Mark","surname":"Oreglia","rank":null,"pid":[{"value":"0000-0001-6203-2209","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Krzysztof Korcyl","name":"Krzysztof","surname":"Korcyl","rank":null,"pid":[{"value":"0000-0001-8085-4505","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Sebastian Schmitt","name":"Sebastian","surname":"Schmitt","rank":null,"pid":[{"value":"0000-0002-7935-0470","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Irinel Caprini","name":"Irinel","surname":"Caprini","rank":null,"pid":[{"value":"0000-0003-3343-3200","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Anthony Doyle","name":"Anthony","surname":"Doyle","rank":null,"pid":[{"value":"0000-0001-6322-6195","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Markus Elsing","name":"Markus","surname":"Elsing","rank":null,"pid":[{"value":"0000-0002-1213-0545","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Christian Schmitt","name":"Christian","surname":"Schmitt","rank":null,"pid":[{"value":"0000-0003-1471-690X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Konstantinos Nikolopoulos","name":"Konstantinos","surname":"Nikolopoulos","rank":null,"pid":[{"value":"0000-0002-3048-489X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Janet Dietrich","name":"Janet","surname":"Dietrich","rank":null,"pid":[{"value":"0000-0001-7061-1585","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Yuri Kulchitsky","name":"Yuri","surname":"Kulchitsky","rank":null,"pid":[{"value":"0000-0002-3036-5575","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Sinead Farrington","name":"Sinead","surname":"Farrington","rank":null,"pid":[{"value":"0000-0001-5350-9271","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Stefano Terzo","name":"Stefano","surname":"Terzo","rank":null,"pid":[{"value":"0000-0003-3388-3906","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"stefania xella","name":"stefania","surname":"xella","rank":null,"pid":[{"value":"0000-0002-0988-1655","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Jonathan Butterworth","name":"Jonathan","surname":"Butterworth","rank":null,"pid":[{"value":"0000-0002-5905-5394","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Gideon Bella","name":"Gideon","surname":"Bella","rank":null,"pid":[{"value":"0000-0002-4009-0990","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Marcello Bindi","name":"Marcello","surname":"Bindi","rank":null,"pid":[{"value":"0000-0001-6172-545X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Heather Gray","name":"Heather","surname":"Gray","rank":null,"pid":[{"value":"0000-0002-5293-4716","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Marcel Vos","name":"Marcel","surname":"Vos","rank":null,"pid":[{"value":"0000-0001-8474-5357","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Ljiljana Morvaj","name":"Ljiljana","surname":"Morvaj","rank":null,"pid":[{"value":"0000-0003-2061-2904","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Didier Ferrere","name":"Didier","surname":"Ferrere","rank":null,"pid":[{"value":"0000-0002-5687-9240","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Mauro Villa","name":"Mauro","surname":"Villa","rank":null,"pid":[{"value":"0000-0002-9181-8048","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Deepak Kar","name":"Deepak","surname":"Kar","rank":null,"pid":[{"value":"0000-0002-4238-9822","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Frank Berghaus","name":"Frank","surname":"Berghaus","rank":null,"pid":[{"value":"0000-0003-1887-3910","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Bruce Yabsley","name":"Bruce","surname":"Yabsley","rank":null,"pid":[{"value":"0000-0002-2680-0474","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Frank Winklmeier","name":"Frank","surname":"Winklmeier","rank":null,"pid":[{"value":"0000-0001-8290-3200","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Zoya Karpova","name":"Zoya","surname":"Karpova","rank":null,"pid":[{"value":"0000-0003-0254-4629","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Jason Nielsen","name":"Jason","surname":"Nielsen","rank":null,"pid":[{"value":"0000-0002-9175-4419","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Claire Gwenlan","name":"Claire","surname":"Gwenlan","rank":null,"pid":[{"value":"0000-0002-3518-0617","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Stephanie Majewski","name":"Stephanie","surname":"Majewski","rank":null,"pid":[{"value":"0000-0002-6871-3395","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"James Mueller","name":"James","surname":"Mueller","rank":null,"pid":[{"value":"0000-0001-5099-4718","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Mario Giordani","name":"Mario","surname":"Giordani","rank":null,"pid":[{"value":"0000-0002-0792-6039","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Laura Fabbri","name":"Laura","surname":"Fabbri","rank":null,"pid":[{"value":"0000-0002-4002-8353","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Michael Begel","name":"Michael","surname":"Begel","rank":null,"pid":[{"value":"0000-0002-1634-4399","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Laurent Schoeffel","name":"Laurent","surname":"Schoeffel","rank":null,"pid":[{"value":"0000-0002-8081-2353","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Bobby Acharya","name":"Bobby","surname":"Acharya","rank":null,"pid":[{"value":"0000-0002-8588-9157","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Jan Kretzschmar","name":"Jan","surname":"Kretzschmar","rank":null,"pid":[{"value":"0000-0002-8515-1355","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Thi Ngoc Loan Truong","name":"Thi Ngoc Loan","surname":"Truong","rank":null,"pid":[{"value":"0000-0001-8249-7150","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Yury Smirnov","name":"Yury","surname":"Smirnov","rank":null,"pid":[{"value":"0000-0002-2891-0781","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Andreas Warburton","name":"Andreas","surname":"Warburton","rank":null,"pid":[{"value":"0000-0002-2298-7315","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Osamu Jinnouchi","name":"Osamu","surname":"Jinnouchi","rank":null,"pid":[{"value":"0000-0001-5073-0974","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Nicola Orlando","name":"Nicola","surname":"Orlando","rank":null,"pid":[{"value":"0000-0003-0616-245X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Magda Anna Chelstowska","name":"Magda Anna","surname":"Chelstowska","rank":null,"pid":[{"value":"0000-0003-1030-2099","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Armin Nairz","name":"Armin","surname":"Nairz","rank":null,"pid":[{"value":"0000-0003-3561-0880","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Chris Malena Delitzsch","name":"Chris Malena","surname":"Delitzsch","rank":null,"pid":[{"value":"0000-0001-7021-3333","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Kendall Reeves","name":"Kendall","surname":"Reeves","rank":null,"pid":[{"value":"0000-0003-3504-4882","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Minghui Liu","name":"Minghui","surname":"Liu","rank":null,"pid":[{"value":"0000-0003-0056-7296","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Clara Troncon","name":"Clara","surname":"Troncon","rank":null,"pid":[{"value":"0000-0002-7997-8524","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Chris Hays","name":"Chris","surname":"Hays","rank":null,"pid":[{"value":"0000-0003-2371-9723","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Driss Benchekroun","name":"Driss","surname":"Benchekroun","rank":null,"pid":[{"value":"0000-0001-5196-8327","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Tamar Djobava","name":"Tamar","surname":"Djobava","rank":null,"pid":[{"value":"0000-0002-9414-8350","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Joany Manjarrés Ramos","name":"Joany","surname":"Manjarrés Ramos","rank":null,"pid":[{"value":"0000-0003-3896-5222","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Craig Wiglesworth","name":"Craig","surname":"Wiglesworth","rank":null,"pid":[{"value":"0000-0001-6219-8946","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Lawrence Lee","name":"Lawrence","surname":"Lee","rank":null,"pid":[{"value":"0000-0002-5590-335X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Elisabetta Barberio","name":"Elisabetta","surname":"Barberio","rank":null,"pid":[{"value":"0000-0002-3111-0910","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Robert McPherson","name":"Robert","surname":"McPherson","rank":null,"pid":[{"value":"0000-0001-9211-7019","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Paul Jackson","name":"Paul","surname":"Jackson","rank":null,"pid":[{"value":"0000-0002-0847-402X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Peter Henrik Hansen","name":"Peter Henrik","surname":"Hansen","rank":null,"pid":[{"value":"0000-0002-6764-4789","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Cristobal Padilla","name":"Cristobal","surname":"Padilla","rank":null,"pid":[{"value":"0000-0001-7951-0166","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Nora Pettersson","name":"Nora","surname":"Pettersson","rank":null,"pid":[{"value":"0000-0001-7451-3544","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Paul Glaysher","name":"Paul","surname":"Glaysher","rank":null,"pid":[{"value":"0000-0002-5437-971X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Patrick Czodrowski","name":"Patrick","surname":"Czodrowski","rank":null,"pid":[{"value":"0000-0003-0723-1437","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"María Moreno Llácer","name":"María","surname":"Moreno Llácer","rank":null,"pid":[{"value":"0000-0003-1113-3645","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Soumya Mohapatra","name":"Soumya","surname":"Mohapatra","rank":null,"pid":[{"value":"0000-0003-3006-6337","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Andrea Gaudiello","name":"Andrea","surname":"Gaudiello","rank":null,"pid":[{"value":"0000-0001-7721-8217","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Maximiliano Sioli","name":"Maximiliano","surname":"Sioli","rank":null,"pid":[{"value":"0000-0002-0912-9121","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Christian Schroeder","name":"Christian","surname":"Schroeder","rank":null,"pid":[{"value":"0000-0001-6449-0668","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Guillaume Unal","name":"Guillaume","surname":"Unal","rank":null,"pid":[{"value":"0000-0001-8130-7423","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Christian Gutschow","name":"Christian","surname":"Gutschow","rank":null,"pid":[{"value":"0000-0003-0857-794X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Spyridon Argyropoulos","name":"Spyridon","surname":"Argyropoulos","rank":null,"pid":[{"value":"0000-0001-7748-1429","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Vadim Bednyakov","name":"Vadim","surname":"Bednyakov","rank":null,"pid":[{"value":"0000-0003-4864-8909","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Caterina Doglioni","name":"Caterina","surname":"Doglioni","rank":null,"pid":[{"value":"0000-0002-1509-0390","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Andrew Haas","name":"Andrew","surname":"Haas","rank":null,"pid":[{"value":"0000-0002-4832-0455","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Masahiro Morii","name":"Masahiro","surname":"Morii","rank":null,"pid":[{"value":"0000-0001-9324-057X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Andrea Coccaro","name":"Andrea","surname":"Coccaro","rank":null,"pid":[{"value":"0000-0003-2368-4559","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Edward Moyse","name":"Edward","surname":"Moyse","rank":null,"pid":[{"value":"0000-0003-4449-6178","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Takuya Nobe","name":"Takuya","surname":"Nobe","rank":null,"pid":[{"value":"0000-0002-5809-325X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Jahred Adelman","name":"Jahred","surname":"Adelman","rank":null,"pid":[{"value":"0000-0002-1041-3496","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Antonio Ereditato","name":"Antonio","surname":"Ereditato","rank":null,"pid":[{"value":"0000-0002-5423-8079","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Stefan Schmitt","name":"Stefan","surname":"Schmitt","rank":null,"pid":[{"value":"0000-0001-8387-1853","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"David DeMarco","name":"David","surname":"DeMarco","rank":null,"pid":[{"value":"0000-0002-8921-8828","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Alessandro La Rosa","name":"Alessandro","surname":"La Rosa","rank":null,"pid":[{"value":"0000-0001-6291-2142","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Liza Mijovic","name":"Liza","surname":"Mijovic","rank":null,"pid":[{"value":"0000-0003-0162-2891","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Stephane Willocq","name":"Stephane","surname":"Willocq","rank":null,"pid":[{"value":"0000-0002-4120-1453","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"George Iakovidis","name":"George","surname":"Iakovidis","rank":null,"pid":[{"value":"0000-0002-0330-5921","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Madar Romain","name":"Madar","surname":"Romain","rank":null,"pid":[{"value":"0000-0002-6875-6408","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"giuseppe iacobucci","name":"giuseppe","surname":"iacobucci","rank":null,"pid":[{"value":"0000-0001-9965-5442","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Allen Mincer","name":"Allen","surname":"Mincer","rank":null,"pid":[{"value":"0000-0002-6307-1418","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"TROCME Benjamin","name":"TROCME","surname":"Benjamin","rank":null,"pid":[{"value":"0000-0001-9500-2487","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Liang Li","name":"Liang","surname":"Li","rank":null,"pid":[{"value":"0000-0001-6411-6107","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"William Murray","name":"William","surname":"Murray","rank":null,"pid":[{"value":"0000-0003-1710-6306","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Chaowaroj Wanotayaroj","name":"Chaowaroj","surname":"Wanotayaroj","rank":null,"pid":[{"value":"0000-0002-8178-5705","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Luc Goossens","name":"Luc","surname":"Goossens","rank":null,"pid":[{"value":"0000-0002-2536-4498","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Kerstin Jon-And","name":"Kerstin","surname":"Jon-And","rank":null,"pid":[{"value":"0000-0001-8201-7700","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Arwa Bannoura","name":"Arwa","surname":"Bannoura","rank":null,"pid":[{"value":"0000-0002-7166-8118","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Filipe Veloso","name":"Filipe","surname":"Veloso","rank":null,"pid":[{"value":"0000-0002-5956-4244","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Maximilian Swiatlowski","name":"Maximilian","surname":"Swiatlowski","rank":null,"pid":[{"value":"0000-0001-7287-0468","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Marija Vranjes Milosavljevic","name":"Marija","surname":"Vranjes Milosavljevic","rank":null,"pid":[{"value":"0000-0003-4477-9733","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Riccardo Maria Bianchi","name":"Riccardo Maria","surname":"Bianchi","rank":null,"pid":[{"value":"0000-0001-7345-7798","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Elias Coniavitis","name":"Elias","surname":"Coniavitis","rank":null,"pid":[{"value":"0000-0002-2148-8012","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Ondrej Penc","name":"Ondrej","surname":"Penc","rank":null,"pid":[{"value":"0000-0002-5433-3981","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Julie Kirk","name":"Julie","surname":"Kirk","rank":null,"pid":[{"value":"0000-0001-8096-7577","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Hongtao Yang","name":"Hongtao","surname":"Yang","rank":null,"pid":[{"value":"0000-0003-3554-7113","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Ewelina Maria Lobodzinska","name":"Ewelina Maria","surname":"Lobodzinska","rank":null,"pid":[{"value":"0000-0001-9012-3431","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Christos Leonidopoulos","name":"Christos","surname":"Leonidopoulos","rank":null,"pid":[{"value":"0000-0002-7241-2114","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Silvia Biondi","name":"Silvia","surname":"Biondi","rank":null,"pid":[{"value":"0000-0002-1492-6715","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Stephane JEZEQUEL","name":"Stephane","surname":"JEZEQUEL","rank":null,"pid":[{"value":"0000-0001-7369-6975","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Zhiqing Zhang","name":"Zhiqing","surname":"Zhang","rank":null,"pid":[{"value":"0000-0002-7853-9079","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Kathleen Whalen","name":"Kathleen","surname":"Whalen","rank":null,"pid":[{"value":"0000-0002-9383-8763","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Sophie HENROT","name":"Sophie","surname":"HENROT","rank":null,"pid":[{"value":"0000-0003-1218-2991","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Louise Heelan","name":"Louise","surname":"Heelan","rank":null,"pid":[{"value":"0000-0002-4879-0131","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Dominik Derendarz","name":"Dominik","surname":"Derendarz","rank":null,"pid":[{"value":"0000-0001-5660-3095","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Ning Zhou","name":"Ning","surname":"Zhou","rank":null,"pid":[{"value":"0000-0002-1775-2511","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Mihai Caprini","name":"Mihai","surname":"Caprini","rank":null,"pid":[{"value":"0000-0002-6806-6730","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Lee Sawyer","name":"Lee","surname":"Sawyer","rank":null,"pid":[{"value":"0000-0001-8295-0605","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Lydia Roos","name":"Lydia","surname":"Roos","rank":null,"pid":[{"value":"0000-0001-7151-9983","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Stanislav Nemecek","name":"Stanislav","surname":"Nemecek","rank":null,"pid":[{"value":"0000-0001-8978-7150","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Matthias Danninger","name":"Matthias","surname":"Danninger","rank":null,"pid":[{"value":"0000-0002-7807-7484","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Lailin Xu","name":"Lailin","surname":"Xu","rank":null,"pid":[{"value":"0000-0001-8997-3199","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Angel Campoverde","name":"Angel","surname":"Campoverde","rank":null,"pid":[{"value":"0000-0003-1968-1216","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Carl Gwilliam","name":"Carl","surname":"Gwilliam","rank":null,"pid":[{"value":"0000-0002-9401-5304","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Fabrizio Salvatore","name":"Fabrizio","surname":"Salvatore","rank":null,"pid":[{"value":"0000-0002-3709-1554","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Werner Wiedenmann","name":"Werner","surname":"Wiedenmann","rank":null,"pid":[{"value":"0000-0003-3605-3633","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Adriaan König","name":"Adriaan","surname":"König","rank":null,"pid":[{"value":"0000-0001-6702-6473","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Romain Kukla","name":"Romain","surname":"Kukla","rank":null,"pid":[{"value":"0000-0002-1140-2465","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Sarah Heim","name":"Sarah","surname":"Heim","rank":null,"pid":[{"value":"0000-0002-2639-6571","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Stamatios Gkaitatzis","name":"Stamatios","surname":"Gkaitatzis","rank":null,"pid":[{"value":"0000-0001-9420-7499","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Alison Elliot","name":"Alison","surname":"Elliot","rank":null,"pid":[{"value":"0000-0003-0921-0314","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Marc Escalier","name":"Marc","surname":"Escalier","rank":null,"pid":[{"value":"0000-0003-4270-2775","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Philipp Stolte","name":"Philipp","surname":"Stolte","rank":null,"pid":[{"value":"0000-0002-8828-3564","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Peter Loch","name":"Peter","surname":"Loch","rank":null,"pid":[{"value":"0000-0002-2005-671X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Tamara Vazquez Schroeder","name":"Tamara","surname":"Vazquez Schroeder","rank":null,"pid":[{"value":"0000-0002-9780-099X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"lily asquith","name":"lily","surname":"asquith","rank":null,"pid":[{"value":"0000-0001-8035-7162","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Javier Sanchez","name":"Javier","surname":"Sanchez","rank":null,"pid":[{"value":"0000-0001-9913-310X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Tomas Davidek","name":"Tomas","surname":"Davidek","rank":null,"pid":[{"value":"0000-0002-3770-8307","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Lidia Dell'Asta","name":"Lidia","surname":"Dell'Asta","rank":null,"pid":[{"value":"0000-0002-9601-4225","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Francisco Alonso","name":"Francisco","surname":"Alonso","rank":null,"pid":[{"value":"0000-0001-9431-8156","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"J. Katharina Behr","name":"J. Katharina","surname":"Behr","rank":null,"pid":[{"value":"0000-0002-5501-4640","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"MARIO MARTINEZ","name":"MARIO","surname":"MARTINEZ","rank":null,"pid":[{"value":"0000-0002-3135-945X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Konstantinos Bachas","name":"Konstantinos","surname":"Bachas","rank":null,"pid":[{"value":"0000-0002-9047-6517","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Arnaud Lucotte","name":"Arnaud","surname":"Lucotte","rank":null,"pid":[{"value":"0000-0002-5992-0640","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"scott snyder","name":"scott","surname":"snyder","rank":null,"pid":[{"value":"0000-0001-8610-8423","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Edson Carquin","name":"Edson","surname":"Carquin","rank":null,"pid":[{"value":"0000-0002-7863-1166","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Kristin Lohwasser","name":"Kristin","surname":"Lohwasser","rank":null,"pid":[{"value":"0000-0003-1833-9160","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Jens Weingarten","name":"Jens","surname":"Weingarten","rank":null,"pid":[{"value":"0000-0003-2165-871X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Victor Maleev","name":"Victor","surname":"Maleev","rank":null,"pid":[{"value":"0000-0003-1028-8602","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Francesca Ungaro","name":"Francesca","surname":"Ungaro","rank":null,"pid":[{"value":"0000-0003-2005-595X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Koji Terashi","name":"Koji","surname":"Terashi","rank":null,"pid":[{"value":"0000-0001-6520-8070","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Kohei Yorita","name":"Kohei","surname":"Yorita","rank":null,"pid":[{"value":"0000-0003-1988-8401","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Roland Jansky","name":"Roland","surname":"Jansky","rank":null,"pid":[{"value":"0000-0003-0456-4658","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Sandro De Cecco","name":"Sandro","surname":"De Cecco","rank":null,"pid":[{"value":"0000-0003-4907-8610","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Wade Fisher","name":"Wade","surname":"Fisher","rank":null,"pid":[{"value":"0000-0003-3043-3045","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Jaroslav Guenther","name":"Jaroslav","surname":"Guenther","rank":null,"pid":[{"value":"0000-0003-3189-3959","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Tatsuya Masubuchi","name":"Tatsuya","surname":"Masubuchi","rank":null,"pid":[{"value":"0000-0001-9984-8009","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Janusz Chwastowski","name":"Janusz","surname":"Chwastowski","rank":null,"pid":[{"value":"0000-0002-6190-8376","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Ian Watson","name":"Ian","surname":"Watson","rank":null,"pid":[{"value":"0000-0003-2141-3413","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Patrick Rieck","name":"Patrick","surname":"Rieck","rank":null,"pid":[{"value":"0000-0003-0290-0566","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Mario Lassnig","name":"Mario","surname":"Lassnig","rank":null,"pid":[{"value":"0000-0002-9541-0592","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Jana Schaarschmidt","name":"Jana","surname":"Schaarschmidt","rank":null,"pid":[{"value":"0000-0002-0433-6439","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Joseph Reichert","name":"Joseph","surname":"Reichert","rank":null,"pid":[{"value":"0000-0003-2110-8021","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Bjarne Stugu","name":"Bjarne","surname":"Stugu","rank":null,"pid":[{"value":"0000-0002-1728-9272","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Paolo Gauzzi","name":"Paolo","surname":"Gauzzi","rank":null,"pid":[{"value":"0000-0003-4841-5822","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Sergei Smirnov","name":"Sergei","surname":"Smirnov","rank":null,"pid":[{"value":"0000-0002-6778-073X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Xiangyang Ju","name":"Xiangyang","surname":"Ju","rank":null,"pid":[{"value":"0000-0002-9745-1638","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Fernando Barreiro","name":"Fernando","surname":"Barreiro","rank":null,"pid":[{"value":"0000-0002-3021-0258","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Markus Cristinziani","name":"Markus","surname":"Cristinziani","rank":null,"pid":[{"value":"0000-0003-3893-9171","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Antonio Amorim","name":"Antonio","surname":"Amorim","rank":null,"pid":[{"value":"0000-0003-0638-2321","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Sune Jakobsen","name":"Sune","surname":"Jakobsen","rank":null,"pid":[{"value":"0000-0002-6564-040X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Alejandro Alonso","name":"Alejandro","surname":"Alonso","rank":null,"pid":[{"value":"0000-0003-1259-0573","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Makoto Shimojima","name":"Makoto","surname":"Shimojima","rank":null,"pid":[{"value":"0000-0002-8738-1664","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Francesco Guescini","name":"Francesco","surname":"Guescini","rank":null,"pid":[{"value":"0000-0001-5351-2673","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Christina Potter","name":"Christina","surname":"Potter","rank":null,"pid":[{"value":"0000-0002-9815-5208","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Evelina Bouhova-Thacker","name":"Evelina","surname":"Bouhova-Thacker","rank":null,"pid":[{"value":"0000-0002-5103-1558","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Eduard Simioni","name":"Eduard","surname":"Simioni","rank":null,"pid":[{"value":"0000-0002-8929-6236","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Paolo Camarri","name":"Paolo","surname":"Camarri","rank":null,"pid":[{"value":"0000-0002-5732-5645","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"William Leight","name":"William","surname":"Leight","rank":null,"pid":[{"value":"0000-0002-2968-7841","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Luis Roberto Flores Castillo","name":"Luis Roberto","surname":"Flores Castillo","rank":null,"pid":[{"value":"0000-0003-1551-5974","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Alessandro null","name":"Alessandro","surname":null,"rank":null,"pid":[{"value":"0000-0002-8224-6105","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Roman Lysak","name":"Roman","surname":"Lysak","rank":null,"pid":[{"value":"0000-0003-2990-1673","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Matteo Franchini","name":"Matteo","surname":"Franchini","rank":null,"pid":[{"value":"0000-0002-4554-252X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Martin Nagel","name":"Martin","surname":"Nagel","rank":null,"pid":[{"value":"0000-0002-2588-6691","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Harald Fox","name":"Harald","surname":"Fox","rank":null,"pid":[{"value":"0000-0003-3089-6090","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Klaus Moenig","name":"Klaus","surname":"Moenig","rank":null,"pid":[{"value":"0000-0002-3169-7117","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Sergey Karpov","name":"Sergey","surname":"Karpov","rank":null,"pid":[{"value":"0000-0002-2230-5353","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Remi Zaidan","name":"Remi","surname":"Zaidan","rank":null,"pid":[{"value":"0000-0002-3710-4554","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Kilian Rosbach","name":"Kilian","surname":"Rosbach","rank":null,"pid":[{"value":"0000-0002-4241-2949","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Pawel Klimek","name":"Pawel","surname":"Klimek","rank":null,"pid":[{"value":"0000-0003-1661-6873","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Nikolaos Konstantinidis","name":"Nikolaos","surname":"Konstantinidis","rank":null,"pid":[{"value":"0000-0002-4140-6360","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Alberto Mengarelli","name":"Alberto","surname":"Mengarelli","rank":null,"pid":[{"value":"0000-0002-1884-854X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Robert Kowalewski","name":"Robert","surname":"Kowalewski","rank":null,"pid":[{"value":"0000-0002-7314-0990","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Miguel Arratia","name":"Miguel","surname":"Arratia","rank":null,"pid":[{"value":"0000-0001-6877-3315","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Paola Giannetti","name":"Paola","surname":"Giannetti","rank":null,"pid":[{"value":"0000-0002-3721-9490","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Joern Lange","name":"Joern","surname":"Lange","rank":null,"pid":[{"value":"0000-0003-1307-1441","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Pavel Řezníček","name":"Pavel","surname":"Řezníček","rank":null,"pid":[{"value":"0000-0003-4017-9829","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Vojtech Pleskot","name":"Vojtech","surname":"Pleskot","rank":null,"pid":[{"value":"0000-0001-5435-497X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Federico Sforza","name":"Federico","surname":"Sforza","rank":null,"pid":[{"value":"0000-0002-4065-7352","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Danijela Bogavac","name":"Danijela","surname":"Bogavac","rank":null,"pid":[{"value":"0000-0003-2138-9062","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Nenad Vranjes","name":"Nenad","surname":"Vranjes","rank":null,"pid":[{"value":"0000-0001-5415-5225","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Peter Watkins","name":"Peter","surname":"Watkins","rank":null,"pid":[{"value":"0000-0002-1290-6833","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Yohei Yamaguchi","name":"Yohei","surname":"Yamaguchi","rank":null,"pid":[{"value":"0000-0002-3725-4800","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Matthias Saimpert","name":"Matthias","surname":"Saimpert","rank":null,"pid":[{"value":"0000-0002-3765-1320","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Alexander Grohsjean","name":"Alexander","surname":"Grohsjean","rank":null,"pid":[{"value":"0000-0003-0748-8494","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Tibor Zenis","name":"Tibor","surname":"Zenis","rank":null,"pid":[{"value":"0000-0001-8265-6916","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Stefano Camarda","name":"Stefano","surname":"Camarda","rank":null,"pid":[{"value":"0000-0003-0479-7689","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Carlos Lacasta","name":"Carlos","surname":"Lacasta","rank":null,"pid":[{"value":"0000-0002-2623-6252","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Oliver Ricken","name":"Oliver","surname":"Ricken","rank":null,"pid":[{"value":"0000-0001-5107-7276","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Claude Leroy","name":"Claude","surname":"Leroy","rank":null,"pid":[{"value":"0000-0003-3105-7045","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Murrough Landon","name":"Murrough","surname":"Landon","rank":null,"pid":[{"value":"0000-0001-6828-9769","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Martin White","name":"Martin","surname":"White","rank":null,"pid":[{"value":"0000-0001-5474-4580","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Dimitrii Krasnopevtsev","name":"Dimitrii","surname":"Krasnopevtsev","rank":null,"pid":[{"value":"0000-0002-6356-372X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Susana Cabrera Urbán","name":"Susana","surname":"Cabrera Urbán","rank":null,"pid":[{"value":"0000-0001-7640-7913","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Hans-Christian Schultz-Coulon","name":"Hans-Christian","surname":"Schultz-Coulon","rank":null,"pid":[{"value":"0000-0002-0860-7240","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Stephen Jiggins","name":"Stephen","surname":"Jiggins","rank":null,"pid":[{"value":"0000-0003-2906-1977","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Andre Sopczak","name":"Andre","surname":"Sopczak","rank":null,"pid":[{"value":"0000-0001-6981-0544","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Cristinel Diaconu","name":"Cristinel","surname":"Diaconu","rank":null,"pid":[{"value":"0000-0002-6193-5091","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Hector de la Torre Perez","name":"Hector","surname":"de la Torre Perez","rank":null,"pid":[{"value":"0000-0002-4516-5269","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Costa Mezquita","name":"Costa","surname":"Mezquita","rank":null,"pid":[{"value":"0000-0002-2064-2954","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Sahal Yacoob","name":"Sahal","surname":"Yacoob","rank":null,"pid":[{"value":"0000-0001-6977-3456","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Jian Liu","name":"Jian","surname":"Liu","rank":null,"pid":[{"value":"0000-0002-8397-7620","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Oxana Smirnova","name":"Oxana","surname":"Smirnova","rank":null,"pid":[{"value":"0000-0003-2517-531X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Thomas Koffas","name":"Thomas","surname":"Koffas","rank":null,"pid":[{"value":"0000-0001-9612-4988","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Stephen Burke","name":"Stephen","surname":"Burke","rank":null,"pid":[{"value":"0000-0002-1962-8493","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Elizaveta Shabalina","name":"Elizaveta","surname":"Shabalina","rank":null,"pid":[{"value":"0000-0003-4849-556X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Nicolas Ellis","name":"Nicolas","surname":"Ellis","rank":null,"pid":[{"value":"0000-0002-1920-4930","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Robert Astalos","name":"Robert","surname":"Astalos","rank":null,"pid":[{"value":"0000-0001-5095-605X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Valerio Dao","name":"Valerio","surname":"Dao","rank":null,"pid":[{"value":"0000-0003-1645-8393","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Javier Alberto Murillo Quijada","name":"Javier Alberto","surname":"Murillo Quijada","rank":null,"pid":[{"value":"0000-0003-4933-2092","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Alan Litke","name":"Alan","surname":"Litke","rank":null,"pid":[{"value":"0000-0003-3973-3642","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Hans Peter Beck","name":"Hans Peter","surname":"Beck","rank":null,"pid":[{"value":"0000-0001-7212-1096","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Flavia De Almeida Dias","name":"Flavia","surname":"De Almeida Dias","rank":null,"pid":[{"value":"0000-0001-6882-5402","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Mykhailo Lisovyi","name":"Mykhailo","surname":"Lisovyi","rank":null,"pid":[{"value":"0000-0002-3014-5855","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Frank Ellinghaus","name":"Frank","surname":"Ellinghaus","rank":null,"pid":[{"value":"0000-0003-3596-5331","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Alexey Ezhilov","name":"Alexey","surname":"Ezhilov","rank":null,"pid":[{"value":"0000-0002-7520-293X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Anna Sfyrla","name":"Anna","surname":"Sfyrla","rank":null,"pid":[{"value":"0000-0002-3003-9905","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Marc-André Pleier","name":"Marc-André","surname":"Pleier","rank":null,"pid":[{"value":"0000-0002-9461-3494","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Aliaksei Hrynevich","name":"Aliaksei","surname":"Hrynevich","rank":null,"pid":[{"value":"0000-0002-5411-114X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Lucia Masetti","name":"Lucia","surname":"Masetti","rank":null,"pid":[{"value":"0000-0002-0038-5372","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Petr Balek","name":"Petr","surname":"Balek","rank":null,"pid":[{"value":"0000-0002-0942-1966","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Marco Sessa","name":"Marco","surname":"Sessa","rank":null,"pid":[{"value":"0000-0002-1402-7525","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Ilija Vukotic","name":"Ilija","surname":"Vukotic","rank":null,"pid":[{"value":"0000-0003-0472-3516","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Tova Holmes","name":"Tova","surname":"Holmes","rank":null,"pid":[{"value":"0000-0002-3959-5174","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Hermann Kolanoski","name":"Hermann","surname":"Kolanoski","rank":null,"pid":[{"value":"0000-0003-0435-2524","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Sigve Haug","name":"Sigve","surname":"Haug","rank":null,"pid":[{"value":"0000-0003-0442-3361","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Bruno Galhardo","name":"Bruno","surname":"Galhardo","rank":null,"pid":[{"value":"0000-0003-0641-301X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Nello Bruscino","name":"Nello","surname":"Bruscino","rank":null,"pid":[{"value":"0000-0002-6168-689X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Jonas Strandberg","name":"Jonas","surname":"Strandberg","rank":null,"pid":[{"value":"0000-0002-8913-0981","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Riccardo Vari","name":"Riccardo","surname":"Vari","rank":null,"pid":[{"value":"0000-0002-2814-1337","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Emmanuel Sauvan","name":"Emmanuel","surname":"Sauvan","rank":null,"pid":[{"value":"0000-0003-1921-2647","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Hartmut Sadrozinski","name":"Hartmut","surname":"Sadrozinski","rank":null,"pid":[{"value":"0000-0003-0019-5410","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Juerg Beringer","name":"Juerg","surname":"Beringer","rank":null,"pid":[{"value":"0000-0002-9975-1781","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"laurent chevalier","name":"laurent","surname":"chevalier","rank":null,"pid":[{"value":"0000-0003-3762-7264","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Dominik Duda","name":"Dominik","surname":"Duda","rank":null,"pid":[{"value":"0000-0002-5916-3467","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Takanori Kono","name":"Takanori","surname":"Kono","rank":null,"pid":[{"value":"0000-0003-1553-2950","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Igor Gorelov","name":"Igor","surname":"Gorelov","rank":null,"pid":[{"value":"0000-0001-5570-0133","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Noam Tal Hod","name":"Noam","surname":"Tal Hod","rank":null,"pid":[{"value":"0000-0001-5241-0544","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Umberto De Sanctis","name":"Umberto","surname":"De Sanctis","rank":null,"pid":[{"value":"0000-0003-4704-525X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Andrii Tykhonov","name":"Andrii","surname":"Tykhonov","rank":null,"pid":[{"value":"0000-0003-2908-7915","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Francois Corriveau","name":"Francois","surname":"Corriveau","rank":null,"pid":[{"value":"0000-0002-4970-7600","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Michaela Queitsch-Maitland","name":"Michaela","surname":"Queitsch-Maitland","rank":null,"pid":[{"value":"0000-0003-4643-515X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Iacopo Vivarelli","name":"Iacopo","surname":"Vivarelli","rank":null,"pid":[{"value":"0000-0003-0097-123X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"sotirios vlachos","name":"sotirios","surname":"vlachos","rank":null,"pid":[{"value":"0000-0002-1879-3745","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Archil Durglishvili","name":"Archil","surname":"Durglishvili","rank":null,"pid":[{"value":"0000-0003-4157-592X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Vladimir Cindro","name":"Vladimir","surname":"Cindro","rank":null,"pid":[{"value":"0000-0002-2037-7185","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Tim Adye","name":"Tim","surname":"Adye","rank":null,"pid":[{"value":"0000-0003-0627-5059","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Ruggero Turra","name":"Ruggero","surname":"Turra","rank":null,"pid":[{"value":"0000-0001-8740-796X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Susana Amor Santos","name":"Susana","surname":"Amor Santos","rank":null,"pid":[{"value":"0000-0001-7566-6067","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Thorsten Wengler","name":"Thorsten","surname":"Wengler","rank":null,"pid":[{"value":"0000-0002-4375-5265","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Sergio Grancagnolo","name":"Sergio","surname":"Grancagnolo","rank":null,"pid":[{"value":"0000-0001-8490-8304","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Koji Sato","name":"Koji","surname":"Sato","rank":null,"pid":[{"value":"0000-0001-8988-4065","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Marjorie Shapiro","name":"Marjorie","surname":"Shapiro","rank":null,"pid":[{"value":"0000-0001-8540-9654","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Paul Thompson","name":"Paul","surname":"Thompson","rank":null,"pid":[{"value":"0000-0002-6239-7715","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Daniele Zanzi","name":"Daniele","surname":"Zanzi","rank":null,"pid":[{"value":"0000-0002-1222-7937","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Adrian Irles","name":"Adrian","surname":"Irles","rank":null,"pid":[{"value":"0000-0001-5668-151X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Walter Hopkins","name":"Walter","surname":"Hopkins","rank":null,"pid":[{"value":"0000-0001-7814-8740","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Aurelio Juste","name":"Aurelio","surname":"Juste","rank":null,"pid":[{"value":"0000-0002-1558-3291","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Yoram Rozen","name":"Yoram","surname":"Rozen","rank":null,"pid":[{"value":"0000-0001-6969-0634","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Mateusz Dyndal","name":"Mateusz","surname":"Dyndal","rank":null,"pid":[{"value":"0000-0001-9632-6352","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Monika Wielers","name":"Monika","surname":"Wielers","rank":null,"pid":[{"value":"0000-0001-9232-4827","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Anna Kaczmarska","name":"Anna","surname":"Kaczmarska","rank":null,"pid":[{"value":"0000-0002-8880-4120","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Evgeny Khramov","name":"Evgeny","surname":"Khramov","rank":null,"pid":[{"value":"0000-0001-7400-6454","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Vincent Alexander Croft","name":"Vincent Alexander","surname":"Croft","rank":null,"pid":[{"value":"0000-0002-8731-4525","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Yuji Yamazaki","name":"Yuji","surname":"Yamazaki","rank":null,"pid":[{"value":"0000-0003-3710-6995","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Sau Lan Wu","name":"Sau Lan","surname":"Wu","rank":null,"pid":[{"value":"0000-0001-5866-1504","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Kevin Kröninger","name":"Kevin","surname":"Kröninger","rank":null,"pid":[{"value":"0000-0001-9873-0228","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Calin Alexa","name":"Calin","surname":"Alexa","rank":null,"pid":[{"value":"0000-0003-0922-7669","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Frank Filthaut","name":"Frank","surname":"Filthaut","rank":null,"pid":[{"value":"0000-0003-3338-2247","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Alaettin Serhan Mete","name":"Alaettin Serhan","surname":"Mete","rank":null,"pid":[{"value":"0000-0002-5508-530X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Michael Duehrssen-Debling","name":"Michael","surname":"Duehrssen-Debling","rank":null,"pid":[{"value":"0000-0002-5833-7058","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Torre Wenaus","name":"Torre","surname":"Wenaus","rank":null,"pid":[{"value":"0000-0002-8678-893X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Michele LIVAN","name":"Michele","surname":"LIVAN","rank":null,"pid":[{"value":"0000-0002-5877-0062","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Jose Guillermo Panduro Vazquez","name":"Jose Guillermo","surname":"Panduro Vazquez","rank":null,"pid":[{"value":"0000-0003-2605-8940","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Pascal Pralavorio","name":"Pascal","surname":"Pralavorio","rank":null,"pid":[{"value":"0000-0002-2452-6715","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Yuto Minami","name":"Yuto","surname":"Minami","rank":null,"pid":[{"value":"0000-0003-2176-8089","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Stephen Hillier","name":"Stephen","surname":"Hillier","rank":null,"pid":[{"value":"0000-0002-7599-6469","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Simon Viel","name":"Simon","surname":"Viel","rank":null,"pid":[{"value":"0000-0001-9554-4059","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Andreas Kugel","name":"Andreas","surname":"Kugel","rank":null,"pid":[{"value":"0000-0002-8493-6660","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Lene Kristian Bryngemark","name":"Lene Kristian","surname":"Bryngemark","rank":null,"pid":[{"value":"0000-0002-8420-3408","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Guenter Duckeck","name":"Guenter","surname":"Duckeck","rank":null,"pid":[{"value":"0000-0002-7756-7801","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Alexey Anisenkov","name":"Alexey","surname":"Anisenkov","rank":null,"pid":[{"value":"0000-0002-7201-5936","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Javier Jiménez Peña","name":"Javier","surname":"Jiménez Peña","rank":null,"pid":[{"value":"0000-0002-8705-628X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Sascha null","name":"Sascha","surname":null,"rank":null,"pid":[{"value":"0000-0003-2941-2829","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Evelyn Thomson","name":"Evelyn","surname":"Thomson","rank":null,"pid":[{"value":"0000-0001-6031-2768","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Andrea Formica","name":"Andrea","surname":"Formica","rank":null,"pid":[{"value":"0000-0001-8308-2643","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Mattias Ellert","name":"Mattias","surname":"Ellert","rank":null,"pid":[{"value":"0000-0001-5265-3175","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Richard Keeler","name":"Richard","surname":"Keeler","rank":null,"pid":[{"value":"0000-0002-0510-4189","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Karolos Potamianos","name":"Karolos","surname":"Potamianos","rank":null,"pid":[{"value":"0000-0001-7839-9785","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Muhammad Alhroob","name":"Muhammad","surname":"Alhroob","rank":null,"pid":[{"value":"0000-0001-7569-7111","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Marie-Helene Genest","name":"Marie-Helene","surname":"Genest","rank":null,"pid":[{"value":"0000-0002-4098-2024","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"melissa franklin","name":"melissa","surname":"franklin","rank":null,"pid":[{"value":"0000-0002-6595-883X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"John Baines","name":"John","surname":"Baines","rank":null,"pid":[{"value":"0000-0003-0770-2702","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Jean-Baptiste de Vivie","name":"Jean-Baptiste","surname":"de Vivie","rank":null,"pid":[{"value":"0000-0001-9163-2211","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Kristian Gregersen","name":"Kristian","surname":"Gregersen","rank":null,"pid":[{"value":"0000-0003-0295-1670","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Elisabeth Schopf","name":"Elisabeth","surname":"Schopf","rank":null,"pid":[{"value":"0000-0002-9340-2214","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Tristan Beau","name":"Tristan","surname":"Beau","rank":null,"pid":[{"value":"0000-0002-2022-2140","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Ask Emil Løvschall-Jensen","name":"Ask Emil","surname":"Løvschall-Jensen","rank":null,"pid":[{"value":"0000-0003-1834-4904","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Karel Smolek","name":"Karel","surname":"Smolek","rank":null,"pid":[{"value":"0000-0002-5996-7000","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Wendy Taylor","name":"Wendy","surname":"Taylor","rank":null,"pid":[{"value":"0000-0002-6596-9125","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"James Robinson","name":"James","surname":"Robinson","rank":null,"pid":[{"value":"0000-0002-2856-9413","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Arno Straessner","name":"Arno","surname":"Straessner","rank":null,"pid":[{"value":"0000-0003-2460-6659","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Bruno Lenzi","name":"Bruno","surname":"Lenzi","rank":null,"pid":[{"value":"0000-0002-1024-4004","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Thomas G McCarthy","name":"Thomas G","surname":"McCarthy","rank":null,"pid":[{"value":"0000-0002-1182-3526","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Maria Curatolo","name":"Maria","surname":"Curatolo","rank":null,"pid":[{"value":"0000-0003-0978-4879","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Zuzana Rurikova","name":"Zuzana","surname":"Rurikova","rank":null,"pid":[{"value":"0000-0003-3051-9607","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Norman Gee","name":"Norman","surname":"Gee","rank":null,"pid":[{"value":"0000-0002-8833-3154","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Craig Buttar","name":"Craig","surname":"Buttar","rank":null,"pid":[{"value":"0000-0003-0188-6491","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Sophie Trincaz-Duvoid","name":"Sophie","surname":"Trincaz-Duvoid","rank":null,"pid":[{"value":"0000-0001-5913-0828","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Danilo Enoque Ferreira de Lima","name":"Danilo Enoque","surname":"Ferreira de Lima","rank":null,"pid":[{"value":"0000-0002-6606-3595","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Nektarios Benekos","name":"Nektarios","surname":"Benekos","rank":null,"pid":[{"value":"0000-0001-7831-8762","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Evgenii Baldin","name":"Evgenii","surname":"Baldin","rank":null,"pid":[{"value":"0000-0002-9854-975X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Craig Sawyer","name":"Craig","surname":"Sawyer","rank":null,"pid":[{"value":"0000-0002-2027-1428","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"George Redlinger","name":"George","surname":"Redlinger","rank":null,"pid":[{"value":"0000-0002-6437-9991","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Philip Sommer","name":"Philip","surname":"Sommer","rank":null,"pid":[{"value":"0000-0003-1703-7304","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Simone Monzani","name":"Simone","surname":"Monzani","rank":null,"pid":[{"value":"0000-0002-0479-2207","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Eirik Gramstad","name":"Eirik","surname":"Gramstad","rank":null,"pid":[{"value":"0000-0001-5792-5352","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Anna Lipniacka","name":"Anna","surname":"Lipniacka","rank":null,"pid":[{"value":"0000-0002-8759-8564","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Oana Vickey Boeriu","name":"Oana","surname":"Vickey Boeriu","rank":null,"pid":[{"value":"0000-0002-6497-6809","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Fridolin Dittus","name":"Fridolin","surname":"Dittus","rank":null,"pid":[{"value":"0000-0002-1760-8237","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Antonio Salvucci","name":"Antonio","surname":"Salvucci","rank":null,"pid":[{"value":"0000-0003-4876-2613","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Carlo Dallapiccola","name":"Carlo","surname":"Dallapiccola","rank":null,"pid":[{"value":"0000-0002-1391-2477","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Konstantinos Ntekas","name":"Konstantinos","surname":"Ntekas","rank":null,"pid":[{"value":"0000-0001-9252-6509","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Juan Antonio Aguilar Saavedra","name":"Juan Antonio","surname":"Aguilar Saavedra","rank":null,"pid":[{"value":"0000-0002-5475-8920","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Martine Bosman","name":"Martine","surname":"Bosman","rank":null,"pid":[{"value":"0000-0002-7290-643X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Trevor Vickey","name":"Trevor","surname":"Vickey","rank":null,"pid":[{"value":"0000-0002-1596-2611","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Christian Ohm","name":"Christian","surname":"Ohm","rank":null,"pid":[{"value":"0000-0002-8015-7512","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"William Davey","name":"William","surname":"Davey","rank":null,"pid":[{"value":"0000-0002-8140-8619","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Wolfgang Walkowiak","name":"Wolfgang","surname":"Walkowiak","rank":null,"pid":[{"value":"0000-0002-0385-3784","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Austin Basye","name":"Austin","surname":"Basye","rank":null,"pid":[{"value":"0000-0002-7519-1310","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Kuhan Wang","name":"Kuhan","surname":"Wang","rank":null,"pid":[{"value":"0000-0002-6151-0034","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Cristiano Alpigiani","name":"Cristiano","surname":"Alpigiani","rank":null,"pid":[{"value":"0000-0002-7641-5814","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"James Monk","name":"James","surname":"Monk","rank":null,"pid":[{"value":"0000-0001-8471-9247","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Christopher Pollard","name":"Christopher","surname":"Pollard","rank":null,"pid":[{"value":"0000-0002-3690-3960","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Jozsef Toth","name":"Jozsef","surname":"Toth","rank":null,"pid":[{"value":"0000-0001-9128-6080","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Vitaliano Chiarella","name":"Vitaliano","surname":"Chiarella","rank":null,"pid":[{"value":"0000-0002-4210-2924","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Jos Vermeulen","name":"Jos","surname":"Vermeulen","rank":null,"pid":[{"value":"0000-0003-4378-5736","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Yann Coadou","name":"Yann","surname":"Coadou","rank":null,"pid":[{"value":"0000-0001-8195-7004","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Christine Kourkoumelis","name":"Christine","surname":"Kourkoumelis","rank":null,"pid":[{"value":"0000-0003-0083-274X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Karishma Sekhon","name":"Karishma","surname":"Sekhon","rank":null,"pid":[{"value":"0000-0001-7677-8394","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Kun Liu","name":"Kun","surname":"Liu","rank":null,"pid":[{"value":"0000-0001-5807-0501","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Matt LeBlanc","name":"Matt","surname":"LeBlanc","rank":null,"pid":[{"value":"0000-0001-5977-6418","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Ewan Hill","name":"Ewan","surname":"Hill","rank":null,"pid":[{"value":"0000-0002-1725-7414","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Diane Cinca","name":"Diane","surname":"Cinca","rank":null,"pid":[{"value":"0000-0003-0944-8998","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Benedict Tobias Winter","name":"Benedict Tobias","surname":"Winter","rank":null,"pid":[{"value":"0000-0001-9606-7688","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Steven Schramm","name":"Steven","surname":"Schramm","rank":null,"pid":[{"value":"0000-0001-9031-6751","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Yusheng Wu","name":"Yusheng","surname":"Wu","rank":null,"pid":[{"value":"0000-0002-1528-4865","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Orel Gueta","name":"Orel","surname":"Gueta","rank":null,"pid":[{"value":"0000-0002-9440-2398","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Federico Meloni","name":"Federico","surname":"Meloni","rank":null,"pid":[{"value":"0000-0001-7075-2214","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Sylvain Tisserant","name":"Sylvain","surname":"Tisserant","rank":null,"pid":[{"value":"0000-0002-0294-6727","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Dimitrios Iliadis","name":"Dimitrios","surname":"Iliadis","rank":null,"pid":[{"value":"0000-0001-6303-2761","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Sebastien Binet","name":"Sebastien","surname":"Binet","rank":null,"pid":[{"value":"0000-0003-4913-6104","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Marcella Bona","name":"Marcella","surname":"Bona","rank":null,"pid":[{"value":"0000-0002-9660-580X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Ki Lie","name":"Ki","surname":"Lie","rank":null,"pid":[{"value":"0000-0002-5779-5989","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"David Sankey","name":"David","surname":"Sankey","rank":null,"pid":[{"value":"0000-0003-0955-4213","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Michele Faucci Giannelli","name":"Michele","surname":"Faucci Giannelli","rank":null,"pid":[{"value":"0000-0003-3731-820X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Georges Azuelos","name":"Georges","surname":"Azuelos","rank":null,"pid":[{"value":"0000-0003-4241-022X","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"David Britton","name":"David","surname":"Britton","rank":null,"pid":[{"value":"0000-0001-9998-4342","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Henri Bachacou","name":"Henri","surname":"Bachacou","rank":null,"pid":[{"value":"0000-0002-2256-4515","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Stefania Stucci","name":"Stefania","surname":"Stucci","rank":null,"pid":[{"value":"0000-0002-1639-4484","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Peter Berta","name":"Peter","surname":"Berta","rank":null,"pid":[{"value":"0000-0003-0780-0345","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"SERKANT ALİ ÇETİN","name":"SERKANT ALİ","surname":"ÇETİN","rank":null,"pid":[{"value":"0000-0001-5050-8441","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Alexey Zhemchugov","name":"Alexey","surname":"Zhemchugov","rank":null,"pid":[{"value":"0000-0002-3360-4965","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Yasushi Nagasaka","name":"Yasushi","surname":"Nagasaka","rank":null,"pid":[{"value":"0000-0002-3669-9525","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"James Walder","name":"James","surname":"Walder","rank":null,"pid":[{"value":"0000-0002-9039-8758","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Marina Rotaru","name":"Marina","surname":"Rotaru","rank":null,"pid":[{"value":"0000-0003-4088-6275","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Chunhui Chen","name":"Chunhui","surname":"Chen","rank":null,"pid":[{"value":"0000-0003-1589-9955","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Valentina Tudorache","name":"Valentina","surname":"Tudorache","rank":null,"pid":[{"value":"0000-0001-5384-3843","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Amanda Cooper-Sarkar","name":"Amanda","surname":"Cooper-Sarkar","rank":null,"pid":[{"value":"0000-0002-7107-5902","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Paul Dervan","name":"Paul","surname":"Dervan","rank":null,"pid":[{"value":"0000-0003-3929-8046","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Laura Gonella","name":"Laura","surname":"Gonella","rank":null,"pid":[{"value":"0000-0002-4919-0808","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Georges Aad","name":"Georges","surname":"Aad","rank":null,"pid":[{"value":"0000-0002-6665-4934","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Hok-Chuen Cheng","name":"Hok-Chuen","surname":"Cheng","rank":null,"pid":[{"value":"0000-0002-8912-4389","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Michal Svatos","name":"Michal","surname":"Svatos","rank":null,"pid":[{"value":"0000-0002-7199-3383","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Jens Janssen","name":"Jens","surname":"Janssen","rank":null,"pid":[{"value":"0000-0002-2391-3078","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Jana Faltova","name":"Jana","surname":"Faltova","rank":null,"pid":[{"value":"0000-0003-4278-7182","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Trisha Farooque","name":"Trisha","surname":"Farooque","rank":null,"pid":[{"value":"0000-0003-1363-9324","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"andrea gabrielli","name":"andrea","surname":"gabrielli","rank":null,"pid":[{"value":"0000-0003-0768-9325","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Santiago Gonzalez de la Hoz","name":"Santiago","surname":"Gonzalez de la Hoz","rank":null,"pid":[{"value":"0000-0001-5304-5390","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"José Ocariz","name":"José","surname":"Ocariz","rank":null,"pid":[{"value":"0000-0003-2262-0780","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null},{"fullname":"Michele Weber","name":"Michele","surname":"Weber","rank":null,"pid":[{"value":"0000-0002-2770-9031","qualifier":{"classid":"ORCID","classname":"ORCID","schemeid":"ORCID","schemename":"ORCID"},"dataInfo":null}],"affiliation":null}] diff --git a/dhp-common/src/test/resources/eu/dnetlib/dhp/oa/merge/authors_publication_sample.json b/dhp-common/src/test/resources/eu/dnetlib/dhp/oa/merge/authors_publication_sample.json new file mode 100644 index 000000000..eac239b93 --- /dev/null +++ b/dhp-common/src/test/resources/eu/dnetlib/dhp/oa/merge/authors_publication_sample.json @@ -0,0 +1,2 @@ +50|arXiv_dedup_::34e03f2336b8b28286550425e65634ea +[{"fullname":"Aad, G.","name":"G.","surname":"Aad","rank":1,"pid":[],"affiliation":null},{"fullname":"Ayoub, M.K.","name":"M. K.","surname":"Ayoub","rank":2,"pid":[],"affiliation":null},{"fullname":"Bassalat, A.","name":"A.","surname":"Bassalat","rank":3,"pid":[],"affiliation":null},{"fullname":"Becot, C.","name":"C.","surname":"Becot","rank":4,"pid":[],"affiliation":null},{"fullname":"Binet, S.","name":"S.","surname":"Binet","rank":5,"pid":[],"affiliation":null},{"fullname":"Bourdarios, C.","name":"C.","surname":"Bourdarios","rank":6,"pid":[],"affiliation":null},{"fullname":"Regie, J.B. de Vivie de","name":"J. B. Vivie","surname":"Regie","rank":7,"pid":[],"affiliation":null},{"fullname":"Delgove, D.","name":"D.","surname":"Delgove","rank":8,"pid":[],"affiliation":null},{"fullname":"Duflot, L.","name":"L.","surname":"Duflot","rank":9,"pid":[],"affiliation":null},{"fullname":"Escalier, M.","name":"M.","surname":"Escalier","rank":10,"pid":[],"affiliation":null},{"fullname":"Fayard, L.","name":"L.","surname":"Fayard","rank":11,"pid":[],"affiliation":null},{"fullname":"Fournier, D.","name":"D.","surname":"Fournier","rank":12,"pid":[],"affiliation":null},{"fullname":"Gkougkousis, E.L.","name":"E. L.","surname":"Gkougkousis","rank":13,"pid":[],"affiliation":null},{"fullname":"Grivaz, J.-F.","name":"J. -F","surname":"Grivaz","rank":14,"pid":[],"affiliation":null},{"fullname":"Guillemin, T.","name":"T.","surname":"Guillemin","rank":15,"pid":[],"affiliation":null},{"fullname":"Hariri, F.","name":"F.","surname":"Hariri","rank":16,"pid":[],"affiliation":null},{"fullname":"Henrot-Versillé, S.","name":"S.","surname":"Henrot-Versillé","rank":17,"pid":[],"affiliation":null},{"fullname":"Hrivnac, J.","name":"J.","surname":"Hrivnac","rank":18,"pid":[],"affiliation":null},{"fullname":"Iconomidou-Fayard, L.","name":"L.","surname":"Iconomidou-Fayard","rank":19,"pid":[],"affiliation":null},{"fullname":"Kado, M.","name":"M.","surname":"Kado","rank":20,"pid":[],"affiliation":null},{"fullname":"Lounis, A.","name":"A.","surname":"Lounis","rank":21,"pid":[],"affiliation":null},{"fullname":"Makovec, N.","name":"N.","surname":"Makovec","rank":22,"pid":[],"affiliation":null},{"fullname":"Morange, N.","name":"N.","surname":"Morange","rank":23,"pid":[],"affiliation":null},{"fullname":"Nellist, C.","name":"C.","surname":"Nellist","rank":24,"pid":[],"affiliation":null},{"fullname":"Poggioli, L.","name":"L.","surname":"Poggioli","rank":25,"pid":[],"affiliation":null},{"fullname":"Puzo, P.","name":"P.","surname":"Puzo","rank":26,"pid":[],"affiliation":null},{"fullname":"Renaud, A.","name":"A.","surname":"Renaud","rank":27,"pid":[],"affiliation":null},{"fullname":"Rousseau, D.","name":"D.","surname":"Rousseau","rank":28,"pid":[],"affiliation":null},{"fullname":"Rybkin, G.","name":"G.","surname":"Rybkin","rank":29,"pid":[],"affiliation":null},{"fullname":"Schaffer, A.C.","name":"A. C.","surname":"Schaffer","rank":30,"pid":[],"affiliation":null},{"fullname":"Scifo, E.","name":"E.","surname":"Scifo","rank":31,"pid":[],"affiliation":null},{"fullname":"Serin, L.","name":"L.","surname":"Serin","rank":32,"pid":[],"affiliation":null},{"fullname":"Simion, S.","name":"S.","surname":"Simion","rank":33,"pid":[],"affiliation":null},{"fullname":"Tanaka, R.","name":"R.","surname":"Tanaka","rank":34,"pid":[],"affiliation":null},{"fullname":"Tran, H.L.","name":"H. L.","surname":"Tran","rank":35,"pid":[],"affiliation":null},{"fullname":"Zerwas, D.","name":"D.","surname":"Zerwas","rank":36,"pid":[],"affiliation":null},{"fullname":"Zhang, Zhongkai","name":"Zhongkai","surname":"Zhang","rank":37,"pid":[],"affiliation":null},{"fullname":"Zhao, Y.","name":"Y.","surname":"Zhao","rank":38,"pid":[],"affiliation":null},{"fullname":"Rahal, G.","name":"G.","surname":"Rahal","rank":39,"pid":[],"affiliation":null},{"fullname":"Barnovska, Z.","name":"Z.","surname":"Barnovska","rank":40,"pid":[],"affiliation":null},{"fullname":"Berger, N.","name":"N.","surname":"Berger","rank":41,"pid":[],"affiliation":null},{"fullname":"Delmastro, M.","name":"M.","surname":"Delmastro","rank":42,"pid":[],"affiliation":null},{"fullname":"Ciaccio, L. Di","name":"L. Di","surname":"Ciaccio","rank":43,"pid":[],"affiliation":null},{"fullname":"Elles, S.","name":"S.","surname":"Elles","rank":44,"pid":[],"affiliation":null},{"fullname":"Hryn’ova, T.","name":"T.","surname":"Hryn’ova","rank":45,"pid":[],"affiliation":null},{"fullname":"Jézéquel, S.","name":"S.","surname":"Jézéquel","rank":46,"pid":[],"affiliation":null},{"fullname":"Koletsou, I.","name":"I.","surname":"Koletsou","rank":47,"pid":[],"affiliation":null},{"fullname":"Lafaye, R.","name":"R.","surname":"Lafaye","rank":48,"pid":[],"affiliation":null},{"fullname":"Leveque, J.","name":"J.","surname":"Leveque","rank":49,"pid":[],"affiliation":null},{"fullname":"Massol, N.","name":"N.","surname":"Massol","rank":50,"pid":[],"affiliation":null},{"fullname":"Sauvage, G.","name":"G.","surname":"Sauvage","rank":51,"pid":[],"affiliation":null},{"fullname":"Sauvan, E.","name":"E.","surname":"Sauvan","rank":52,"pid":[],"affiliation":null},{"fullname":"Simard, O.","name":"O.","surname":"Simard","rank":53,"pid":[],"affiliation":null},{"fullname":"Todorov, T.","name":"T.","surname":"Todorov","rank":54,"pid":[],"affiliation":null},{"fullname":"Wingerter-Seez, I.","name":"I.","surname":"Wingerter-Seez","rank":55,"pid":[],"affiliation":null},{"fullname":"Albrand, S.","name":"S.","surname":"Albrand","rank":56,"pid":[],"affiliation":null},{"fullname":"Brown, J.","name":"J.","surname":"Brown","rank":57,"pid":[],"affiliation":null},{"fullname":"Collot, J.","name":"J.","surname":"Collot","rank":58,"pid":[],"affiliation":null},{"fullname":"Crépé-Renaudin, S.","name":"S.","surname":"Crépé-Renaudin","rank":59,"pid":[],"affiliation":null},{"fullname":"Delsart, P.A.","name":"P. A.","surname":"Delsart","rank":60,"pid":[],"affiliation":null},{"fullname":"Gabaldon, C.","name":"C.","surname":"Gabaldon","rank":61,"pid":[],"affiliation":null},{"fullname":"Genest, M.H.","name":"M. H.","surname":"Genest","rank":62,"pid":[],"affiliation":null},{"fullname":"Hostachy, J.Y.","name":"J. Y.","surname":"Hostachy","rank":63,"pid":[],"affiliation":null},{"fullname":"Ledroit-Guillon, F.","name":"F.","surname":"Ledroit-Guillon","rank":64,"pid":[],"affiliation":null},{"fullname":"Lleres, A.","name":"A.","surname":"Lleres","rank":65,"pid":[],"affiliation":null},{"fullname":"Lucotte, A.","name":"A.","surname":"Lucotte","rank":66,"pid":[],"affiliation":null},{"fullname":"Malek, F.","name":"F.","surname":"Malek","rank":67,"pid":[],"affiliation":null},{"fullname":"Monini, C.","name":"C.","surname":"Monini","rank":68,"pid":[],"affiliation":null},{"fullname":"Stark, J.","name":"J.","surname":"Stark","rank":69,"pid":[],"affiliation":null},{"fullname":"Trocmé, B.","name":"B.","surname":"Trocmé","rank":70,"pid":[],"affiliation":null},{"fullname":"Wu, M.","name":"M.","surname":"Wu","rank":71,"pid":[],"affiliation":null},{"fullname":"Alio, L.","name":"L.","surname":"Alio","rank":72,"pid":[],"affiliation":null},{"fullname":"Barbero, M.","name":"M.","surname":"Barbero","rank":73,"pid":[],"affiliation":null},{"fullname":"Coadou, Y.","name":"Y.","surname":"Coadou","rank":74,"pid":[],"affiliation":null},{"fullname":"Diaconu, C.","name":"C.","surname":"Diaconu","rank":75,"pid":[],"affiliation":null},{"fullname":"Diglio, Sara","name":"Sara","surname":"Diglio","rank":76,"pid":[],"affiliation":null},{"fullname":"Djama, F.","name":"F.","surname":"Djama","rank":77,"pid":[],"affiliation":null},{"fullname":"Duccu, O.","name":"O.","surname":"Duccu","rank":78,"pid":[],"affiliation":null},{"fullname":"Feligioni, L.","name":"L.","surname":"Feligioni","rank":79,"pid":[],"affiliation":null},{"fullname":"Gao, J.","name":"J.","surname":"Gao","rank":80,"pid":[],"affiliation":null},{"fullname":"Hallewell, G.D.","name":"G. D.","surname":"Hallewell","rank":81,"pid":[],"affiliation":null},{"fullname":"Hubaut, F.","name":"F.","surname":"Hubaut","rank":82,"pid":[],"affiliation":null},{"fullname":"Kahn, S.J.","name":"S. J.","surname":"Kahn","rank":83,"pid":[],"affiliation":null},{"fullname":"Knoops, E. B. F. G.","name":"E. B. F. G.","surname":"Knoops","rank":84,"pid":[],"affiliation":null},{"fullname":"Guirriec, E. Le","name":"E. Le","surname":"Guirriec","rank":85,"pid":[],"affiliation":null},{"fullname":"Liu, J.","name":"J.","surname":"Liu","rank":86,"pid":[],"affiliation":null},{"fullname":"Liu, K.","name":"K.","surname":"Liu","rank":87,"pid":[],"affiliation":null},{"fullname":"Madaffari, D.","name":"D.","surname":"Madaffari","rank":88,"pid":[],"affiliation":null},{"fullname":"Mochizuki, K.","name":"K.","surname":"Mochizuki","rank":89,"pid":[],"affiliation":null},{"fullname":"Monnier, E.","name":"E.","surname":"Monnier","rank":90,"pid":[],"affiliation":null},{"fullname":"Muanza, S.","name":"S.","surname":"Muanza","rank":91,"pid":[],"affiliation":null},{"fullname":"Nagai, Y.","name":"Y.","surname":"Nagai","rank":92,"pid":[],"affiliation":null},{"fullname":"Nagy, E.","name":"E.","surname":"Nagy","rank":93,"pid":[],"affiliation":null},{"fullname":"Pralavorio, P.","name":"P.","surname":"Pralavorio","rank":94,"pid":[],"affiliation":null},{"fullname":"Rozanov, A.","name":"A.","surname":"Rozanov","rank":95,"pid":[],"affiliation":null},{"fullname":"Serre, T.","name":"T.","surname":"Serre","rank":96,"pid":[],"affiliation":null},{"fullname":"Talby, M.","name":"M.","surname":"Talby","rank":97,"pid":[],"affiliation":null},{"fullname":"Torres, R.E. Ticse","name":"R. E. Ticse","surname":"Torres","rank":98,"pid":[],"affiliation":null},{"fullname":"Tiouchichine, E.","name":"E.","surname":"Tiouchichine","rank":99,"pid":[],"affiliation":null},{"fullname":"Tisserant, S.","name":"S.","surname":"Tisserant","rank":100,"pid":[],"affiliation":null},{"fullname":"Toth, J.","name":"J.","surname":"Toth","rank":101,"pid":[],"affiliation":null},{"fullname":"Touchard, F.","name":"F.","surname":"Touchard","rank":102,"pid":[],"affiliation":null},{"fullname":"Vacavant, L.","name":"L.","surname":"Vacavant","rank":103,"pid":[],"affiliation":null},{"fullname":"Boumediene, D.","name":"D.","surname":"Boumediene","rank":104,"pid":[],"affiliation":null},{"fullname":"Busato, Emmanuel","name":"Emmanuel","surname":"Busato","rank":105,"pid":[],"affiliation":null},{"fullname":"Calvet, D.","name":"D.","surname":"Calvet","rank":106,"pid":[],"affiliation":null},{"fullname":"Calvet, S.","name":"S.","surname":"Calvet","rank":107,"pid":[],"affiliation":null},{"fullname":"Donini, J.","name":"J.","surname":"Donini","rank":108,"pid":[],"affiliation":null},{"fullname":"Dubreuil, E.","name":"E.","surname":"Dubreuil","rank":109,"pid":[],"affiliation":null},{"fullname":"Gilles, G.","name":"G.","surname":"Gilles","rank":110,"pid":[],"affiliation":null},{"fullname":"Gris, Ph.","name":"Ph","surname":"Gris","rank":111,"pid":[],"affiliation":null},{"fullname":"Liao, H.","name":"H.","surname":"Liao","rank":112,"pid":[],"affiliation":null},{"fullname":"Madar, R.","name":"R.","surname":"Madar","rank":113,"pid":[],"affiliation":null},{"fullname":"Pallin, D.","name":"D.","surname":"Pallin","rank":114,"pid":[],"affiliation":null},{"fullname":"Saez, S.M. Romano","name":"S. M. Romano","surname":"Saez","rank":115,"pid":[],"affiliation":null},{"fullname":"Santoni, C.","name":"C.","surname":"Santoni","rank":116,"pid":[],"affiliation":null},{"fullname":"Simon, D.","name":"D.","surname":"Simon","rank":117,"pid":[],"affiliation":null},{"fullname":"Theveneaux-Pelzer, Timothée","name":"Timothée","surname":"Theveneaux-Pelzer","rank":118,"pid":[],"affiliation":null},{"fullname":"Vazeille, F.","name":"F.","surname":"Vazeille","rank":119,"pid":[],"affiliation":null},{"fullname":"Yatsenko, E.","name":"E.","surname":"Yatsenko","rank":120,"pid":[],"affiliation":null}] \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/SparkEnrichGraphWithOrcidAuthors.scala b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/SparkEnrichGraphWithOrcidAuthors.scala index e190b2b33..9b85ba4f2 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/SparkEnrichGraphWithOrcidAuthors.scala +++ b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/SparkEnrichGraphWithOrcidAuthors.scala @@ -27,6 +27,8 @@ class SparkEnrichGraphWithOrcidAuthors(propertyPath: String, args: Array[String] def enrichResult(spark: SparkSession, graphPath: String, orcidPath: String, outputPath: String): Unit = { val orcidPublication = generateOrcidTable(spark, orcidPath) + + implicit val publicationEncoder = Encoders.bean(classOf[Publication]) val aschema = new StructType() @@ -68,6 +70,16 @@ class SparkEnrichGraphWithOrcidAuthors(propertyPath: String, args: Array[String] .select("dnet_id", "orcid_authors") .cache() + + orcidPublication + .join( + entities, + lower(col("schema")).equalTo(lower(col("pid_schema"))) && + lower(col("value")).equalTo(lower(col("pid_value"))), + "inner" + ) + .groupBy(col("dnet_id")).agg(collect_set(struct(col("pid_schema"), col("pid_value")))).write.mode("Overwrite").save("/user/sandro.labruzzo/enrich_pub") + val publication = spark.read.schema(publicationEncoder.schema).json(graphPath).as[Publication] publication @@ -95,13 +107,14 @@ class SparkEnrichGraphWithOrcidAuthors(propertyPath: String, args: Array[String] .where( "identifier.schema = 'doi' or identifier.schema ='pmid' or identifier.schema ='pmc' or identifier.schema ='arxiv' or identifier.schema ='handle'" ) - orcidAuthors + val orcidPublication =orcidAuthors .join(orcidWorks, orcidAuthors("orcid").equalTo(orcidWorks("orcid"))) .select( col("identifier.schema").alias("schema"), col("identifier.value").alias("value"), struct(orcidAuthors("orcid").alias("orcid"), col("givenName"), col("familyName")).alias("author") ) + orcidPublication } } From 59111713fac09a4508899f8102946bb5f04c7910 Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Tue, 28 Nov 2023 09:00:48 +0100 Subject: [PATCH 112/148] added comment --- .../eu/dnetlib/dhp/oa/merge/AuthorMerger.java | 95 ++++--------------- .../eu/dnetlib/oa/merge/AuthorMergerTest.java | 8 +- .../SparkEnrichGraphWithOrcidAuthors.scala | 2 +- 3 files changed, 21 insertions(+), 84 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java index 852ee163d..a1c3c2cc0 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java @@ -1,17 +1,11 @@ package eu.dnetlib.dhp.oa.merge; -import java.io.FileWriter; -import java.io.IOException; import java.text.Normalizer; import java.util.*; -import java.util.function.Function; import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.MutablePair; -import org.apache.commons.lang3.tuple.Pair; import org.jetbrains.annotations.NotNull; import com.wcohen.ss.JaroWinkler; @@ -161,42 +155,8 @@ public class AuthorMerger { .replaceAll("(\\n)+", " ") .trim(); -// return Arrays.stream(fullname.split("[\\s | , | ;]+")).map(String::toLowerCase).sorted().collect(Collectors.joining()); } -// -// public static List enrichOrcid2(List baseAuthor, List orcidAuthor) { -// if (baseAuthor == null || baseAuthor.isEmpty()) -// return orcidAuthor; -// -// if (orcidAuthor == null || orcidAuthor.isEmpty()) -// return baseAuthor; -// -// if (baseAuthor.size() == 1 && orcidAuthor.size() > 10) -// return baseAuthor; -// -// -// Map> pubClusters = baseAuthor.stream().collect(Collectors.toMap(AuthorMerger::generateAuthorkey, Arrays::asList, (a, b) -> { -// a.addAll(b); -// return a; -// })); -// -// Map> orcidClusters = baseAuthor.stream().collect(Collectors.toMap(AuthorMerger::generateAuthorkey, Arrays::asList, (a, b) -> { -// a.addAll(b); -// return a; -// })); -// -// System.out.println(pubClusters.keySet().size()); -// System.out.println(orcidClusters.keySet().size()); -// -// -// -// -// return null; -// -// -// } - static int hammingDist(String str1, String str2) { if (str1.length() != str2.length()) return Math.max(str1.length(), str2.length()); @@ -220,7 +180,14 @@ public class AuthorMerger { return null; } - public static boolean checkSimilarity2(final Author left, final Author right) { + /** + * This method tries to figure out when two author are the same in the contest + * of ORCID enrichment + * @param left Author in the OAF entity + * @param right Author ORCID + * @return based on a heuristic on the names of the authors if they are the same. + */ + public static boolean checkORCIDSimilarity(final Author left, final Author right) { final Person pl = parse(left); final Person pr = parse(right); @@ -267,8 +234,16 @@ public class AuthorMerger { else return false; } + // - public static List enrichOrcid2(List baseAuthor, List orcidAuthor) { + + /** + * Method to enrich ORCID information in one list of authors based on another list + * @param baseAuthor the Author List in the OAF Entity + * @param orcidAuthor The list of ORCID Author intersected + * @return The Author List of the OAF Entity enriched with the orcid Author + */ + public static List enrichOrcid(List baseAuthor, List orcidAuthor) { if (baseAuthor == null || baseAuthor.isEmpty()) return orcidAuthor; @@ -283,7 +258,7 @@ public class AuthorMerger { oAuthor.addAll(orcidAuthor); baseAuthor.forEach(ba -> { - Optional aMatch = oAuthor.stream().filter(oa -> checkSimilarity2(ba, oa)).findFirst(); + Optional aMatch = oAuthor.stream().filter(oa -> checkORCIDSimilarity(ba, oa)).findFirst(); if (aMatch.isPresent()) { final Author sameAuthor = aMatch.get(); addPid(ba, sameAuthor.getPid()); @@ -293,40 +268,6 @@ public class AuthorMerger { return baseAuthor; } - public static List enrichOrcid(List baseAuthor, List orcidAuthor) { - - if (baseAuthor == null || baseAuthor.isEmpty()) - return orcidAuthor; - - if (orcidAuthor == null || orcidAuthor.isEmpty()) - return baseAuthor; - - if (baseAuthor.size() == 1 && orcidAuthor.size() > 10) - return baseAuthor; - - final Double similarityMatrix[][] = new Double[baseAuthor.size()][orcidAuthor.size()]; - - final List maxColums = new ArrayList<>(); - - for (int i = 0; i < orcidAuthor.size(); i++) - maxColums.add(new SimilarityCellInfo()); - - for (int i = 0; i < baseAuthor.size(); i++) { - for (int j = 0; j < orcidAuthor.size(); j++) { - similarityMatrix[i][j] = sim(baseAuthor.get(i), orcidAuthor.get(j)); - if (maxColums.get(j).maxColumnSimilarity < similarityMatrix[i][j]) - maxColums.get(j).setValues(i, j, similarityMatrix[i][j]); - } - } - maxColums - .stream() - .sorted() - .filter(si -> si.maxColumnSimilarity > 0.85) - .forEach(si -> addPid(baseAuthor.get(si.authorPosition), orcidAuthor.get(si.orcidPosition).getPid())); - return baseAuthor; - - } - private static void addPid(final Author a, final List pids) { if (a.getPid() == null) { diff --git a/dhp-common/src/test/java/eu/dnetlib/oa/merge/AuthorMergerTest.java b/dhp-common/src/test/java/eu/dnetlib/oa/merge/AuthorMergerTest.java index 9eccab5f1..a11d49b1e 100644 --- a/dhp-common/src/test/java/eu/dnetlib/oa/merge/AuthorMergerTest.java +++ b/dhp-common/src/test/java/eu/dnetlib/oa/merge/AuthorMergerTest.java @@ -4,13 +4,9 @@ package eu.dnetlib.oa.merge; import static org.junit.jupiter.api.Assertions.*; import java.io.BufferedReader; -import java.io.FileReader; -import java.io.IOException; import java.io.InputStreamReader; -import java.util.Arrays; import java.util.List; import java.util.Objects; -import java.util.stream.Collectors; import org.junit.jupiter.api.Test; import org.junit.platform.commons.util.StringUtils; @@ -67,7 +63,7 @@ public class AuthorMergerTest { long start = System.currentTimeMillis(); // final List enrichedList = AuthorMerger.enrichOrcid(publicationAuthors, orcidAuthors); - final List enrichedList = AuthorMerger.enrichOrcid2(publicationAuthors, orcidAuthors); + final List enrichedList = AuthorMerger.enrichOrcid(publicationAuthors, orcidAuthors); long enrichedAuthorWithPid = enrichedList .stream() @@ -105,7 +101,7 @@ public class AuthorMergerTest { right.setSurname("Anand"); right.setFullname("Rachna, Anand"); // System.out.println(AuthorMerger.normalize(right.getFullname())); - boolean same = AuthorMerger.checkSimilarity2(left, right); + boolean same = AuthorMerger.checkORCIDSimilarity(left, right); assertTrue(same); diff --git a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/SparkEnrichGraphWithOrcidAuthors.scala b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/SparkEnrichGraphWithOrcidAuthors.scala index 9b85ba4f2..3c9e04a21 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/SparkEnrichGraphWithOrcidAuthors.scala +++ b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/SparkEnrichGraphWithOrcidAuthors.scala @@ -89,7 +89,7 @@ class SparkEnrichGraphWithOrcidAuthors(propertyPath: String, args: Array[String] p } case (p: Publication, r: Row) => - p.setAuthor(AuthorMerger.enrichOrcid2(p.getAuthor, AuthorEnricher.toOAFAuthor(r))) + p.setAuthor(AuthorMerger.enrichOrcid(p.getAuthor, AuthorEnricher.toOAFAuthor(r))) p } .write From aa239ec673a0891e0adc45617a2239724d9e9daa Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Wed, 29 Nov 2023 11:17:41 +0100 Subject: [PATCH 113/148] Changed implementation of check similarity to verify exact match of name instead of the first char --- .../java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java | 3 +-- .../java/eu/dnetlib/oa/merge/AuthorMergerTest.java | 10 +++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java index a1c3c2cc0..62e6764b7 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java @@ -222,7 +222,7 @@ public class AuthorMerger { nl -> pr .getName() .stream() - .anyMatch(nr -> nr.substring(0, 1).equalsIgnoreCase(nl.substring(0, 1))))) + .anyMatch(nr -> nr.equalsIgnoreCase(nl)))) return true; } @@ -236,7 +236,6 @@ public class AuthorMerger { } // - /** * Method to enrich ORCID information in one list of authors based on another list * @param baseAuthor the Author List in the OAF Entity diff --git a/dhp-common/src/test/java/eu/dnetlib/oa/merge/AuthorMergerTest.java b/dhp-common/src/test/java/eu/dnetlib/oa/merge/AuthorMergerTest.java index a11d49b1e..c0a8d6927 100644 --- a/dhp-common/src/test/java/eu/dnetlib/oa/merge/AuthorMergerTest.java +++ b/dhp-common/src/test/java/eu/dnetlib/oa/merge/AuthorMergerTest.java @@ -19,15 +19,19 @@ import eu.dnetlib.dhp.schema.oaf.Author; public class AuthorMergerTest { - @Test public void testEnrcichAuthor() throws Exception { final ObjectMapper mapper = new ObjectMapper(); BufferedReader pr = new BufferedReader(new InputStreamReader( - Objects.requireNonNull(AuthorMergerTest.class.getResourceAsStream("/eu/dnetlib/dhp/oa/merge/authors_publication_sample.json")))); + Objects + .requireNonNull( + AuthorMergerTest.class + .getResourceAsStream("/eu/dnetlib/dhp/oa/merge/authors_publication_sample.json")))); BufferedReader or = new BufferedReader(new InputStreamReader( - Objects.requireNonNull(AuthorMergerTest.class.getResourceAsStream("/eu/dnetlib/dhp/oa/merge/authors_orcid_sample.json")))); + Objects + .requireNonNull( + AuthorMergerTest.class.getResourceAsStream("/eu/dnetlib/dhp/oa/merge/authors_orcid_sample.json")))); TypeReference> aclass = new TypeReference>() { }; From 279100fa5220df87e17b535e14619a49fa726c70 Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Wed, 29 Nov 2023 11:17:58 +0100 Subject: [PATCH 114/148] added test --- .../dhp/enrich/orcid/EnrichOrcidTest.scala | 72 ++++++++++++++++++- 1 file changed, 69 insertions(+), 3 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/enrich/orcid/EnrichOrcidTest.scala b/dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/enrich/orcid/EnrichOrcidTest.scala index 0ddb7c0aa..f58b06318 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/enrich/orcid/EnrichOrcidTest.scala +++ b/dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/enrich/orcid/EnrichOrcidTest.scala @@ -1,12 +1,78 @@ package eu.dnetlib.dhp.enrich.orcid - -import org.apache.spark.SparkConf -import org.apache.spark.sql.SparkSession +import eu.dnetlib.dhp.schema.oaf.Publication +import org.apache.spark.sql.{Column, Encoder, Encoders, Row, SparkSession} import org.junit.jupiter.api.Test import org.slf4j.{Logger, LoggerFactory} +import org.apache.spark.sql.functions._ + + +case class Pid(pidScheme: String, pidValue: String) {} + +case class AuthorPid(fullName: String, pids: List[Pid]) {} + +case class PubSummary(id: String, authorWithPids: List[AuthorPid]) class EnrichOrcidTest { val log: Logger = LoggerFactory.getLogger(getClass) + + def orcid_intersection_wrong(p: PubSummary): PubSummary = { + + if (p.authorWithPids.isEmpty) + null + else { + val incorrectAuthor = p.authorWithPids.filter(a => a.pids.filter(p => p.pidScheme != null && p.pidScheme.toLowerCase.contains("orcid")).map(p => p.pidValue.toLowerCase).distinct.size > 1) + if (incorrectAuthor.nonEmpty) { + PubSummary(p.id, incorrectAuthor) + } + else { + null + } + } + } + + + + def test() = { + val spark = SparkSession.builder().master("local[*]").getOrCreate() + spark.sparkContext.setLogLevel("ERROR") + + val schema = Encoders.bean(classOf[Publication]).schema + + + val simplifyAuthor = udf((r: Seq[Row]) => { + r + .map(k => + AuthorPid(k.getAs[String]("fullname"), + k.getAs[Seq[Row]]("pid") + .map( + p => Pid(p.getAs[Row]("qualifier").getAs[String]("classid"), p.getAs[String]("value")) + ).toList) + ).filter(l => l.pids.nonEmpty) + .toList + } + ) + + val wrong_orcid_intersection = udf((a: Seq[Row]) => { + a.map(author => { + val pids_with_orcid: Seq[Row] = author.getAs[Seq[Row]]("pids").filter(p => p.getAs[String]("pidScheme")!= null && p.getAs[String]("pidScheme").toLowerCase.contains("orcid")) + if (pids_with_orcid.exists(p => p.getAs[String]("pidScheme").equals("ORCID"))) { + if (pids_with_orcid.map(p => p.getAs[String]("pidValue").toLowerCase).distinct.size > 1) { + AuthorPid(author.getAs[String]("fullName"),pids_with_orcid.map(p => Pid(p.getAs[String]("pidScheme"),p.getAs[String]("pidValue"))).toList ) + + } + else + null + } else + + null + }).filter(author => author != null) + }) + val enriched = spark.read.schema(schema).json("/Users/sandro/orcid_test/publication_enriched").select(col("id"), simplifyAuthor(col("author")).alias("authors")) + .select(col("id"), wrong_orcid_intersection(col("authors")).alias("wi")).where("wi is not null") + enriched.show(20, 1000, true) + } + + } From af1c2634b3fc61c208dfd8e57fff1b11459dba5f Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Wed, 29 Nov 2023 12:45:30 +0100 Subject: [PATCH 115/148] added instanceTypeMapping original field in the mapping of - DOIBoost - Datacite - PubMed - Scholexplorer Datasource --- .../DataciteToOAFTransformation.scala | 15 ++++++++----- .../eu/dnetlib/dhp/sx/bio/BioDBToOAF.scala | 21 +++++++++++++++++-- .../dhp/sx/bio/pubmed/PubMedToOaf.scala | 18 ++++++++++++---- .../doiboost/crossref/Crossref2Oaf.scala | 8 +++++-- 4 files changed, 49 insertions(+), 13 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/datacite/DataciteToOAFTransformation.scala b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/datacite/DataciteToOAFTransformation.scala index 45f5f9729..17f1b90b0 100644 --- a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/datacite/DataciteToOAFTransformation.scala +++ b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/datacite/DataciteToOAFTransformation.scala @@ -166,7 +166,7 @@ object DataciteToOAFTransformation { resourceTypeGeneral: String, schemaOrg: String, vocabularies: VocabularyGroup - ): (Qualifier, Qualifier) = { + ): (Qualifier, Qualifier, String) = { if (resourceType != null && resourceType.nonEmpty) { val typeQualifier = vocabularies.getSynonymAsQualifier(ModelConstants.DNET_PUBLICATION_RESOURCE, resourceType) @@ -176,7 +176,7 @@ object DataciteToOAFTransformation { vocabularies.getSynonymAsQualifier( ModelConstants.DNET_RESULT_TYPOLOGIES, typeQualifier.getClassid - ) + ), resourceType ) } if (schemaOrg != null && schemaOrg.nonEmpty) { @@ -188,7 +188,7 @@ object DataciteToOAFTransformation { vocabularies.getSynonymAsQualifier( ModelConstants.DNET_RESULT_TYPOLOGIES, typeQualifier.getClassid - ) + ), schemaOrg ) } @@ -203,7 +203,7 @@ object DataciteToOAFTransformation { vocabularies.getSynonymAsQualifier( ModelConstants.DNET_RESULT_TYPOLOGIES, typeQualifier.getClassid - ) + ), resourceTypeGeneral ) } @@ -216,12 +216,17 @@ object DataciteToOAFTransformation { schemaOrg: String, vocabularies: VocabularyGroup ): Result = { - val typeQualifiers: (Qualifier, Qualifier) = + val typeQualifiers: (Qualifier, Qualifier, String) = getTypeQualifier(resourceType, resourceTypeGeneral, schemaOrg, vocabularies) if (typeQualifiers == null) return null val i = new Instance i.setInstancetype(typeQualifiers._1) + // ADD ORIGINAL TYPE + val itm = new InstanceTypeMapping + itm.setOriginalType(typeQualifiers._3) + i.setInstanceTypeMapping(List(itm).asJava) + typeQualifiers._2.getClassname match { case "dataset" => val r = new OafDataset diff --git a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/BioDBToOAF.scala b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/BioDBToOAF.scala index c079f7537..3624c5369 100644 --- a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/BioDBToOAF.scala +++ b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/BioDBToOAF.scala @@ -176,7 +176,7 @@ object BioDBToOAF { i.setUrl(List(s"${resolvedURL(input.pidType)}${input.pid}").asJava) } - if (input.pidType.equalsIgnoreCase("clinicaltrials.gov")) + if (input.pidType.equalsIgnoreCase("clinicaltrials.gov")) { i.setInstancetype( OafMapperUtils.qualifier( "0037", @@ -185,7 +185,10 @@ object BioDBToOAF { ModelConstants.DNET_PUBLICATION_RESOURCE ) ) - else + val itm = new InstanceTypeMapping + itm.setOriginalType(input.pidType) + i.setInstanceTypeMapping(List(itm).asJava) + } else { i.setInstancetype( OafMapperUtils.qualifier( "0046", @@ -194,6 +197,10 @@ object BioDBToOAF { ModelConstants.DNET_PUBLICATION_RESOURCE ) ) + val itm = new InstanceTypeMapping + itm.setOriginalType("Bioentity") + i.setInstanceTypeMapping(List(itm).asJava) + } if (input.datasource == null || input.datasource.isEmpty) return null @@ -265,6 +272,9 @@ object BioDBToOAF { ModelConstants.DNET_PUBLICATION_RESOURCE ) ) + val itm = new InstanceTypeMapping + itm.setOriginalType("Bioentity") + i.setInstanceTypeMapping(List(itm).asJava) i.setCollectedfrom(collectedFromMap("uniprot")) d.setInstance(List(i).asJava) @@ -471,6 +481,9 @@ object BioDBToOAF { ModelConstants.DNET_PUBLICATION_RESOURCE ) ) + val itm = new InstanceTypeMapping + itm.setOriginalType("Bioentity") + i.setInstanceTypeMapping(List(itm).asJava) i.setCollectedfrom(collectedFromMap("pdb")) d.setInstance(List(i).asJava) @@ -571,6 +584,10 @@ object BioDBToOAF { ModelConstants.DNET_PUBLICATION_RESOURCE ) ) + val itm = new InstanceTypeMapping + itm.setOriginalType("Bioentity") + i.setInstanceTypeMapping(List(itm).asJava) + i.setCollectedfrom(collectedFromMap("ebi")) d.setInstance(List(i).asJava) diff --git a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/pubmed/PubMedToOaf.scala b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/pubmed/PubMedToOaf.scala index 410686f97..40ac48a7a 100644 --- a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/pubmed/PubMedToOaf.scala +++ b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/pubmed/PubMedToOaf.scala @@ -188,12 +188,22 @@ object PubMedToOaf { val cojbCategory = getVocabularyTerm(ModelConstants.DNET_PUBLICATION_RESOURCE, vocabularies, ja.get.getValue) pubmedInstance.setInstancetype(cojbCategory) + // ADD ORIGINAL TYPE to the publication + val itm = new InstanceTypeMapping + itm.setOriginalType(ja.get.getValue) + pubmedInstance.setInstanceTypeMapping(List(itm).asJava) } else { val i_type = article.getPublicationTypes.asScala - .map(s => getVocabularyTerm(ModelConstants.DNET_PUBLICATION_RESOURCE, vocabularies, s.getValue)) - .find(q => q != null) - if (i_type.isDefined) - pubmedInstance.setInstancetype(i_type.get) + .map(s => (s.getValue,getVocabularyTerm(ModelConstants.DNET_PUBLICATION_RESOURCE, vocabularies, s.getValue))) + .find(q => q._2 != null) + + if (i_type.isDefined) { + pubmedInstance.setInstancetype(i_type.get._2) + // ADD ORIGINAL TYPE to the publication + val itm = new InstanceTypeMapping + itm.setOriginalType(i_type.get._1) + pubmedInstance.setInstanceTypeMapping(List(itm).asJava) + } else return null } diff --git a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala index 565d34e62..f45c303f0 100644 --- a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala +++ b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala @@ -107,7 +107,7 @@ case object Crossref2Oaf { .map(f => f.id) } - def mappingResult(result: Result, json: JValue, cobjCategory: String): Result = { + def mappingResult(result: Result, json: JValue, cobjCategory: String, originalType:String): Result = { implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats //MAPPING Crossref DOI into PID @@ -283,6 +283,10 @@ case object Crossref2Oaf { ModelConstants.DNET_PUBLICATION_RESOURCE ) ) + //ADD ORIGINAL TYPE to the mapping + val itm = new InstanceTypeMapping + itm.setOriginalType(originalType) + instance.setInstanceTypeMapping(List(itm).asJava) result.setResourcetype( OafMapperUtils.qualifier( cobjCategory.substring(0, 4), @@ -367,7 +371,7 @@ case object Crossref2Oaf { objectType, mappingCrossrefSubType.getOrElse(objectSubType, "0038 Other literature type") ) - mappingResult(result, json, cOBJCategory) + mappingResult(result, json, cOBJCategory, originalType) if (result == null || result.getId == null) return List() From 86b5775e0859233ececfbc7425afee5a5b2c2c6f Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Wed, 29 Nov 2023 13:15:43 +0100 Subject: [PATCH 116/148] added vocabulary in instanceTypeMapping for - DOIBoost - Datacite - PubMed - Scholexplorer Datasource --- .../dnetlib/dhp/datacite/DataciteToOAFTransformation.scala | 2 ++ .../src/main/scala/eu/dnetlib/dhp/sx/bio/BioDBToOAF.scala | 5 +++++ .../scala/eu/dnetlib/dhp/sx/bio/pubmed/PubMedToOaf.scala | 2 ++ .../scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala | 1 + 4 files changed, 10 insertions(+) diff --git a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/datacite/DataciteToOAFTransformation.scala b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/datacite/DataciteToOAFTransformation.scala index 17f1b90b0..ee3660918 100644 --- a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/datacite/DataciteToOAFTransformation.scala +++ b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/datacite/DataciteToOAFTransformation.scala @@ -225,8 +225,10 @@ object DataciteToOAFTransformation { // ADD ORIGINAL TYPE val itm = new InstanceTypeMapping itm.setOriginalType(typeQualifiers._3) + itm.setVocabularyName(ModelConstants.OPENAIRE_COAR_RESOURCE_TYPES_3_1) i.setInstanceTypeMapping(List(itm).asJava) + typeQualifiers._2.getClassname match { case "dataset" => val r = new OafDataset diff --git a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/BioDBToOAF.scala b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/BioDBToOAF.scala index 3624c5369..89bc8d948 100644 --- a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/BioDBToOAF.scala +++ b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/BioDBToOAF.scala @@ -187,6 +187,7 @@ object BioDBToOAF { ) val itm = new InstanceTypeMapping itm.setOriginalType(input.pidType) + itm.setVocabularyName(ModelConstants.OPENAIRE_COAR_RESOURCE_TYPES_3_1) i.setInstanceTypeMapping(List(itm).asJava) } else { i.setInstancetype( @@ -199,6 +200,7 @@ object BioDBToOAF { ) val itm = new InstanceTypeMapping itm.setOriginalType("Bioentity") + itm.setVocabularyName(ModelConstants.OPENAIRE_COAR_RESOURCE_TYPES_3_1) i.setInstanceTypeMapping(List(itm).asJava) } @@ -274,6 +276,7 @@ object BioDBToOAF { ) val itm = new InstanceTypeMapping itm.setOriginalType("Bioentity") + itm.setVocabularyName(ModelConstants.OPENAIRE_COAR_RESOURCE_TYPES_3_1) i.setInstanceTypeMapping(List(itm).asJava) i.setCollectedfrom(collectedFromMap("uniprot")) @@ -483,6 +486,7 @@ object BioDBToOAF { ) val itm = new InstanceTypeMapping itm.setOriginalType("Bioentity") + itm.setVocabularyName(ModelConstants.OPENAIRE_COAR_RESOURCE_TYPES_3_1) i.setInstanceTypeMapping(List(itm).asJava) i.setCollectedfrom(collectedFromMap("pdb")) @@ -586,6 +590,7 @@ object BioDBToOAF { ) val itm = new InstanceTypeMapping itm.setOriginalType("Bioentity") + itm.setVocabularyName(ModelConstants.OPENAIRE_COAR_RESOURCE_TYPES_3_1) i.setInstanceTypeMapping(List(itm).asJava) diff --git a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/pubmed/PubMedToOaf.scala b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/pubmed/PubMedToOaf.scala index 40ac48a7a..f3f8b4f02 100644 --- a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/pubmed/PubMedToOaf.scala +++ b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/pubmed/PubMedToOaf.scala @@ -191,6 +191,7 @@ object PubMedToOaf { // ADD ORIGINAL TYPE to the publication val itm = new InstanceTypeMapping itm.setOriginalType(ja.get.getValue) + itm.setVocabularyName(ModelConstants.OPENAIRE_COAR_RESOURCE_TYPES_3_1) pubmedInstance.setInstanceTypeMapping(List(itm).asJava) } else { val i_type = article.getPublicationTypes.asScala @@ -202,6 +203,7 @@ object PubMedToOaf { // ADD ORIGINAL TYPE to the publication val itm = new InstanceTypeMapping itm.setOriginalType(i_type.get._1) + itm.setVocabularyName(ModelConstants.OPENAIRE_COAR_RESOURCE_TYPES_3_1) pubmedInstance.setInstanceTypeMapping(List(itm).asJava) } else diff --git a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala index f45c303f0..65e395357 100644 --- a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala +++ b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala @@ -286,6 +286,7 @@ case object Crossref2Oaf { //ADD ORIGINAL TYPE to the mapping val itm = new InstanceTypeMapping itm.setOriginalType(originalType) + itm.setVocabularyName(ModelConstants.OPENAIRE_COAR_RESOURCE_TYPES_3_1) instance.setInstanceTypeMapping(List(itm).asJava) result.setResourcetype( OafMapperUtils.qualifier( From 7b5e04f37ec57971fbd588a9a6cca0314d06b045 Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Thu, 30 Nov 2023 14:36:50 +0100 Subject: [PATCH 117/148] removed Orcid intersection on DOIBoost --- .../preprocess/oozie_app/workflow.xml | 26 ---- .../doiboost/process/oozie_app/workflow.xml | 34 +----- .../doiboost/SparkGenerateDoiBoost.scala | 17 +-- .../dhp/enrich/orcid/oozie_app/workflow.xml | 2 +- .../dhp/enrich/orcid/AuthorEnricher.scala | 7 +- .../SparkEnrichGraphWithOrcidAuthors.scala | 106 ++++++++-------- .../dhp/enrich/orcid/EnrichOrcidTest.scala | 113 ++++++++++-------- 7 files changed, 128 insertions(+), 177 deletions(-) diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/preprocess/oozie_app/workflow.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/preprocess/oozie_app/workflow.xml index 40a17b486..ed6853229 100644 --- a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/preprocess/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/preprocess/oozie_app/workflow.xml @@ -133,32 +133,6 @@ --targetPath${inputPathMAG}/dataset --masteryarn-cluster - - - - - - - - yarn-cluster - cluster - Convert ORCID to Dataset - eu.dnetlib.doiboost.orcid.SparkPreprocessORCID - dhp-doiboost-${projectVersion}.jar - - --executor-memory=${sparkExecutorMemory} - --executor-cores=${sparkExecutorCores} - --driver-memory=${sparkDriverMemory} - --conf spark.sql.shuffle.partitions=3840 - --conf spark.extraListeners=${spark2ExtraListeners} - --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} - --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} - --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - - --sourcePath${inputPathOrcid} - --workingPath${workingPathOrcid} - --masteryarn-cluster - diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/process/oozie_app/workflow.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/process/oozie_app/workflow.xml index 29a12f4df..8f28d706d 100644 --- a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/process/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/process/oozie_app/workflow.xml @@ -59,10 +59,10 @@ - - workingPathOrcid - the ORCID working path - + + + + @@ -170,32 +170,6 @@ --targetPath${workingPath}/uwPublication --masteryarn-cluster - - - - - - - - yarn-cluster - cluster - Convert ORCID to Dataset - eu.dnetlib.doiboost.orcid.SparkConvertORCIDToOAF - dhp-doiboost-${projectVersion}.jar - - --executor-memory=${sparkExecutorMemory} - --executor-cores=${sparkExecutorCores} - --driver-memory=${sparkDriverMemory} - --conf spark.sql.shuffle.partitions=3840 - --conf spark.extraListeners=${spark2ExtraListeners} - --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} - --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} - --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - - --workingPath${workingPathOrcid} - --targetPath${workingPath}/orcidPublication - --masteryarn-cluster - diff --git a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/SparkGenerateDoiBoost.scala b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/SparkGenerateDoiBoost.scala index 2cbd53097..07d6a0287 100644 --- a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/SparkGenerateDoiBoost.scala +++ b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/SparkGenerateDoiBoost.scala @@ -66,7 +66,7 @@ object SparkGenerateDoiBoost { Encoders.tuple(Encoders.STRING, mapEncoderPub) implicit val mapEncoderRel: Encoder[Relation] = Encoders.kryo[Relation] - logger.info("Phase 2) Join Crossref with UnpayWall") + logger.info("Phase 1) Join Crossref with UnpayWall") val crossrefPublication: Dataset[(String, Publication)] = spark.read.load(s"$workingDirPath/crossrefPublication").as[Publication].map(p => (p.getId, p)) @@ -91,20 +91,11 @@ object SparkGenerateDoiBoost { .write .mode(SaveMode.Overwrite) .save(s"$workingDirPath/firstJoin") - logger.info("Phase 3) Join Result with ORCID") - val fj: Dataset[(String, Publication)] = - spark.read.load(s"$workingDirPath/firstJoin").as[Publication].map(p => (p.getId, p)) - val orcidPublication: Dataset[(String, Publication)] = - spark.read.load(s"$workingDirPath/orcidPublication").as[Publication].map(p => (p.getId, p)) - fj.joinWith(orcidPublication, fj("_1").equalTo(orcidPublication("_1")), "left") - .map(applyMerge) - .write - .mode(SaveMode.Overwrite) - .save(s"$workingDirPath/secondJoin") - logger.info("Phase 4) Join Result with MAG") + + logger.info("Phase 2) Join Result with MAG") val sj: Dataset[(String, Publication)] = - spark.read.load(s"$workingDirPath/secondJoin").as[Publication].map(p => (p.getId, p)) + spark.read.load(s"$workingDirPath/firstJoin").as[Publication].map(p => (p.getId, p)) val magPublication: Dataset[(String, Publication)] = spark.read.load(s"$workingDirPath/magPublication").as[Publication].map(p => (p.getId, p)) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/enrich/orcid/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/enrich/orcid/oozie_app/workflow.xml index 1284cceda..87c4dcb4f 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/enrich/orcid/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/enrich/orcid/oozie_app/workflow.xml @@ -40,7 +40,7 @@ --orcidPath${orcidPath} --targetPath${targetPath} - --graphPath${graphPath}/publication + --graphPath${graphPath} --masteryarn diff --git a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/AuthorEnricher.scala b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/AuthorEnricher.scala index a67de4b95..15513c8af 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/AuthorEnricher.scala +++ b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/AuthorEnricher.scala @@ -1,5 +1,6 @@ package eu.dnetlib.dhp.enrich.orcid +import eu.dnetlib.dhp.schema.common.ModelConstants import eu.dnetlib.dhp.schema.oaf.{Author, Publication} import eu.dnetlib.dhp.schema.sx.OafUtils import org.apache.spark.sql.Row @@ -13,9 +14,11 @@ object AuthorEnricher extends Serializable { a.setName(givenName) a.setSurname(familyName) a.setFullname(s"$givenName $familyName") - a.setPid(List(OafUtils.createSP(orcid, "ORCID", "ORCID")).asJava) + val pid = OafUtils.createSP(orcid, ModelConstants.ORCID, ModelConstants.ORCID) + pid.setDataInfo(OafUtils.generateDataInfo()) + pid.getDataInfo.setProvenanceaction(OafUtils.createQualifier("ORCID_ENRICHMENT", "ORCID_ENRICHMENT")) + a.setPid(List(pid).asJava) a - } def toOAFAuthor(r: Row): java.util.List[Author] = { diff --git a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/SparkEnrichGraphWithOrcidAuthors.scala b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/SparkEnrichGraphWithOrcidAuthors.scala index 3c9e04a21..0d994d202 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/SparkEnrichGraphWithOrcidAuthors.scala +++ b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/SparkEnrichGraphWithOrcidAuthors.scala @@ -1,13 +1,11 @@ package eu.dnetlib.dhp.enrich.orcid -import com.fasterxml.jackson.databind.ObjectMapper import eu.dnetlib.dhp.application.AbstractScalaApplication import eu.dnetlib.dhp.oa.merge.AuthorMerger -import eu.dnetlib.dhp.schema.oaf.{Author, DataInfo, Instance, Publication, StructuredProperty} -import org.apache.spark.sql.{Dataset, Encoder, Encoders, Row, SaveMode, SparkSession} -import org.apache.spark.sql.functions.{col, collect_set, concat, explode, expr, first, flatten, lower, size, struct} +import eu.dnetlib.dhp.schema.oaf.{OtherResearchProduct, Publication, Result, Software} +import org.apache.spark.sql.functions._ +import org.apache.spark.sql._ import org.slf4j.{Logger, LoggerFactory} -import org.apache.spark.sql.types._ class SparkEnrichGraphWithOrcidAuthors(propertyPath: String, args: Array[String], log: Logger) extends AbstractScalaApplication(propertyPath, args, log: Logger) { @@ -22,33 +20,49 @@ class SparkEnrichGraphWithOrcidAuthors(propertyPath: String, args: Array[String] log.info(s"orcidPath is '$orcidPath'") val targetPath = parser.get("targetPath") log.info(s"targetPath is '$targetPath'") - enrichResult(spark, graphPath, orcidPath, targetPath) + val orcidPublication: Dataset[Row] = generateOrcidTable(spark, orcidPath) + enrichResult( + spark, + s"$graphPath/publication", + orcidPublication, + s"$targetPath/publication", + Encoders.bean(classOf[Publication]) + ) + enrichResult( + spark, + s"$graphPath/dataset", + orcidPublication, + s"$targetPath/dataset", + Encoders.bean(classOf[eu.dnetlib.dhp.schema.oaf.Dataset]) + ) + enrichResult( + spark, + s"$graphPath/software", + orcidPublication, + s"$targetPath/software", + Encoders.bean(classOf[Software]) + ) + enrichResult( + spark, + s"$graphPath/otherresearchproduct", + orcidPublication, + s"$targetPath/otherresearchproduct", + Encoders.bean(classOf[OtherResearchProduct]) + ) } - def enrichResult(spark: SparkSession, graphPath: String, orcidPath: String, outputPath: String): Unit = { - val orcidPublication = generateOrcidTable(spark, orcidPath) + private def enrichResult[T <: Result]( + spark: SparkSession, + graphPath: String, + orcidPublication: Dataset[Row], + outputPath: String, + enc: Encoder[T] + ): Unit = { - - implicit val publicationEncoder = Encoders.bean(classOf[Publication]) - - val aschema = new StructType() - .add("id", StringType) - .add("dataInfo", Encoders.bean(classOf[DataInfo]).schema) - .add( - "author",Encoders.bean(classOf[Author]).schema - - ) - - val schema = new StructType() - .add("id", StringType) - .add("dataInfo", Encoders.bean(classOf[DataInfo]).schema) - .add( - "instance", - ArrayType(new StructType().add("pid", ArrayType(Encoders.bean(classOf[StructuredProperty]).schema))) - ) val entities = spark.read - .schema(schema) + .schema(enc.schema) .json(graphPath) + .select(col("id"), col("datainfo"), col("instance")) .where("datainfo.deletedbyinference = false") .drop("datainfo") .withColumn("instances", explode(col("instance"))) @@ -58,7 +72,8 @@ class SparkEnrichGraphWithOrcidAuthors(propertyPath: String, args: Array[String] col("pids.value").alias("pid_value"), col("id").alias("dnet_id") ) - val orcidDnet = orcidPublication + + val orcidDnet = orcidPublication .join( entities, lower(col("schema")).equalTo(lower(col("pid_schema"))) && @@ -69,36 +84,25 @@ class SparkEnrichGraphWithOrcidAuthors(propertyPath: String, args: Array[String] .agg(collect_set(orcidPublication("author")).alias("orcid_authors")) .select("dnet_id", "orcid_authors") .cache() + orcidDnet.count() + val result = spark.read.schema(enc.schema).json(graphPath).as[T](enc) - - orcidPublication - .join( - entities, - lower(col("schema")).equalTo(lower(col("pid_schema"))) && - lower(col("value")).equalTo(lower(col("pid_value"))), - "inner" - ) - .groupBy(col("dnet_id")).agg(collect_set(struct(col("pid_schema"), col("pid_value")))).write.mode("Overwrite").save("/user/sandro.labruzzo/enrich_pub") - - val publication = spark.read.schema(publicationEncoder.schema).json(graphPath).as[Publication] - - publication - .joinWith(orcidDnet, publication("id").equalTo(orcidDnet("dnet_id")), "left") + result + .joinWith(orcidDnet, result("id").equalTo(orcidDnet("dnet_id")), "left") .map { - case (p: Publication, null) => { - p - } - case (p: Publication, r: Row) => + case (r: T, null) => + r + case (p: T, r: Row) => p.setAuthor(AuthorMerger.enrichOrcid(p.getAuthor, AuthorEnricher.toOAFAuthor(r))) p - } + }(enc) .write .mode(SaveMode.Overwrite) .option("compression", "gzip") .json(outputPath) } - def generateOrcidTable(spark: SparkSession, inputPath: String): Dataset[Row] = { + private def generateOrcidTable(spark: SparkSession, inputPath: String): Dataset[Row] = { val orcidAuthors = spark.read.load(s"$inputPath/Authors").select("orcid", "familyName", "givenName", "creditName", "otherNames") val orcidWorks = spark.read @@ -107,14 +111,14 @@ class SparkEnrichGraphWithOrcidAuthors(propertyPath: String, args: Array[String] .where( "identifier.schema = 'doi' or identifier.schema ='pmid' or identifier.schema ='pmc' or identifier.schema ='arxiv' or identifier.schema ='handle'" ) - val orcidPublication =orcidAuthors + val orcidPublication = orcidAuthors .join(orcidWorks, orcidAuthors("orcid").equalTo(orcidWorks("orcid"))) .select( col("identifier.schema").alias("schema"), col("identifier.value").alias("value"), struct(orcidAuthors("orcid").alias("orcid"), col("givenName"), col("familyName")).alias("author") ) - orcidPublication + orcidPublication.cache() } } @@ -123,10 +127,8 @@ object SparkEnrichGraphWithOrcidAuthors { val log: Logger = LoggerFactory.getLogger(SparkEnrichGraphWithOrcidAuthors.getClass) def main(args: Array[String]): Unit = { - new SparkEnrichGraphWithOrcidAuthors("/eu/dnetlib/dhp/enrich/orcid/enrich_graph_orcid_parameters.json", args, log) .initialize() .run() - } } diff --git a/dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/enrich/orcid/EnrichOrcidTest.scala b/dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/enrich/orcid/EnrichOrcidTest.scala index f58b06318..84483b1a2 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/enrich/orcid/EnrichOrcidTest.scala +++ b/dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/enrich/orcid/EnrichOrcidTest.scala @@ -1,78 +1,85 @@ package eu.dnetlib.dhp.enrich.orcid -import eu.dnetlib.dhp.schema.oaf.Publication + +import eu.dnetlib.dhp.schema.oaf.{Author, Publication} import org.apache.spark.sql.{Column, Encoder, Encoders, Row, SparkSession} import org.junit.jupiter.api.Test import org.slf4j.{Logger, LoggerFactory} import org.apache.spark.sql.functions._ -case class Pid(pidScheme: String, pidValue: String) {} - -case class AuthorPid(fullName: String, pids: List[Pid]) {} - -case class PubSummary(id: String, authorWithPids: List[AuthorPid]) - class EnrichOrcidTest { val log: Logger = LoggerFactory.getLogger(getClass) - def orcid_intersection_wrong(p: PubSummary): PubSummary = { - - if (p.authorWithPids.isEmpty) - null - else { - val incorrectAuthor = p.authorWithPids.filter(a => a.pids.filter(p => p.pidScheme != null && p.pidScheme.toLowerCase.contains("orcid")).map(p => p.pidValue.toLowerCase).distinct.size > 1) - if (incorrectAuthor.nonEmpty) { - PubSummary(p.id, incorrectAuthor) - } - else { - null - } - } - } - - def test() = { val spark = SparkSession.builder().master("local[*]").getOrCreate() - spark.sparkContext.setLogLevel("ERROR") +// spark.sparkContext.setLogLevel("ERROR") + +// new SparkEnrichGraphWithOrcidAuthors(null, null, null) +// .enrichResult( +// spark, +// "/Users/sandro/orcid_test/publication", +// "", +// "/tmp/graph/", +// Encoders.bean(classOf[Publication]) +// ) val schema = Encoders.bean(classOf[Publication]).schema +// +// val simplifyAuthor = udf((r: Seq[Row]) => { +// r +// .map(k => +// AuthorPid( +// k.getAs[String]("fullname"), +// k.getAs[Seq[Row]]("pid") +// .map(p => Pid(p.getAs[Row]("qualifier").getAs[String]("classid"), p.getAs[String]("value"))) +// .toList +// ) +// ) +// .filter(l => l.pids.nonEmpty) +// .toList +// }) +// +// val wrong_orcid_intersection = udf((a: Seq[Row]) => { +// a.map(author => { +// val pids_with_orcid: Seq[Row] = author +// .getAs[Seq[Row]]("pids") +// .filter(p => +// p.getAs[String]("pidScheme") != null && p.getAs[String]("pidScheme").toLowerCase.contains("orcid") +// ) +// if (pids_with_orcid.exists(p => p.getAs[String]("pidScheme").equals("ORCID"))) { +// if (pids_with_orcid.map(p => p.getAs[String]("pidValue").toLowerCase).distinct.size > 1) { +// AuthorPid( +// author.getAs[String]("fullName"), +// pids_with_orcid.map(p => Pid(p.getAs[String]("pidScheme"), p.getAs[String]("pidValue"))).toList +// ) +// +// } else +// null +// } else +// null +// }).filter(author => author != null) +// }) + + + Encoders + import spark.implicits._ + +// val enriched = spark.read +// .schema(schema) +// .json("/Users/sandro/orcid_test/publication_enriched") +// .select(col("id"), explode(col("author")).as("authors")) +// .withColumn("ap", col("authors.pid.qualifier.classid")) +// .withColumn("dp", col("authors.pid.datainfo.provenanceAction.classid")) +// +// .show() - val simplifyAuthor = udf((r: Seq[Row]) => { - r - .map(k => - AuthorPid(k.getAs[String]("fullname"), - k.getAs[Seq[Row]]("pid") - .map( - p => Pid(p.getAs[Row]("qualifier").getAs[String]("classid"), p.getAs[String]("value")) - ).toList) - ).filter(l => l.pids.nonEmpty) - .toList - } - ) - val wrong_orcid_intersection = udf((a: Seq[Row]) => { - a.map(author => { - val pids_with_orcid: Seq[Row] = author.getAs[Seq[Row]]("pids").filter(p => p.getAs[String]("pidScheme")!= null && p.getAs[String]("pidScheme").toLowerCase.contains("orcid")) - if (pids_with_orcid.exists(p => p.getAs[String]("pidScheme").equals("ORCID"))) { - if (pids_with_orcid.map(p => p.getAs[String]("pidValue").toLowerCase).distinct.size > 1) { - AuthorPid(author.getAs[String]("fullName"),pids_with_orcid.map(p => Pid(p.getAs[String]("pidScheme"),p.getAs[String]("pidValue"))).toList ) - } - else - null - } else - null - }).filter(author => author != null) - }) - val enriched = spark.read.schema(schema).json("/Users/sandro/orcid_test/publication_enriched").select(col("id"), simplifyAuthor(col("author")).alias("authors")) - .select(col("id"), wrong_orcid_intersection(col("authors")).alias("wi")).where("wi is not null") - enriched.show(20, 1000, true) } - } From f718caaac974ed8e931053c757aaef8d9945c38c Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Thu, 30 Nov 2023 14:51:00 +0100 Subject: [PATCH 118/148] Added copy of the untouched entities of the graph --- .../dhp/enrich/orcid/oozie_app/workflow.xml | 46 +++++++++++++++++-- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/enrich/orcid/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/enrich/orcid/oozie_app/workflow.xml index 87c4dcb4f..a4330b966 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/enrich/orcid/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/enrich/orcid/oozie_app/workflow.xml @@ -1,4 +1,4 @@ - + orcidPath @@ -38,15 +38,51 @@ --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --orcidPath${orcidPath} - --targetPath${targetPath} - --graphPath${graphPath} - --masteryarn + --orcidPath + ${orcidPath} + --targetPath + ${targetPath} + --graphPath + ${graphPath} + --master + yarn + + + ${nameNode}/${graphPath}/datasource + ${nameNode}/${targetPath}/datasource + + + + + + + ${nameNode}/${graphPath}/organization + ${nameNode}/${targetPath}/organization + + + + + + + ${nameNode}/${graphPath}/project + ${nameNode}/${targetPath}/project + + + + + + + ${nameNode}/${graphPath}/relation + ${nameNode}/${targetPath}/relation + + + + \ No newline at end of file From cdfb7588dd4b23fc83e764ae490dd051c8e1576f Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Thu, 30 Nov 2023 15:31:42 +0100 Subject: [PATCH 119/148] code formatting --- .../eu/dnetlib/dhp/oa/merge/AuthorMerger.java | 551 +++++++++--------- .../PrepareAffiliationRelationsTest.java | 4 +- .../doiboost/SparkGenerateDoiBoost.scala | 1 - .../SparkEnrichGraphWithOrcidAuthors.scala | 2 +- .../dhp/enrich/orcid/EnrichOrcidTest.scala | 10 +- 5 files changed, 268 insertions(+), 300 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java index 62e6764b7..d7224af78 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java @@ -4,352 +4,329 @@ package eu.dnetlib.dhp.oa.merge; import java.text.Normalizer; import java.util.*; import java.util.stream.Collectors; - import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.NotNull; - import com.wcohen.ss.JaroWinkler; - import eu.dnetlib.dhp.schema.oaf.Author; import eu.dnetlib.dhp.schema.oaf.StructuredProperty; import eu.dnetlib.pace.model.Person; import scala.Tuple2; -class SimilarityCellInfo implements Comparable { - - public int authorPosition = 0; - public int orcidPosition = 0; - - public double maxColumnSimilarity = 0.0; - - public SimilarityCellInfo() { - } - - public void setValues(final int authPos, final int orcidPos, final double similarity) { - this.authorPosition = authPos; - this.orcidPosition = orcidPos; - this.maxColumnSimilarity = similarity; - } - - @Override - public int compareTo(@NotNull SimilarityCellInfo o) { - return Double.compare(maxColumnSimilarity, o.maxColumnSimilarity); - } -} public class AuthorMerger { - private static final Double THRESHOLD = 0.95; + private static final Double THRESHOLD = 0.95; - private AuthorMerger() { - } + private AuthorMerger() { + } - public static List merge(List> authors) { + public static List merge(List> authors) { - authors.sort((o1, o2) -> -Integer.compare(countAuthorsPids(o1), countAuthorsPids(o2))); + authors.sort((o1, o2) -> -Integer.compare(countAuthorsPids(o1), countAuthorsPids(o2))); - List author = new ArrayList<>(); + List author = new ArrayList<>(); - for (List a : authors) { - author = mergeAuthor(author, a); - } + for (List a : authors) { + author = mergeAuthor(author, a); + } - return author; + return author; - } + } - public static List mergeAuthor(final List a, final List b, Double threshold) { - int pa = countAuthorsPids(a); - int pb = countAuthorsPids(b); - List base; - List enrich; - int sa = authorsSize(a); - int sb = authorsSize(b); + public static List mergeAuthor(final List a, final List b, Double threshold) { + int pa = countAuthorsPids(a); + int pb = countAuthorsPids(b); + List base; + List enrich; + int sa = authorsSize(a); + int sb = authorsSize(b); - if (sa == sb) { - base = pa > pb ? a : b; - enrich = pa > pb ? b : a; - } else { - base = sa > sb ? a : b; - enrich = sa > sb ? b : a; - } - enrichPidFromList(base, enrich, threshold); - return base; - } + if (sa == sb) { + base = pa > pb ? a : b; + enrich = pa > pb ? b : a; + } else { + base = sa > sb ? a : b; + enrich = sa > sb ? b : a; + } + enrichPidFromList(base, enrich, threshold); + return base; + } - public static List mergeAuthor(final List a, final List b) { - return mergeAuthor(a, b, THRESHOLD); - } + public static List mergeAuthor(final List a, final List b) { + return mergeAuthor(a, b, THRESHOLD); + } - private static void enrichPidFromList(List base, List enrich, Double threshold) { - if (base == null || enrich == null) - return; + private static void enrichPidFromList(List base, List enrich, Double threshold) { + if (base == null || enrich == null) + return; - // (if an Author has more than 1 pid, it appears 2 times in the list) - final Map basePidAuthorMap = base - .stream() - .filter(a -> a.getPid() != null && !a.getPid().isEmpty()) - .flatMap( - a -> a - .getPid() - .stream() - .filter(Objects::nonNull) - .map(p -> new Tuple2<>(pidToComparableString(p), a))) - .collect(Collectors.toMap(Tuple2::_1, Tuple2::_2, (x1, x2) -> x1)); + // (if an Author has more than 1 pid, it appears 2 times in the list) + final Map basePidAuthorMap = base + .stream() + .filter(a -> a.getPid() != null && !a.getPid().isEmpty()) + .flatMap( + a -> a + .getPid() + .stream() + .filter(Objects::nonNull) + .map(p -> new Tuple2<>(pidToComparableString(p), a))) + .collect(Collectors.toMap(Tuple2::_1, Tuple2::_2, (x1, x2) -> x1)); - // (list of pid that are missing in the other list) - final List> pidToEnrich = enrich - .stream() - .filter(a -> a.getPid() != null && !a.getPid().isEmpty()) - .flatMap( - a -> a - .getPid() - .stream() - .filter(Objects::nonNull) - .filter(p -> !basePidAuthorMap.containsKey(pidToComparableString(p))) - .map(p -> new Tuple2<>(p, a))) - .collect(Collectors.toList()); + // (list of pid that are missing in the other list) + final List> pidToEnrich = enrich + .stream() + .filter(a -> a.getPid() != null && !a.getPid().isEmpty()) + .flatMap( + a -> a + .getPid() + .stream() + .filter(Objects::nonNull) + .filter(p -> !basePidAuthorMap.containsKey(pidToComparableString(p))) + .map(p -> new Tuple2<>(p, a))) + .collect(Collectors.toList()); - pidToEnrich - .forEach( - a -> { - Optional> simAuthor = base - .stream() - .map(ba -> new Tuple2<>(sim(ba, a._2()), ba)) - .max(Comparator.comparing(Tuple2::_1)); + pidToEnrich + .forEach( + a -> { + Optional> simAuthor = base + .stream() + .map(ba -> new Tuple2<>(sim(ba, a._2()), ba)) + .max(Comparator.comparing(Tuple2::_1)); - if (simAuthor.isPresent()) { - double th = threshold; - // increase the threshold if the surname is too short - if (simAuthor.get()._2().getSurname() != null - && simAuthor.get()._2().getSurname().length() <= 3 && threshold > 0.0) - th = 0.99; + if (simAuthor.isPresent()) { + double th = threshold; + // increase the threshold if the surname is too short + if (simAuthor.get()._2().getSurname() != null + && simAuthor.get()._2().getSurname().length() <= 3 && threshold > 0.0) + th = 0.99; - if (simAuthor.get()._1() > th) { - Author r = simAuthor.get()._2(); - if (r.getPid() == null) { - r.setPid(new ArrayList<>()); - } + if (simAuthor.get()._1() > th) { + Author r = simAuthor.get()._2(); + if (r.getPid() == null) { + r.setPid(new ArrayList<>()); + } - // TERRIBLE HACK but for some reason when we create and Array with Arrays.asList, - // it creates of fixed size, and the add method raise UnsupportedOperationException at - // java.util.AbstractList.add - final List tmp = new ArrayList<>(r.getPid()); - tmp.add(a._1()); - r.setPid(tmp); - } - } - }); - } + // TERRIBLE HACK but for some reason when we create and Array with Arrays.asList, + // it creates of fixed size, and the add method raise UnsupportedOperationException at + // java.util.AbstractList.add + final List tmp = new ArrayList<>(r.getPid()); + tmp.add(a._1()); + r.setPid(tmp); + } + } + }); + } - public static String normalizeFullName(final String fullname) { - return nfd(fullname) - .toLowerCase() - // do not compact the regexes in a single expression, would cause StackOverflowError - // in case - // of large input strings - .replaceAll("(\\W)+", " ") - .replaceAll("(\\p{InCombiningDiacriticalMarks})+", " ") - .replaceAll("(\\p{Punct})+", " ") - .replaceAll("(\\d)+", " ") - .replaceAll("(\\n)+", " ") + public static String normalizeFullName(final String fullname) { + return nfd(fullname) + .toLowerCase() + // do not compact the regexes in a single expression, would cause StackOverflowError + // in case + // of large input strings + .replaceAll("(\\W)+", " ") + .replaceAll("(\\p{InCombiningDiacriticalMarks})+", " ") + .replaceAll("(\\p{Punct})+", " ") + .replaceAll("(\\d)+", " ") + .replaceAll("(\\n)+", " ") - .trim(); - } + .trim(); + } - static int hammingDist(String str1, String str2) { - if (str1.length() != str2.length()) - return Math.max(str1.length(), str2.length()); - int i = 0, count = 0; - while (i < str1.length()) { - if (str1.charAt(i) != str2.charAt(i)) - count++; - i++; - } - return count; - } + static int hammingDist(String str1, String str2) { + if (str1.length() != str2.length()) + return Math.max(str1.length(), str2.length()); + int i = 0, count = 0; + while (i < str1.length()) { + if (str1.charAt(i) != str2.charAt(i)) + count++; + i++; + } + return count; + } - private static String authorFieldToBeCompared(Author author) { - if (StringUtils.isNotBlank(author.getSurname())) { - return author.getSurname(); + private static String authorFieldToBeCompared(Author author) { + if (StringUtils.isNotBlank(author.getSurname())) { + return author.getSurname(); - } - if (StringUtils.isNotBlank(author.getFullname())) { - return author.getFullname(); - } - return null; - } + } + if (StringUtils.isNotBlank(author.getFullname())) { + return author.getFullname(); + } + return null; + } - /** - * This method tries to figure out when two author are the same in the contest - * of ORCID enrichment - * @param left Author in the OAF entity - * @param right Author ORCID - * @return based on a heuristic on the names of the authors if they are the same. - */ - public static boolean checkORCIDSimilarity(final Author left, final Author right) { - final Person pl = parse(left); - final Person pr = parse(right); + /** + * This method tries to figure out when two author are the same in the contest + * of ORCID enrichment + * + * @param left Author in the OAF entity + * @param right Author ORCID + * @return based on a heuristic on the names of the authors if they are the same. + */ + public static boolean checkORCIDSimilarity(final Author left, final Author right) { + final Person pl = parse(left); + final Person pr = parse(right); - // If one of them didn't have a surname we verify if they have the fullName not empty - // and verify if the normalized version is equal - if (!(pl.getSurname() != null && pl.getSurname().stream().anyMatch(StringUtils::isNotBlank) && - pr.getSurname() != null && pr.getSurname().stream().anyMatch(StringUtils::isNotBlank))) { + // If one of them didn't have a surname we verify if they have the fullName not empty + // and verify if the normalized version is equal + if (!(pl.getSurname() != null && pl.getSurname().stream().anyMatch(StringUtils::isNotBlank) && + pr.getSurname() != null && pr.getSurname().stream().anyMatch(StringUtils::isNotBlank))) { - if (pl.getFullname() != null && !pl.getFullname().isEmpty() && pr.getFullname() != null - && !pr.getFullname().isEmpty()) { - return pl - .getFullname() - .stream() - .anyMatch( - fl -> pr.getFullname().stream().anyMatch(fr -> normalize(fl).equalsIgnoreCase(normalize(fr)))); - } else { - return false; - } - } - // The Authors have one surname in common - if (pl.getSurname().stream().anyMatch(sl -> pr.getSurname().stream().anyMatch(sr -> sr.equalsIgnoreCase(sl)))) { + if (pl.getFullname() != null && !pl.getFullname().isEmpty() && pr.getFullname() != null + && !pr.getFullname().isEmpty()) { + return pl + .getFullname() + .stream() + .anyMatch( + fl -> pr.getFullname().stream().anyMatch(fr -> normalize(fl).equalsIgnoreCase(normalize(fr)))); + } else { + return false; + } + } + // The Authors have one surname in common + if (pl.getSurname().stream().anyMatch(sl -> pr.getSurname().stream().anyMatch(sr -> sr.equalsIgnoreCase(sl)))) { - // If one of them has only a surname and is the same we can say that they are the same author - if ((pl.getName() == null || pl.getName().stream().allMatch(StringUtils::isBlank)) || - (pr.getName() == null || pr.getName().stream().allMatch(StringUtils::isBlank))) - return true; - // The authors have the same initials of Name in common - if (pl - .getName() - .stream() - .anyMatch( - nl -> pr - .getName() - .stream() - .anyMatch(nr -> nr.equalsIgnoreCase(nl)))) - return true; - } + // If one of them has only a surname and is the same we can say that they are the same author + if ((pl.getName() == null || pl.getName().stream().allMatch(StringUtils::isBlank)) || + (pr.getName() == null || pr.getName().stream().allMatch(StringUtils::isBlank))) + return true; + // The authors have the same initials of Name in common + if (pl + .getName() + .stream() + .anyMatch( + nl -> pr + .getName() + .stream() + .anyMatch(nr -> nr.equalsIgnoreCase(nl)))) + return true; + } - // Sometimes we noticed that publication have author wrote in inverse order Surname, Name - // We verify if we have an exact match between name and surname - if (pl.getSurname().stream().anyMatch(sl -> pr.getName().stream().anyMatch(nr -> nr.equalsIgnoreCase(sl))) && - pl.getName().stream().anyMatch(nl -> pr.getSurname().stream().anyMatch(sr -> sr.equalsIgnoreCase(nl)))) - return true; - else - return false; - } - // + // Sometimes we noticed that publication have author wrote in inverse order Surname, Name + // We verify if we have an exact match between name and surname + if (pl.getSurname().stream().anyMatch(sl -> pr.getName().stream().anyMatch(nr -> nr.equalsIgnoreCase(sl))) && + pl.getName().stream().anyMatch(nl -> pr.getSurname().stream().anyMatch(sr -> sr.equalsIgnoreCase(nl)))) + return true; + else + return false; + } + // - /** - * Method to enrich ORCID information in one list of authors based on another list - * @param baseAuthor the Author List in the OAF Entity - * @param orcidAuthor The list of ORCID Author intersected - * @return The Author List of the OAF Entity enriched with the orcid Author - */ - public static List enrichOrcid(List baseAuthor, List orcidAuthor) { + /** + * Method to enrich ORCID information in one list of authors based on another list + * + * @param baseAuthor the Author List in the OAF Entity + * @param orcidAuthor The list of ORCID Author intersected + * @return The Author List of the OAF Entity enriched with the orcid Author + */ + public static List enrichOrcid(List baseAuthor, List orcidAuthor) { - if (baseAuthor == null || baseAuthor.isEmpty()) - return orcidAuthor; + if (baseAuthor == null || baseAuthor.isEmpty()) + return orcidAuthor; - if (orcidAuthor == null || orcidAuthor.isEmpty()) - return baseAuthor; + if (orcidAuthor == null || orcidAuthor.isEmpty()) + return baseAuthor; - if (baseAuthor.size() == 1 && orcidAuthor.size() > 10) - return baseAuthor; + if (baseAuthor.size() == 1 && orcidAuthor.size() > 10) + return baseAuthor; - final List oAuthor = new ArrayList<>(); - oAuthor.addAll(orcidAuthor); + final List oAuthor = new ArrayList<>(); + oAuthor.addAll(orcidAuthor); - baseAuthor.forEach(ba -> { - Optional aMatch = oAuthor.stream().filter(oa -> checkORCIDSimilarity(ba, oa)).findFirst(); - if (aMatch.isPresent()) { - final Author sameAuthor = aMatch.get(); - addPid(ba, sameAuthor.getPid()); - oAuthor.remove(sameAuthor); - } - }); - return baseAuthor; - } + baseAuthor.forEach(ba -> { + Optional aMatch = oAuthor.stream().filter(oa -> checkORCIDSimilarity(ba, oa)).findFirst(); + if (aMatch.isPresent()) { + final Author sameAuthor = aMatch.get(); + addPid(ba, sameAuthor.getPid()); + oAuthor.remove(sameAuthor); + } + }); + return baseAuthor; + } - private static void addPid(final Author a, final List pids) { + private static void addPid(final Author a, final List pids) { - if (a.getPid() == null) { - a.setPid(new ArrayList<>()); - } + if (a.getPid() == null) { + a.setPid(new ArrayList<>()); + } - a.getPid().addAll(pids); + a.getPid().addAll(pids); - } + } - public static String pidToComparableString(StructuredProperty pid) { - final String classid = pid.getQualifier().getClassid() != null ? pid.getQualifier().getClassid().toLowerCase() - : ""; - return (pid.getQualifier() != null ? classid : "") - + (pid.getValue() != null ? pid.getValue().toLowerCase() : ""); - } + public static String pidToComparableString(StructuredProperty pid) { + final String classid = pid.getQualifier().getClassid() != null ? pid.getQualifier().getClassid().toLowerCase() + : ""; + return (pid.getQualifier() != null ? classid : "") + + (pid.getValue() != null ? pid.getValue().toLowerCase() : ""); + } - public static int countAuthorsPids(List authors) { - if (authors == null) - return 0; + public static int countAuthorsPids(List authors) { + if (authors == null) + return 0; - return (int) authors.stream().filter(AuthorMerger::hasPid).count(); - } + return (int) authors.stream().filter(AuthorMerger::hasPid).count(); + } - private static int authorsSize(List authors) { - if (authors == null) - return 0; - return authors.size(); - } + private static int authorsSize(List authors) { + if (authors == null) + return 0; + return authors.size(); + } - private static Double sim(Author a, Author b) { + private static Double sim(Author a, Author b) { - final Person pa = parse(a); - final Person pb = parse(b); + final Person pa = parse(a); + final Person pb = parse(b); - // if both are accurate (e.g. they have name and surname) - if (pa.isAccurate() & pb.isAccurate()) { - return new JaroWinkler().score(normalize(pa.getSurnameString()), normalize(pb.getSurnameString())) * 0.5 - + new JaroWinkler().score(normalize(pa.getNameString()), normalize(pb.getNameString())) * 0.5; - } else { - return new JaroWinkler() - .score(normalize(pa.getNormalisedFullname()), normalize(pb.getNormalisedFullname())); - } - } + // if both are accurate (e.g. they have name and surname) + if (pa.isAccurate() & pb.isAccurate()) { + return new JaroWinkler().score(normalize(pa.getSurnameString()), normalize(pb.getSurnameString())) * 0.5 + + new JaroWinkler().score(normalize(pa.getNameString()), normalize(pb.getNameString())) * 0.5; + } else { + return new JaroWinkler() + .score(normalize(pa.getNormalisedFullname()), normalize(pb.getNormalisedFullname())); + } + } - private static boolean hasPid(Author a) { - if (a == null || a.getPid() == null || a.getPid().isEmpty()) - return false; - return a.getPid().stream().anyMatch(p -> p != null && StringUtils.isNotBlank(p.getValue())); - } + private static boolean hasPid(Author a) { + if (a == null || a.getPid() == null || a.getPid().isEmpty()) + return false; + return a.getPid().stream().anyMatch(p -> p != null && StringUtils.isNotBlank(p.getValue())); + } - private static Person parse(Author author) { - if (StringUtils.isNotBlank(author.getSurname())) { - return new Person(author.getSurname() + ", " + author.getName(), false); - } else { - if (StringUtils.isNotBlank(author.getFullname())) - return new Person(author.getFullname(), false); - else - return new Person("", false); - } - } + private static Person parse(Author author) { + if (StringUtils.isNotBlank(author.getSurname())) { + return new Person(author.getSurname() + ", " + author.getName(), false); + } else { + if (StringUtils.isNotBlank(author.getFullname())) + return new Person(author.getFullname(), false); + else + return new Person("", false); + } + } - public static String normalize(final String s) { - String[] normalized = nfd(s) - .toLowerCase() - // do not compact the regexes in a single expression, would cause StackOverflowError - // in case - // of large input strings - .replaceAll("(\\W)+", " ") - .replaceAll("(\\p{InCombiningDiacriticalMarks})+", " ") - .replaceAll("(\\p{Punct})+", " ") - .replaceAll("(\\d)+", " ") - .replaceAll("(\\n)+", " ") - .trim() - .split(" "); + public static String normalize(final String s) { + String[] normalized = nfd(s) + .toLowerCase() + // do not compact the regexes in a single expression, would cause StackOverflowError + // in case + // of large input strings + .replaceAll("(\\W)+", " ") + .replaceAll("(\\p{InCombiningDiacriticalMarks})+", " ") + .replaceAll("(\\p{Punct})+", " ") + .replaceAll("(\\d)+", " ") + .replaceAll("(\\n)+", " ") + .trim() + .split(" "); - Arrays.sort(normalized); + Arrays.sort(normalized); - return String.join(" ", normalized); - } + return String.join(" ", normalized); + } - private static String nfd(final String s) { - return Normalizer.normalize(s, Normalizer.Form.NFD); - } + private static String nfd(final String s) { + return Normalizer.normalize(s, Normalizer.Form.NFD); + } } diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelationsTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelationsTest.java index e2639996c..b87738879 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelationsTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/bipaffiliations/PrepareAffiliationRelationsTest.java @@ -79,8 +79,8 @@ public class PrepareAffiliationRelationsTest { .getPath(); String pubmedAffiliationRelationsPath = getClass() - .getResource("/eu/dnetlib/dhp/actionmanager/bipaffiliations/doi_to_ror.json") - .getPath(); + .getResource("/eu/dnetlib/dhp/actionmanager/bipaffiliations/doi_to_ror.json") + .getPath(); String outputPath = workingDir.toString() + "/actionSet"; diff --git a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/SparkGenerateDoiBoost.scala b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/SparkGenerateDoiBoost.scala index 07d6a0287..9ffaeeeef 100644 --- a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/SparkGenerateDoiBoost.scala +++ b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/SparkGenerateDoiBoost.scala @@ -92,7 +92,6 @@ object SparkGenerateDoiBoost { .mode(SaveMode.Overwrite) .save(s"$workingDirPath/firstJoin") - logger.info("Phase 2) Join Result with MAG") val sj: Dataset[(String, Publication)] = spark.read.load(s"$workingDirPath/firstJoin").as[Publication].map(p => (p.getId, p)) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/SparkEnrichGraphWithOrcidAuthors.scala b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/SparkEnrichGraphWithOrcidAuthors.scala index 0d994d202..645df8181 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/SparkEnrichGraphWithOrcidAuthors.scala +++ b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/SparkEnrichGraphWithOrcidAuthors.scala @@ -73,7 +73,7 @@ class SparkEnrichGraphWithOrcidAuthors(propertyPath: String, args: Array[String] col("id").alias("dnet_id") ) - val orcidDnet = orcidPublication + val orcidDnet = orcidPublication .join( entities, lower(col("schema")).equalTo(lower(col("pid_schema"))) && diff --git a/dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/enrich/orcid/EnrichOrcidTest.scala b/dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/enrich/orcid/EnrichOrcidTest.scala index 84483b1a2..cb05332b2 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/enrich/orcid/EnrichOrcidTest.scala +++ b/dhp-workflows/dhp-graph-mapper/src/test/scala/eu/dnetlib/dhp/enrich/orcid/EnrichOrcidTest.scala @@ -6,13 +6,10 @@ import org.junit.jupiter.api.Test import org.slf4j.{Logger, LoggerFactory} import org.apache.spark.sql.functions._ - class EnrichOrcidTest { val log: Logger = LoggerFactory.getLogger(getClass) - - def test() = { val spark = SparkSession.builder().master("local[*]").getOrCreate() // spark.sparkContext.setLogLevel("ERROR") @@ -63,8 +60,7 @@ class EnrichOrcidTest { // }).filter(author => author != null) // }) - - Encoders + Encoders import spark.implicits._ // val enriched = spark.read @@ -76,10 +72,6 @@ class EnrichOrcidTest { // // .show() - - - - } } From 48430a32a63cc80b32bce14104b9f11037960adf Mon Sep 17 00:00:00 2001 From: dimitrispie Date: Fri, 1 Dec 2023 11:35:01 +0200 Subject: [PATCH 120/148] Update StatsAtomicActionsJob.java Added indi_funded_result_with_fundref indicator --- .../StatsAtomicActionsJob.java | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsAtomicActionsJob.java b/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsAtomicActionsJob.java index a64a63c24..0c3c2f3c0 100644 --- a/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsAtomicActionsJob.java +++ b/dhp-workflows/dhp-stats-actionsets/src/main/java/eu/dnetlib/dhp/actionmanager/stats_actionsets/StatsAtomicActionsJob.java @@ -96,14 +96,16 @@ public class StatsAtomicActionsJob implements Serializable { .sql( String .format( - "select b.%s as id, is_gold, is_bronze_oa, is_hybrid,green_oa, in_diamond_journal,f.fundref as publicly_funded " - + "from %s.indi_pub_bronze_oa b " + - "left outer join %s.indi_pub_gold_oa g on g.id=b.id " + - "left outer join %s.indi_pub_hybrid h on b.id=h.id " + - "left outer join %s.indi_pub_green_oa gr on b.id=gr.id " + + "select r.%s as id, is_gold, is_bronze_oa, is_hybrid,green_oa, in_diamond_journal,f.publicly_funded as publicly_funded " + + + "from %s.publication r " + + "left outer join %s.indi_pub_bronze_oa b on r.id=b.id " + + "left outer join %s.indi_pub_gold_oa g on r.id=g.id " + + "left outer join %s.indi_pub_hybrid h on r.id=h.id " + + "left outer join %s.indi_pub_green_oa gr on r.id=gr.id " + "left outer join %s.indi_pub_diamond d on b.id=d.id " + - "left outer join %s.indi_funded_result_with_fundref f on b.id=f.id ", - resultAttributeName, dbname, dbname, dbname, dbname, dbname, dbname)) + "left outer join %s.indi_pub_publicly_funded f on r.id=f.id ", + resultAttributeName, dbname, dbname, dbname, dbname, dbname, dbname, dbname)) .as(Encoders.bean(StatsResultEnhancementModel.class)) .write() .mode(SaveMode.Overwrite) @@ -136,12 +138,12 @@ public class StatsAtomicActionsJob implements Serializable { r.setIsInDiamondJournal(usm.isIn_diamond_journal()); r.setIsGreen(usm.isGreen_oa()); r.setPubliclyFunded(usm.isPublicly_funded()); - if (usm.isIs_bronze_oa()) - r.setOpenAccessColor(OpenAccessColor.bronze); - else if (usm.isIs_gold()) + if (usm.isIs_gold()) r.setOpenAccessColor(OpenAccessColor.gold); else if (usm.isIs_hybrid()) r.setOpenAccessColor(OpenAccessColor.hybrid); + else if (usm.isIs_bronze_oa()) + r.setOpenAccessColor(OpenAccessColor.bronze); return r; }, Encoders.bean(Result.class)); } From bf0fd27c362bc07a0fe5579ac7787dfe0134cb30 Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Fri, 1 Dec 2023 12:16:42 +0100 Subject: [PATCH 121/148] Removed unused function Applied PR Comment of Giambattista in the PR --- .../java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java | 11 ----------- .../orcid/SparkEnrichGraphWithOrcidAuthors.scala | 8 ++++++-- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java index d7224af78..0153e52d5 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java @@ -132,17 +132,6 @@ public class AuthorMerger { .trim(); } - static int hammingDist(String str1, String str2) { - if (str1.length() != str2.length()) - return Math.max(str1.length(), str2.length()); - int i = 0, count = 0; - while (i < str1.length()) { - if (str1.charAt(i) != str2.charAt(i)) - count++; - i++; - } - return count; - } private static String authorFieldToBeCompared(Author author) { if (StringUtils.isNotBlank(author.getSurname())) { diff --git a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/SparkEnrichGraphWithOrcidAuthors.scala b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/SparkEnrichGraphWithOrcidAuthors.scala index 645df8181..4822059c6 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/SparkEnrichGraphWithOrcidAuthors.scala +++ b/dhp-workflows/dhp-graph-mapper/src/main/scala/eu/dnetlib/dhp/enrich/orcid/SparkEnrichGraphWithOrcidAuthors.scala @@ -2,10 +2,12 @@ package eu.dnetlib.dhp.enrich.orcid import eu.dnetlib.dhp.application.AbstractScalaApplication import eu.dnetlib.dhp.oa.merge.AuthorMerger +import eu.dnetlib.dhp.schema.common.ModelSupport import eu.dnetlib.dhp.schema.oaf.{OtherResearchProduct, Publication, Result, Software} import org.apache.spark.sql.functions._ import org.apache.spark.sql._ import org.slf4j.{Logger, LoggerFactory} +import scala.collection.JavaConverters._ class SparkEnrichGraphWithOrcidAuthors(propertyPath: String, args: Array[String], log: Logger) extends AbstractScalaApplication(propertyPath, args, log: Logger) { @@ -21,6 +23,8 @@ class SparkEnrichGraphWithOrcidAuthors(propertyPath: String, args: Array[String] val targetPath = parser.get("targetPath") log.info(s"targetPath is '$targetPath'") val orcidPublication: Dataset[Row] = generateOrcidTable(spark, orcidPath) +// ModelSupport.entityTypes.entrySet().asScala.filter(k => k.getKey.getClass isInstance(Result)) + enrichResult( spark, s"$graphPath/publication", @@ -63,7 +67,7 @@ class SparkEnrichGraphWithOrcidAuthors(propertyPath: String, args: Array[String] .schema(enc.schema) .json(graphPath) .select(col("id"), col("datainfo"), col("instance")) - .where("datainfo.deletedbyinference = false") + .where("datainfo.deletedbyinference != true") .drop("datainfo") .withColumn("instances", explode(col("instance"))) .withColumn("pids", explode(col("instances.pid"))) @@ -109,7 +113,7 @@ class SparkEnrichGraphWithOrcidAuthors(propertyPath: String, args: Array[String] .load(s"$inputPath/Works") .select(col("orcid"), explode(col("pids")).alias("identifier")) .where( - "identifier.schema = 'doi' or identifier.schema ='pmid' or identifier.schema ='pmc' or identifier.schema ='arxiv' or identifier.schema ='handle'" + "identifier.schema IN('doi','pmid','pmc','arxiv','handle')" ) val orcidPublication = orcidAuthors .join(orcidWorks, orcidAuthors("orcid").equalTo(orcidWorks("orcid"))) From 76594ded23455ecf67addcbebd3b4ca45b73199e Mon Sep 17 00:00:00 2001 From: dimitrispie Date: Fri, 1 Dec 2023 13:38:19 +0200 Subject: [PATCH 122/148] Changes to indicators Fixes on open access colours indicators - indi_pub_green_oa - indi_pub_gold_oa - indi_pub_hybrid - indi_pub_bronze_oa - indi_pub_diamond --- .../scripts/step16-createIndicatorsTables.sql | 129 +++++++++++++----- .../scripts/step20-createMonitorDB.sql | 2 + .../scripts/step20-createMonitorDBAll.sql | 2 + .../graph/stats/oozie_app/scripts/step3.sql | 18 +++ .../graph/stats/oozie_app/scripts/step4.sql | 15 ++ .../graph/stats/oozie_app/scripts/step5.sql | 16 +++ .../graph/stats/oozie_app/scripts/step6.sql | 14 ++ 7 files changed, 162 insertions(+), 34 deletions(-) diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql index 8180e6527..fea449de6 100755 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql @@ -1,6 +1,18 @@ -- Sprint 1 ---- drop table if exists ${stats_db_name}.indi_pub_green_oa purge; +--create table if not exists ${stats_db_name}.indi_pub_green_oa stored as parquet as +--select distinct p.id, coalesce(green_oa, 0) as green_oa +--from ${stats_db_name}.publication p +-- left outer join ( +-- select p.id, 1 as green_oa +-- from ${stats_db_name}.publication p +-- join ${stats_db_name}.result_instance ri on ri.id = p.id +-- join ${stats_db_name}.datasource on datasource.id = ri.hostedby +-- where datasource.type like '%Repository%' +-- and (ri.accessright = 'Open Access' +-- or ri.accessright = 'Embargo' or ri.accessright = 'Open Source')) tmp +-- on p.id= tmp.id; create table if not exists ${stats_db_name}.indi_pub_green_oa stored as parquet as select distinct p.id, coalesce(green_oa, 0) as green_oa from ${stats_db_name}.publication p @@ -11,7 +23,7 @@ from ${stats_db_name}.publication p join ${stats_db_name}.datasource on datasource.id = ri.hostedby where datasource.type like '%Repository%' and (ri.accessright = 'Open Access' - or ri.accessright = 'Embargo' or ri.accessright = 'Open Source')) tmp + or ri.accessright = 'Embargo' or ri.accessright = 'Open Source') and datasource.name!='Other') tmp on p.id= tmp.id; drop table if exists ${stats_db_name}.indi_pub_grey_lit purge; @@ -183,15 +195,24 @@ drop table if exists ${stats_db_name}.tmp purge; ---- Sprint 4 ---- drop table if exists ${stats_db_name}.indi_pub_diamond purge; +--create table if not exists ${stats_db_name}.indi_pub_diamond stored as parquet as +--select distinct pd.id, coalesce(in_diamond_journal, 0) as in_diamond_journal +--from ${stats_db_name}.publication_datasources pd +-- left outer join ( +-- select pd.id, 1 as in_diamond_journal from ${stats_db_name}.publication_datasources pd +-- join ${stats_db_name}.datasource d on d.id=pd.datasource +-- join STATS_EXT.plan_s_jn ps where (ps.issn_print=d.issn_printed and ps.issn_online=d.issn_online) +-- and (ps.journal_is_in_doaj=true or ps.journal_is_oa=true) and ps.has_apc=false) tmp +-- on pd.id=tmp.id; + create table if not exists ${stats_db_name}.indi_pub_diamond stored as parquet as select distinct pd.id, coalesce(in_diamond_journal, 0) as in_diamond_journal from ${stats_db_name}.publication_datasources pd - left outer join ( - select pd.id, 1 as in_diamond_journal from ${stats_db_name}.publication_datasources pd - join ${stats_db_name}.datasource d on d.id=pd.datasource - join STATS_EXT.plan_s_jn ps where (ps.issn_print=d.issn_printed and ps.issn_online=d.issn_online) - and (ps.journal_is_in_doaj=true or ps.journal_is_oa=true) and ps.has_apc=false) tmp - on pd.id=tmp.id; +left outer join (select pd.id, 1 as in_diamond_journal from ${stats_db_name}.publication_datasources pd +join ${stats_db_name}.datasource d on d.id=pd.datasource +join STATS_EXT.plan_s_jn ps where (ps.issn_print=d.issn_printed and ps.issn_online=d.issn_online) +and (ps.journal_is_in_doaj=true or ps.journal_is_oa=true) and ps.has_apc=false) tmp +on pd.id=tmp.id; drop table if exists ${stats_db_name}.indi_pub_in_transformative purge; @@ -312,28 +333,55 @@ drop table if exists ${stats_db_name}.indi_pub_gold_oa purge; -- JOIN gold_oa on issn.issn = gold_oa.issn) tmp -- on pd.id=tmp.id; +--create table if not exists ${stats_db_name}.indi_pub_gold_oa stored as parquet as +--with gold_oa as ( +--SELECT issn,issn_l from stats_ext.issn_gold_oa_dataset_v5), +--issn AS (SELECT * FROM +--(SELECT id,issn_printed as issn FROM ${stats_db_name}.datasource +--WHERE issn_printed IS NOT NULL +--UNION ALL +--SELECT id, issn_online as issn FROM ${stats_db_name}.datasource +--WHERE issn_online IS NOT NULL or id like '%doajarticles%') as issn +--WHERE LENGTH(issn) > 7), +--alljournals AS(select issn, issn_l from stats_ext.alljournals +--where journal_is_in_doaj=true or journal_is_oa=true) +--SELECT DISTINCT pd.id, coalesce(is_gold, 0) as is_gold +--FROM ${stats_db_name}.publication_datasources pd +--left outer join ( +--select pd.id, 1 as is_gold FROM ${stats_db_name}.publication_datasources pd +--JOIN issn on issn.id=pd.datasource +--JOIN gold_oa on issn.issn = gold_oa.issn +--join alljournals on issn.issn=alljournals.issn +--left outer join ${stats_db_name}.result_instance ri on ri.id=pd.id +--and ri.accessright!='Closed Access' and ri.accessright_uw='gold') tmp +--on pd.id=tmp.id; create table if not exists ${stats_db_name}.indi_pub_gold_oa stored as parquet as with gold_oa as ( -SELECT issn,issn_l from stats_ext.issn_gold_oa_dataset_v5), -issn AS (SELECT * FROM -(SELECT id,issn_printed as issn FROM ${stats_db_name}.datasource -WHERE issn_printed IS NOT NULL -UNION ALL -SELECT id, issn_online as issn FROM ${stats_db_name}.datasource -WHERE issn_online IS NOT NULL or id like '%doajarticles%') as issn -WHERE LENGTH(issn) > 7), -alljournals AS(select issn, issn_l from stats_ext.alljournals -where journal_is_in_doaj=true or journal_is_oa=true) +select distinct issn from ( + SELECT issn_l as issn from stats_ext.issn_gold_oa_dataset_v5 + UNION ALL + SELECT issn as issn from stats_ext.issn_gold_oa_dataset_v5 + UNION ALL + select issn from stats_ext.alljournals where journal_is_in_doaj=true or journal_is_oa=true + UNION ALL + select issn_l as issn from stats_ext.alljournals where journal_is_in_doaj=true or journal_is_oa=true) foo), +dd as ( +select distinct * from ( + select id, issn_printed as issn from ${stats_db_name}.datasource d where d.id like '%doajarticles%' + UNION ALL + select id, issn_online as issn from ${stats_db_name}.datasource d where d.id like '%doajarticles%' + UNION ALL + select id, issn_printed as issn from ${stats_db_name}.datasource d join gold_oa on gold_oa.issn=d.issn_printed + UNION ALL + select id, issn_online as issn from ${stats_db_name}.datasource d join gold_oa on gold_oa.issn=d.issn_online) foo +) SELECT DISTINCT pd.id, coalesce(is_gold, 0) as is_gold FROM ${stats_db_name}.publication_datasources pd left outer join ( -select pd.id, 1 as is_gold FROM ${stats_db_name}.publication_datasources pd -JOIN issn on issn.id=pd.datasource -JOIN gold_oa on issn.issn = gold_oa.issn -join alljournals on issn.issn=alljournals.issn -left outer join ${stats_db_name}.result_instance ri on ri.id=pd.id -and ri.accessright!='Closed Access' and ri.accessright_uw='gold') tmp -on pd.id=tmp.id; + select pd.id, 1 as is_gold + FROM ${stats_db_name}.publication_datasources pd + join dd on dd.id=pd.datasource + left outer join ${stats_db_name}.result_accessroute ra on ra.id = pd.id where ra.accessroute = 'gold') tmp on tmp.id=pd.id; drop table if exists ${stats_db_name}.indi_pub_hybrid_oa_with_cc purge; @@ -421,15 +469,26 @@ drop table if exists ${stats_db_name}.indi_pub_hybrid purge; -- where (gold_oa.journal_is_in_doaj=false or gold_oa.journal_is_oa=false))tmp -- on pd.id=tmp.id; +--create table if not exists ${stats_db_name}.indi_pub_hybrid stored as parquet as +--select distinct pd.id,coalesce(is_hybrid,0) is_hybrid from ${stats_db_name}.publication_datasources pd +--left outer join (select pd.id, 1 as is_hybrid from ${stats_db_name}.publication_datasources pd +--join ${stats_db_name}.datasource d on pd.datasource=d.id +--join ${stats_db_name}.result_instance ri on ri.id=pd.id +--join ${stats_db_name}.indi_pub_gold_oa indi_gold on indi_gold.id=pd.id +--join ${stats_db_name}.result_accessroute ra on ra.id=pd.id +--where d.type like '%Journal%' and ri.accessright!='Closed Access' and (ri.accessright_uw!='gold' +--or indi_gold.is_gold=0) and (ra.accessroute='hybrid' or ri.license is not null)) tmp +--on pd.id=tmp.id; + create table if not exists ${stats_db_name}.indi_pub_hybrid stored as parquet as -select distinct pd.id,coalesce(is_hybrid,0) is_hybrid from ${stats_db_name}.publication_datasources pd -left outer join (select pd.id, 1 as is_hybrid from ${stats_db_name}.publication_datasources pd -join ${stats_db_name}.datasource d on pd.datasource=d.id +select distinct pd.id,coalesce(is_hybrid,0) is_hybrid from ${stats_db_name}.publication pd +left outer join (select pd.id, 1 as is_hybrid from ${stats_db_name}.publication pd join ${stats_db_name}.result_instance ri on ri.id=pd.id join ${stats_db_name}.indi_pub_gold_oa indi_gold on indi_gold.id=pd.id join ${stats_db_name}.result_accessroute ra on ra.id=pd.id -where d.type like '%Journal%' and ri.accessright!='Closed Access' and (ri.accessright_uw!='gold' -or indi_gold.is_gold=0) and (ra.accessroute='hybrid' or ri.license is not null)) tmp +join ${stats_db_name}.datasource d on d.id=ri.hostedby +where indi_gold.is_gold=0 and ((d.type like '%Journal%' and ri.accessright!='Closed Access' and ri.accessright!='Restricted' and ri.license is not null) or +ra.accessroute='hybrid'))tmp on pd.id=tmp.id; drop table if exists ${stats_db_name}.indi_org_fairness purge; @@ -814,14 +873,16 @@ drop table if exists ${stats_db_name}.indi_pub_bronze_oa purge; --and ri.accessright='Open Access') tmp on tmp.id=p.id; create table ${stats_db_name}.indi_pub_bronze_oa stored as parquet as -select distinct pd.id,coalesce(is_bronze_oa,0) is_bronze_oa from ${stats_db_name}.publication_datasources pd -left outer join (select pd.id, 1 as is_bronze_oa from ${stats_db_name}.publication_datasources pd -join ${stats_db_name}.datasource d on pd.datasource=d.id +select distinct pd.id,coalesce(is_bronze_oa,0) is_bronze_oa from ${stats_db_name}.publication pd +left outer join (select pd.id, 1 as is_bronze_oa from ${stats_db_name}.publication pd join ${stats_db_name}.result_instance ri on ri.id=pd.id join ${stats_db_name}.indi_pub_gold_oa indi_gold on indi_gold.id=pd.id +join ${stats_db_name}.indi_pub_hybrid indi_hybrid on indi_hybrid.id=pd.id join ${stats_db_name}.result_accessroute ra on ra.id=pd.id -where d.type like '%Journal%' and ri.accessright!='Closed Access' and (ri.accessright_uw!='gold' -or indi_gold.is_gold=0) and (ra.accessroute='bronze' or ri.license is null)) tmp +join ${stats_db_name}.datasource d on d.id=ri.hostedby +where indi_gold.is_gold=0 and indi_hybrid.is_hybrid=0 +and ((d.type like '%Journal%' and ri.accessright!='Closed Access' +and ri.accessright!='Restricted' and ri.license is null) or ra.accessroute='bronze')) tmp on pd.id=tmp.id; CREATE TEMPORARY TABLE ${stats_db_name}.project_year_result_year as diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql index b52abd865..c61a19e5c 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql @@ -64,6 +64,8 @@ create table TARGET.result_accessroute stored as parquet as select * from SOURCE create table TARGET.result_orcid stored as parquet as select * from SOURCE.result_orcid orig where exists (select 1 from TARGET.result r where r.id=orig.id); +create table TARGET.result_instance stored as parquet as select * from SOURCE.result_instance orig where exists (select 1 from TARGET.result r where r.id=orig.id); + create view TARGET.foo1 as select * from SOURCE.result_result rr where rr.source in (select id from TARGET.result); create view TARGET.foo2 as select * from SOURCE.result_result rr where rr.target in (select id from TARGET.result); create table TARGET.result_result STORED AS PARQUET as select distinct * from (select * from TARGET.foo1 union all select * from TARGET.foo2) foufou; diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql index 2b6a68514..167aac726 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql @@ -248,6 +248,8 @@ create table TARGET.indi_impact_measures stored as parquet as select * from SOUR create table TARGET.indi_pub_interdisciplinarity stored as parquet as select * from SOURCE.indi_pub_interdisciplinarity orig where exists (select 1 from TARGET.result r where r.id=orig.id); create table TARGET.result_apc_affiliations stored as parquet as select * from SOURCE.result_apc_affiliations orig where exists (select 1 from TARGET.result r where r.id=orig.id); +create table TARGET.result_instance stored as parquet as select * from SOURCE.result_instance orig where exists (select 1 from TARGET.result r where r.id=orig.id); +create table TARGET.result_orcid stored as parquet as select * from SOURCE.result_orcid orig where exists (select 1 from TARGET.result r where r.id=orig.id); create table TARGET.indi_is_project_result_after stored as parquet as select * from SOURCE.indi_is_project_result_after orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); create view TARGET.indi_is_funder_plan_s as select * from SOURCE.indi_is_funder_plan_s; diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step3.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step3.sql index eb97263a7..0384de4ec 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step3.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step3.sql @@ -5,6 +5,8 @@ ------------------------------------------------------ -- Dataset temporary table supporting updates +DROP TABLE IF EXISTS ${stats_db_name}.dataset_tmp purge; + CREATE TABLE ${stats_db_name}.dataset_tmp ( id STRING, @@ -40,6 +42,8 @@ SELECT substr(d.id, 4) AS id, FROM ${openaire_db_name}.dataset d WHERE d.datainfo.deletedbyinference = FALSE and d.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.dataset_citations purge; + CREATE TABLE ${stats_db_name}.dataset_citations STORED AS PARQUET AS SELECT substr(d.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS cites FROM ${openaire_db_name}.dataset d @@ -47,12 +51,16 @@ FROM ${openaire_db_name}.dataset d WHERE xpath_string(citation.value, "//citation/id[@type='openaire']/@value") != "" and d.datainfo.deletedbyinference = false and d.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.dataset_classifications purge; + CREATE TABLE ${stats_db_name}.dataset_classifications STORED AS PARQUET AS SELECT substr(p.id, 4) AS id, instancetype.classname AS type FROM ${openaire_db_name}.dataset p LATERAL VIEW explode(p.instance.instancetype) instances AS instancetype where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.dataset_concepts purge; + CREATE TABLE ${stats_db_name}.dataset_concepts STORED AS PARQUET AS SELECT substr(p.id, 4) as id, case when contexts.context.id RLIKE '^[^::]+::[^::]+::.+$' then contexts.context.id @@ -62,6 +70,8 @@ from ${openaire_db_name}.dataset p LATERAL VIEW explode(p.context) contexts as context where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.dataset_datasources purge; + CREATE TABLE ${stats_db_name}.dataset_datasources STORED AS PARQUET AS SELECT p.id, case when d.id IS NULL THEN 'other' ELSE p.datasource END AS datasource FROM ( @@ -74,23 +84,31 @@ FROM ( FROM ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference = false and d.datainfo.invisible=false) d ON p.datasource = d.id; +DROP TABLE IF EXISTS ${stats_db_name}.dataset_languages purge; + CREATE TABLE ${stats_db_name}.dataset_languages STORED AS PARQUET AS SELECT substr(p.id, 4) AS id, p.language.classname AS language FROM ${openaire_db_name}.dataset p where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.dataset_oids purge; + CREATE TABLE ${stats_db_name}.dataset_oids STORED AS PARQUET AS SELECT substr(p.id, 4) AS id, oids.ids AS oid FROM ${openaire_db_name}.dataset p LATERAL VIEW explode(p.originalid) oids AS ids where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.dataset_pids purge; + CREATE TABLE ${stats_db_name}.dataset_pids STORED AS PARQUET AS SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value AS pid FROM ${openaire_db_name}.dataset p LATERAL VIEW explode(p.pid) pids AS ppid where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.dataset_topics purge; + CREATE TABLE ${stats_db_name}.dataset_topics STORED AS PARQUET AS SELECT substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS type, subjects.subject.value AS topic FROM ${openaire_db_name}.dataset p diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step4.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step4.sql index 0d1f6323e..d8f4d65e4 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step4.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step4.sql @@ -5,6 +5,7 @@ -------------------------------------------------------- -- Software temporary table supporting updates +DROP TABLE IF EXISTS ${stats_db_name}.software_tmp purge; CREATE TABLE ${stats_db_name}.software_tmp ( id STRING, @@ -40,6 +41,8 @@ SELECT substr(s.id, 4) as id, from ${openaire_db_name}.software s where s.datainfo.deletedbyinference = false and s.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.software_citations purge; + CREATE TABLE ${stats_db_name}.software_citations STORED AS PARQUET AS SELECT substr(s.id, 4) as id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS cites FROM ${openaire_db_name}.software s @@ -47,6 +50,8 @@ FROM ${openaire_db_name}.software s where xpath_string(citation.value, "//citation/id[@type='openaire']/@value") != "" and s.datainfo.deletedbyinference = false and s.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.software_classifications purge; + CREATE TABLE ${stats_db_name}.software_classifications STORED AS PARQUET AS SELECT substr(p.id, 4) AS id, instancetype.classname AS type FROM ${openaire_db_name}.software p @@ -62,6 +67,8 @@ FROM ${openaire_db_name}.software p LATERAL VIEW explode(p.context) contexts AS context where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.software_datasources purge; + CREATE TABLE ${stats_db_name}.software_datasources STORED AS PARQUET AS SELECT p.id, CASE WHEN d.id IS NULL THEN 'other' ELSE p.datasource end as datasource FROM ( @@ -74,23 +81,31 @@ FROM ( FROM ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference = false and d.datainfo.invisible=false) d ON p.datasource = d.id; +DROP TABLE IF EXISTS ${stats_db_name}.software_languages purge; + CREATE TABLE ${stats_db_name}.software_languages STORED AS PARQUET AS select substr(p.id, 4) AS id, p.language.classname AS language FROM ${openaire_db_name}.software p where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.software_oids purge; + CREATE TABLE ${stats_db_name}.software_oids STORED AS PARQUET AS SELECT substr(p.id, 4) AS id, oids.ids AS oid FROM ${openaire_db_name}.software p LATERAL VIEW explode(p.originalid) oids AS ids where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.software_pids purge; + CREATE TABLE ${stats_db_name}.software_pids STORED AS PARQUET AS SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value AS pid FROM ${openaire_db_name}.software p LATERAL VIEW explode(p.pid) pids AS ppid where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.software_topics purge; + CREATE TABLE ${stats_db_name}.software_topics STORED AS PARQUET AS SELECT substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS type, subjects.subject.value AS topic FROM ${openaire_db_name}.software p diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step5.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step5.sql index 06b616d6a..fae0fbb63 100755 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step5.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step5.sql @@ -5,6 +5,8 @@ -------------------------------------------------------------------------------- -- Otherresearchproduct temporary table supporting updates +DROP TABLE IF EXISTS ${stats_db_name}.otherresearchproduct_tmp purge; + CREATE TABLE ${stats_db_name}.otherresearchproduct_tmp ( id STRING, @@ -40,6 +42,8 @@ FROM ${openaire_db_name}.otherresearchproduct o WHERE o.datainfo.deletedbyinference = FALSE and o.datainfo.invisible=false; -- Otherresearchproduct_citations +DROP TABLE IF EXISTS ${stats_db_name}.otherresearchproduct_citations purge; + CREATE TABLE ${stats_db_name}.otherresearchproduct_citations STORED AS PARQUET AS SELECT substr(o.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS cites FROM ${openaire_db_name}.otherresearchproduct o LATERAL VIEW explode(o.extrainfo) citations AS citation @@ -51,6 +55,8 @@ SELECT substr(p.id, 4) AS id, instancetype.classname AS type FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.instance.instancetype) instances AS instancetype where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.otherresearchproduct_concepts purge; + CREATE TABLE ${stats_db_name}.otherresearchproduct_concepts STORED AS PARQUET AS SELECT substr(p.id, 4) as id, case when contexts.context.id RLIKE '^[^::]+::[^::]+::.+$' then contexts.context.id @@ -59,6 +65,8 @@ SELECT substr(p.id, 4) as id, case FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.context) contexts AS context where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.otherresearchproduct_datasources purge; + CREATE TABLE ${stats_db_name}.otherresearchproduct_datasources STORED AS PARQUET AS SELECT p.id, CASE WHEN d.id IS NULL THEN 'other' ELSE p.datasource END AS datasource FROM (SELECT substr(p.id, 4) AS id, substr(instances.instance.hostedby.key, 4) AS datasource @@ -68,21 +76,29 @@ FROM (SELECT substr(p.id, 4) AS id, substr(instances.instance.hostedby.key, 4) A from ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference = false and d.datainfo.invisible=false) d on p.datasource = d.id; +DROP TABLE IF EXISTS ${stats_db_name}.otherresearchproduct_languages purge; + CREATE TABLE ${stats_db_name}.otherresearchproduct_languages STORED AS PARQUET AS SELECT substr(p.id, 4) AS id, p.language.classname AS language FROM ${openaire_db_name}.otherresearchproduct p where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.otherresearchproduct_oids purge; + CREATE TABLE ${stats_db_name}.otherresearchproduct_oids STORED AS PARQUET AS SELECT substr(p.id, 4) AS id, oids.ids AS oid FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.originalid) oids AS ids where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.otherresearchproduct_pids purge; + CREATE TABLE ${stats_db_name}.otherresearchproduct_pids STORED AS PARQUET AS SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value AS pid FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.pid) pids AS ppid where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.otherresearchproduct_topics purge; + CREATE TABLE ${stats_db_name}.otherresearchproduct_topics STORED AS PARQUET AS SELECT substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS type, subjects.subject.value AS topic FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.subject) subjects AS subject diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step6.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step6.sql index e0522e149..e5b3f504e 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step6.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step6.sql @@ -3,29 +3,39 @@ -- Project table/view and Project related tables/views ------------------------------------------------------ ------------------------------------------------------ +DROP TABLE IF EXISTS ${stats_db_name}.project_oids purge; + CREATE TABLE ${stats_db_name}.project_oids STORED AS PARQUET AS SELECT substr(p.id, 4) AS id, oids.ids AS oid FROM ${openaire_db_name}.project p LATERAL VIEW explode(p.originalid) oids AS ids where p.datainfo.deletedbyinference=false and p.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.project_organizations purge; + CREATE TABLE ${stats_db_name}.project_organizations STORED AS PARQUET AS SELECT substr(r.source, 4) AS id, substr(r.target, 4) AS organization from ${openaire_db_name}.relation r WHERE r.reltype = 'projectOrganization' and r.source like '40|%' and r.datainfo.deletedbyinference = false and r.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.project_results purge; + CREATE TABLE ${stats_db_name}.project_results STORED AS PARQUET AS SELECT substr(r.target, 4) AS id, substr(r.source, 4) AS result, r.datainfo.provenanceaction.classname as provenance FROM ${openaire_db_name}.relation r WHERE r.reltype = 'resultProject' and r.target like '40|%' and r.datainfo.deletedbyinference = false and r.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.project_classification purge; + create table ${stats_db_name}.project_classification STORED AS PARQUET as select substr(p.id, 4) as id, class.h2020programme.code, class.level1, class.level2, class.level3 from ${openaire_db_name}.project p lateral view explode(p.h2020classification) classifs as class where p.datainfo.deletedbyinference=false and p.datainfo.invisible=false and class.h2020programme is not null; +DROP TABLE IF EXISTS ${stats_db_name}.project_tmp purge; + CREATE TABLE ${stats_db_name}.project_tmp ( id STRING, @@ -80,12 +90,16 @@ SELECT substr(p.id, 4) AS id, FROM ${openaire_db_name}.project p WHERE p.datainfo.deletedbyinference = false and p.datainfo.invisible=false; +DROP TABLE IF EXISTS ${stats_db_name}.funder purge; + create table ${stats_db_name}.funder STORED AS PARQUET as select distinct xpath_string(fund, '//funder/id') as id, xpath_string(fund, '//funder/name') as name, xpath_string(fund, '//funder/shortname') as shortname from ${openaire_db_name}.project p lateral view explode(p.fundingtree.value) fundingtree as fund; +DROP TABLE IF EXISTS ${stats_db_name}.project_organization_contribution purge; + CREATE TABLE ${stats_db_name}.project_organization_contribution STORED AS PARQUET AS SELECT distinct substr(r.source, 4) AS project, substr(r.target, 4) AS organization, properties[0].value contribution, properties[1].value currency From a397112cb86a6f26798b18ce159837b032878fe1 Mon Sep 17 00:00:00 2001 From: dimitrispie Date: Fri, 1 Dec 2023 15:00:18 +0200 Subject: [PATCH 123/148] Add new indicator Add indi_pub_publicly_funded --- .../scripts/step16-createIndicatorsTables.sql | 21 +++++++++++++++++++ .../scripts/step20-createMonitorDB.sql | 1 + .../scripts/step20-createMonitorDBAll.sql | 1 + 3 files changed, 23 insertions(+) diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql index fea449de6..5aa14e2c2 100755 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql @@ -1177,3 +1177,24 @@ select allresults.ri_initiative, result_findable.no_result_findable/allresults.n from allresults join result_findable on result_findable.ri_initiative=allresults.ri_initiative; +create table if not exists ${stats_db_name}.indi_pub_publicly_funded stored as parquet as +with org_names_pids as +(select org.id,name, pid from ${stats_db_name}.organization org +join ${stats_db_name}.organization_pids op on org.id=op.id), +publicly_funded_orgs as +(select distinct name from +(select pf.name from stats_ext.insitutions_for_publicly_funded pf +join ${stats_db_name}.fundref f on f.name=pf.name where f.type='government' +union all +select pf.name from stats_ext.insitutions_for_publicly_funded pf +join ${stats_db_name}.project p on p.funder=pf.name +union all +select pf.name from stats_ext.insitutions_for_publicly_funded pf +join org_names_pids op on (op.name=pf.name or op.pid=pf.ror) +and pf.publicly_funded='yes') foo) +select distinct p.id, coalesce(publicly_funded, 0) as publicly_funded +from ${stats_db_name}.publication p +left outer join ( +select distinct ro.id, 1 as publicly_funded from result_organization ro +join ${stats_db_name}.organization o on o.id=ro.organization +join publicly_funded_orgs pfo on o.name=pfo.name) tmp on p.id=tmp.id; \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql index c61a19e5c..cc8348f26 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql @@ -63,6 +63,7 @@ create table TARGET.result_fos stored as parquet as select * from SOURCE.result_ create table TARGET.result_accessroute stored as parquet as select * from SOURCE.result_accessroute orig where exists (select 1 from TARGET.result r where r.id=orig.id); create table TARGET.result_orcid stored as parquet as select * from SOURCE.result_orcid orig where exists (select 1 from TARGET.result r where r.id=orig.id); +create table TARGET.indi_pub_publicly_funded stored as parquet as select * from SOURCE.indi_pub_publicly_funded orig where exists (select 1 from TARGET.result r where r.id=orig.id); create table TARGET.result_instance stored as parquet as select * from SOURCE.result_instance orig where exists (select 1 from TARGET.result r where r.id=orig.id); diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql index 167aac726..68417f3e0 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql @@ -250,6 +250,7 @@ create table TARGET.indi_pub_interdisciplinarity stored as parquet as select * f create table TARGET.result_apc_affiliations stored as parquet as select * from SOURCE.result_apc_affiliations orig where exists (select 1 from TARGET.result r where r.id=orig.id); create table TARGET.result_instance stored as parquet as select * from SOURCE.result_instance orig where exists (select 1 from TARGET.result r where r.id=orig.id); create table TARGET.result_orcid stored as parquet as select * from SOURCE.result_orcid orig where exists (select 1 from TARGET.result r where r.id=orig.id); +create table TARGET.indi_pub_publicly_funded stored as parquet as select * from SOURCE.indi_pub_publicly_funded orig where exists (select 1 from TARGET.result r where r.id=orig.id); create table TARGET.indi_is_project_result_after stored as parquet as select * from SOURCE.indi_is_project_result_after orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); create view TARGET.indi_is_funder_plan_s as select * from SOURCE.indi_is_funder_plan_s; From c9d995dde0d48b56e99af43e4c91273817def678 Mon Sep 17 00:00:00 2001 From: dimitrispie Date: Fri, 1 Dec 2023 15:44:35 +0200 Subject: [PATCH 124/148] New institutions added --- .../stats/oozie_app/scripts/step20-createMonitorDBAll.sql | 5 ++++- .../scripts/step20-createMonitorDB_institutions.sql | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql index 68417f3e0..42812d159 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql @@ -78,7 +78,10 @@ create table TARGET.result stored as parquet as 'openorgs____::4d4051b56708688235252f1d8fddb8c1', -- Iscte - Instituto Universitário de Lisboa 'openorgs____::5d55fb216b14691cf68218daf5d78cd9', -- Munster Technological University 'openorgs____::0fccc7640f0cb44d5cd1b06b312a06b9', -- Cardiff University - 'openorgs____::8839b55dae0c84d56fd533f52d5d483a' -- Leibniz Institute of Ecological Urban and Regional Development + 'openorgs____::8839b55dae0c84d56fd533f52d5d483a', -- Leibniz Institute of Ecological Urban and Regional Development + 'openorgs____::526468206bca24c1c90da6a312295cf4', -- Cyprus University of Technology + 'openorgs____::b5ca9d4340e26454e367e2908ef3872f' -- Alma Mater Studiorum University of Bologna + ) )) foo; create view if not exists TARGET.category as select * from SOURCE.category; diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql index d2f08b391..2c0ac337c 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql @@ -58,5 +58,7 @@ create table TARGET.result stored as parquet as 'openorgs____::4d4051b56708688235252f1d8fddb8c1', -- Iscte - Instituto Universitário de Lisboa 'openorgs____::5d55fb216b14691cf68218daf5d78cd9', -- Munster Technological University 'openorgs____::0fccc7640f0cb44d5cd1b06b312a06b9', -- Cardiff University - 'openorgs____::8839b55dae0c84d56fd533f52d5d483a' -- Leibniz Institute of Ecological Urban and Regional Development + 'openorgs____::8839b55dae0c84d56fd533f52d5d483a', -- Leibniz Institute of Ecological Urban and Regional Development + 'openorgs____::526468206bca24c1c90da6a312295cf4', -- Cyprus University of Technology + 'openorgs____::b5ca9d4340e26454e367e2908ef3872f' -- Alma Mater Studiorum University of Bologna ))) foo; \ No newline at end of file From 33cb483c756808d9fc2363d5b27fdb5eeb35c355 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 1 Dec 2023 15:03:05 +0100 Subject: [PATCH 125/148] using objectSubType as originalType in Crossref2Oaf, code formatting --- .../dhp/datacite/DataciteToOAFTransformation.scala | 10 ++++++---- .../main/scala/eu/dnetlib/dhp/sx/bio/BioDBToOAF.scala | 1 - .../eu/dnetlib/dhp/sx/bio/pubmed/PubMedToOaf.scala | 5 ++--- .../eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/datacite/DataciteToOAFTransformation.scala b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/datacite/DataciteToOAFTransformation.scala index ee3660918..7603715b9 100644 --- a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/datacite/DataciteToOAFTransformation.scala +++ b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/datacite/DataciteToOAFTransformation.scala @@ -176,7 +176,8 @@ object DataciteToOAFTransformation { vocabularies.getSynonymAsQualifier( ModelConstants.DNET_RESULT_TYPOLOGIES, typeQualifier.getClassid - ), resourceType + ), + resourceType ) } if (schemaOrg != null && schemaOrg.nonEmpty) { @@ -188,7 +189,8 @@ object DataciteToOAFTransformation { vocabularies.getSynonymAsQualifier( ModelConstants.DNET_RESULT_TYPOLOGIES, typeQualifier.getClassid - ), schemaOrg + ), + schemaOrg ) } @@ -203,7 +205,8 @@ object DataciteToOAFTransformation { vocabularies.getSynonymAsQualifier( ModelConstants.DNET_RESULT_TYPOLOGIES, typeQualifier.getClassid - ), resourceTypeGeneral + ), + resourceTypeGeneral ) } @@ -228,7 +231,6 @@ object DataciteToOAFTransformation { itm.setVocabularyName(ModelConstants.OPENAIRE_COAR_RESOURCE_TYPES_3_1) i.setInstanceTypeMapping(List(itm).asJava) - typeQualifiers._2.getClassname match { case "dataset" => val r = new OafDataset diff --git a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/BioDBToOAF.scala b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/BioDBToOAF.scala index 89bc8d948..df356548a 100644 --- a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/BioDBToOAF.scala +++ b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/BioDBToOAF.scala @@ -593,7 +593,6 @@ object BioDBToOAF { itm.setVocabularyName(ModelConstants.OPENAIRE_COAR_RESOURCE_TYPES_3_1) i.setInstanceTypeMapping(List(itm).asJava) - i.setCollectedfrom(collectedFromMap("ebi")) d.setInstance(List(i).asJava) i.setDateofacceptance( diff --git a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/pubmed/PubMedToOaf.scala b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/pubmed/PubMedToOaf.scala index f3f8b4f02..d59d73bd0 100644 --- a/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/pubmed/PubMedToOaf.scala +++ b/dhp-workflows/dhp-aggregation/src/main/scala/eu/dnetlib/dhp/sx/bio/pubmed/PubMedToOaf.scala @@ -195,7 +195,7 @@ object PubMedToOaf { pubmedInstance.setInstanceTypeMapping(List(itm).asJava) } else { val i_type = article.getPublicationTypes.asScala - .map(s => (s.getValue,getVocabularyTerm(ModelConstants.DNET_PUBLICATION_RESOURCE, vocabularies, s.getValue))) + .map(s => (s.getValue, getVocabularyTerm(ModelConstants.DNET_PUBLICATION_RESOURCE, vocabularies, s.getValue))) .find(q => q._2 != null) if (i_type.isDefined) { @@ -205,8 +205,7 @@ object PubMedToOaf { itm.setOriginalType(i_type.get._1) itm.setVocabularyName(ModelConstants.OPENAIRE_COAR_RESOURCE_TYPES_3_1) pubmedInstance.setInstanceTypeMapping(List(itm).asJava) - } - else + } else return null } val result = createResult(pubmedInstance.getInstancetype, vocabularies) diff --git a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala index 65e395357..53576fc10 100644 --- a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala +++ b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala @@ -107,7 +107,7 @@ case object Crossref2Oaf { .map(f => f.id) } - def mappingResult(result: Result, json: JValue, cobjCategory: String, originalType:String): Result = { + def mappingResult(result: Result, json: JValue, cobjCategory: String, originalType: String): Result = { implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats //MAPPING Crossref DOI into PID @@ -372,7 +372,7 @@ case object Crossref2Oaf { objectType, mappingCrossrefSubType.getOrElse(objectSubType, "0038 Other literature type") ) - mappingResult(result, json, cOBJCategory, originalType) + mappingResult(result, json, cOBJCategory, objectSubType) if (result == null || result.getId == null) return List() From d33f578e544c23eb8e1a1dd4edfd79b0a42d854a Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 1 Dec 2023 15:14:17 +0100 Subject: [PATCH 126/148] code formatting --- .../eu/dnetlib/dhp/oa/merge/AuthorMerger.java | 511 +++++++++--------- 1 file changed, 256 insertions(+), 255 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java index 0153e52d5..0461c9353 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/oa/merge/AuthorMerger.java @@ -4,318 +4,319 @@ package eu.dnetlib.dhp.oa.merge; import java.text.Normalizer; import java.util.*; import java.util.stream.Collectors; + import org.apache.commons.lang3.StringUtils; + import com.wcohen.ss.JaroWinkler; + import eu.dnetlib.dhp.schema.oaf.Author; import eu.dnetlib.dhp.schema.oaf.StructuredProperty; import eu.dnetlib.pace.model.Person; import scala.Tuple2; - public class AuthorMerger { - private static final Double THRESHOLD = 0.95; + private static final Double THRESHOLD = 0.95; - private AuthorMerger() { - } + private AuthorMerger() { + } - public static List merge(List> authors) { + public static List merge(List> authors) { - authors.sort((o1, o2) -> -Integer.compare(countAuthorsPids(o1), countAuthorsPids(o2))); + authors.sort((o1, o2) -> -Integer.compare(countAuthorsPids(o1), countAuthorsPids(o2))); - List author = new ArrayList<>(); + List author = new ArrayList<>(); - for (List a : authors) { - author = mergeAuthor(author, a); - } + for (List a : authors) { + author = mergeAuthor(author, a); + } - return author; + return author; - } + } - public static List mergeAuthor(final List a, final List b, Double threshold) { - int pa = countAuthorsPids(a); - int pb = countAuthorsPids(b); - List base; - List enrich; - int sa = authorsSize(a); - int sb = authorsSize(b); + public static List mergeAuthor(final List a, final List b, Double threshold) { + int pa = countAuthorsPids(a); + int pb = countAuthorsPids(b); + List base; + List enrich; + int sa = authorsSize(a); + int sb = authorsSize(b); - if (sa == sb) { - base = pa > pb ? a : b; - enrich = pa > pb ? b : a; - } else { - base = sa > sb ? a : b; - enrich = sa > sb ? b : a; - } - enrichPidFromList(base, enrich, threshold); - return base; - } + if (sa == sb) { + base = pa > pb ? a : b; + enrich = pa > pb ? b : a; + } else { + base = sa > sb ? a : b; + enrich = sa > sb ? b : a; + } + enrichPidFromList(base, enrich, threshold); + return base; + } - public static List mergeAuthor(final List a, final List b) { - return mergeAuthor(a, b, THRESHOLD); - } + public static List mergeAuthor(final List a, final List b) { + return mergeAuthor(a, b, THRESHOLD); + } - private static void enrichPidFromList(List base, List enrich, Double threshold) { - if (base == null || enrich == null) - return; + private static void enrichPidFromList(List base, List enrich, Double threshold) { + if (base == null || enrich == null) + return; - // (if an Author has more than 1 pid, it appears 2 times in the list) - final Map basePidAuthorMap = base - .stream() - .filter(a -> a.getPid() != null && !a.getPid().isEmpty()) - .flatMap( - a -> a - .getPid() - .stream() - .filter(Objects::nonNull) - .map(p -> new Tuple2<>(pidToComparableString(p), a))) - .collect(Collectors.toMap(Tuple2::_1, Tuple2::_2, (x1, x2) -> x1)); + // (if an Author has more than 1 pid, it appears 2 times in the list) + final Map basePidAuthorMap = base + .stream() + .filter(a -> a.getPid() != null && !a.getPid().isEmpty()) + .flatMap( + a -> a + .getPid() + .stream() + .filter(Objects::nonNull) + .map(p -> new Tuple2<>(pidToComparableString(p), a))) + .collect(Collectors.toMap(Tuple2::_1, Tuple2::_2, (x1, x2) -> x1)); - // (list of pid that are missing in the other list) - final List> pidToEnrich = enrich - .stream() - .filter(a -> a.getPid() != null && !a.getPid().isEmpty()) - .flatMap( - a -> a - .getPid() - .stream() - .filter(Objects::nonNull) - .filter(p -> !basePidAuthorMap.containsKey(pidToComparableString(p))) - .map(p -> new Tuple2<>(p, a))) - .collect(Collectors.toList()); + // (list of pid that are missing in the other list) + final List> pidToEnrich = enrich + .stream() + .filter(a -> a.getPid() != null && !a.getPid().isEmpty()) + .flatMap( + a -> a + .getPid() + .stream() + .filter(Objects::nonNull) + .filter(p -> !basePidAuthorMap.containsKey(pidToComparableString(p))) + .map(p -> new Tuple2<>(p, a))) + .collect(Collectors.toList()); - pidToEnrich - .forEach( - a -> { - Optional> simAuthor = base - .stream() - .map(ba -> new Tuple2<>(sim(ba, a._2()), ba)) - .max(Comparator.comparing(Tuple2::_1)); + pidToEnrich + .forEach( + a -> { + Optional> simAuthor = base + .stream() + .map(ba -> new Tuple2<>(sim(ba, a._2()), ba)) + .max(Comparator.comparing(Tuple2::_1)); - if (simAuthor.isPresent()) { - double th = threshold; - // increase the threshold if the surname is too short - if (simAuthor.get()._2().getSurname() != null - && simAuthor.get()._2().getSurname().length() <= 3 && threshold > 0.0) - th = 0.99; + if (simAuthor.isPresent()) { + double th = threshold; + // increase the threshold if the surname is too short + if (simAuthor.get()._2().getSurname() != null + && simAuthor.get()._2().getSurname().length() <= 3 && threshold > 0.0) + th = 0.99; - if (simAuthor.get()._1() > th) { - Author r = simAuthor.get()._2(); - if (r.getPid() == null) { - r.setPid(new ArrayList<>()); - } + if (simAuthor.get()._1() > th) { + Author r = simAuthor.get()._2(); + if (r.getPid() == null) { + r.setPid(new ArrayList<>()); + } - // TERRIBLE HACK but for some reason when we create and Array with Arrays.asList, - // it creates of fixed size, and the add method raise UnsupportedOperationException at - // java.util.AbstractList.add - final List tmp = new ArrayList<>(r.getPid()); - tmp.add(a._1()); - r.setPid(tmp); - } - } - }); - } + // TERRIBLE HACK but for some reason when we create and Array with Arrays.asList, + // it creates of fixed size, and the add method raise UnsupportedOperationException at + // java.util.AbstractList.add + final List tmp = new ArrayList<>(r.getPid()); + tmp.add(a._1()); + r.setPid(tmp); + } + } + }); + } - public static String normalizeFullName(final String fullname) { - return nfd(fullname) - .toLowerCase() - // do not compact the regexes in a single expression, would cause StackOverflowError - // in case - // of large input strings - .replaceAll("(\\W)+", " ") - .replaceAll("(\\p{InCombiningDiacriticalMarks})+", " ") - .replaceAll("(\\p{Punct})+", " ") - .replaceAll("(\\d)+", " ") - .replaceAll("(\\n)+", " ") + public static String normalizeFullName(final String fullname) { + return nfd(fullname) + .toLowerCase() + // do not compact the regexes in a single expression, would cause StackOverflowError + // in case + // of large input strings + .replaceAll("(\\W)+", " ") + .replaceAll("(\\p{InCombiningDiacriticalMarks})+", " ") + .replaceAll("(\\p{Punct})+", " ") + .replaceAll("(\\d)+", " ") + .replaceAll("(\\n)+", " ") - .trim(); - } + .trim(); + } + private static String authorFieldToBeCompared(Author author) { + if (StringUtils.isNotBlank(author.getSurname())) { + return author.getSurname(); - private static String authorFieldToBeCompared(Author author) { - if (StringUtils.isNotBlank(author.getSurname())) { - return author.getSurname(); + } + if (StringUtils.isNotBlank(author.getFullname())) { + return author.getFullname(); + } + return null; + } - } - if (StringUtils.isNotBlank(author.getFullname())) { - return author.getFullname(); - } - return null; - } + /** + * This method tries to figure out when two author are the same in the contest + * of ORCID enrichment + * + * @param left Author in the OAF entity + * @param right Author ORCID + * @return based on a heuristic on the names of the authors if they are the same. + */ + public static boolean checkORCIDSimilarity(final Author left, final Author right) { + final Person pl = parse(left); + final Person pr = parse(right); - /** - * This method tries to figure out when two author are the same in the contest - * of ORCID enrichment - * - * @param left Author in the OAF entity - * @param right Author ORCID - * @return based on a heuristic on the names of the authors if they are the same. - */ - public static boolean checkORCIDSimilarity(final Author left, final Author right) { - final Person pl = parse(left); - final Person pr = parse(right); + // If one of them didn't have a surname we verify if they have the fullName not empty + // and verify if the normalized version is equal + if (!(pl.getSurname() != null && pl.getSurname().stream().anyMatch(StringUtils::isNotBlank) && + pr.getSurname() != null && pr.getSurname().stream().anyMatch(StringUtils::isNotBlank))) { - // If one of them didn't have a surname we verify if they have the fullName not empty - // and verify if the normalized version is equal - if (!(pl.getSurname() != null && pl.getSurname().stream().anyMatch(StringUtils::isNotBlank) && - pr.getSurname() != null && pr.getSurname().stream().anyMatch(StringUtils::isNotBlank))) { + if (pl.getFullname() != null && !pl.getFullname().isEmpty() && pr.getFullname() != null + && !pr.getFullname().isEmpty()) { + return pl + .getFullname() + .stream() + .anyMatch( + fl -> pr.getFullname().stream().anyMatch(fr -> normalize(fl).equalsIgnoreCase(normalize(fr)))); + } else { + return false; + } + } + // The Authors have one surname in common + if (pl.getSurname().stream().anyMatch(sl -> pr.getSurname().stream().anyMatch(sr -> sr.equalsIgnoreCase(sl)))) { - if (pl.getFullname() != null && !pl.getFullname().isEmpty() && pr.getFullname() != null - && !pr.getFullname().isEmpty()) { - return pl - .getFullname() - .stream() - .anyMatch( - fl -> pr.getFullname().stream().anyMatch(fr -> normalize(fl).equalsIgnoreCase(normalize(fr)))); - } else { - return false; - } - } - // The Authors have one surname in common - if (pl.getSurname().stream().anyMatch(sl -> pr.getSurname().stream().anyMatch(sr -> sr.equalsIgnoreCase(sl)))) { + // If one of them has only a surname and is the same we can say that they are the same author + if ((pl.getName() == null || pl.getName().stream().allMatch(StringUtils::isBlank)) || + (pr.getName() == null || pr.getName().stream().allMatch(StringUtils::isBlank))) + return true; + // The authors have the same initials of Name in common + if (pl + .getName() + .stream() + .anyMatch( + nl -> pr + .getName() + .stream() + .anyMatch(nr -> nr.equalsIgnoreCase(nl)))) + return true; + } - // If one of them has only a surname and is the same we can say that they are the same author - if ((pl.getName() == null || pl.getName().stream().allMatch(StringUtils::isBlank)) || - (pr.getName() == null || pr.getName().stream().allMatch(StringUtils::isBlank))) - return true; - // The authors have the same initials of Name in common - if (pl - .getName() - .stream() - .anyMatch( - nl -> pr - .getName() - .stream() - .anyMatch(nr -> nr.equalsIgnoreCase(nl)))) - return true; - } + // Sometimes we noticed that publication have author wrote in inverse order Surname, Name + // We verify if we have an exact match between name and surname + if (pl.getSurname().stream().anyMatch(sl -> pr.getName().stream().anyMatch(nr -> nr.equalsIgnoreCase(sl))) && + pl.getName().stream().anyMatch(nl -> pr.getSurname().stream().anyMatch(sr -> sr.equalsIgnoreCase(nl)))) + return true; + else + return false; + } + // - // Sometimes we noticed that publication have author wrote in inverse order Surname, Name - // We verify if we have an exact match between name and surname - if (pl.getSurname().stream().anyMatch(sl -> pr.getName().stream().anyMatch(nr -> nr.equalsIgnoreCase(sl))) && - pl.getName().stream().anyMatch(nl -> pr.getSurname().stream().anyMatch(sr -> sr.equalsIgnoreCase(nl)))) - return true; - else - return false; - } - // + /** + * Method to enrich ORCID information in one list of authors based on another list + * + * @param baseAuthor the Author List in the OAF Entity + * @param orcidAuthor The list of ORCID Author intersected + * @return The Author List of the OAF Entity enriched with the orcid Author + */ + public static List enrichOrcid(List baseAuthor, List orcidAuthor) { - /** - * Method to enrich ORCID information in one list of authors based on another list - * - * @param baseAuthor the Author List in the OAF Entity - * @param orcidAuthor The list of ORCID Author intersected - * @return The Author List of the OAF Entity enriched with the orcid Author - */ - public static List enrichOrcid(List baseAuthor, List orcidAuthor) { + if (baseAuthor == null || baseAuthor.isEmpty()) + return orcidAuthor; - if (baseAuthor == null || baseAuthor.isEmpty()) - return orcidAuthor; + if (orcidAuthor == null || orcidAuthor.isEmpty()) + return baseAuthor; - if (orcidAuthor == null || orcidAuthor.isEmpty()) - return baseAuthor; + if (baseAuthor.size() == 1 && orcidAuthor.size() > 10) + return baseAuthor; - if (baseAuthor.size() == 1 && orcidAuthor.size() > 10) - return baseAuthor; + final List oAuthor = new ArrayList<>(); + oAuthor.addAll(orcidAuthor); - final List oAuthor = new ArrayList<>(); - oAuthor.addAll(orcidAuthor); + baseAuthor.forEach(ba -> { + Optional aMatch = oAuthor.stream().filter(oa -> checkORCIDSimilarity(ba, oa)).findFirst(); + if (aMatch.isPresent()) { + final Author sameAuthor = aMatch.get(); + addPid(ba, sameAuthor.getPid()); + oAuthor.remove(sameAuthor); + } + }); + return baseAuthor; + } - baseAuthor.forEach(ba -> { - Optional aMatch = oAuthor.stream().filter(oa -> checkORCIDSimilarity(ba, oa)).findFirst(); - if (aMatch.isPresent()) { - final Author sameAuthor = aMatch.get(); - addPid(ba, sameAuthor.getPid()); - oAuthor.remove(sameAuthor); - } - }); - return baseAuthor; - } + private static void addPid(final Author a, final List pids) { - private static void addPid(final Author a, final List pids) { + if (a.getPid() == null) { + a.setPid(new ArrayList<>()); + } - if (a.getPid() == null) { - a.setPid(new ArrayList<>()); - } + a.getPid().addAll(pids); - a.getPid().addAll(pids); + } - } + public static String pidToComparableString(StructuredProperty pid) { + final String classid = pid.getQualifier().getClassid() != null ? pid.getQualifier().getClassid().toLowerCase() + : ""; + return (pid.getQualifier() != null ? classid : "") + + (pid.getValue() != null ? pid.getValue().toLowerCase() : ""); + } - public static String pidToComparableString(StructuredProperty pid) { - final String classid = pid.getQualifier().getClassid() != null ? pid.getQualifier().getClassid().toLowerCase() - : ""; - return (pid.getQualifier() != null ? classid : "") - + (pid.getValue() != null ? pid.getValue().toLowerCase() : ""); - } + public static int countAuthorsPids(List authors) { + if (authors == null) + return 0; - public static int countAuthorsPids(List authors) { - if (authors == null) - return 0; + return (int) authors.stream().filter(AuthorMerger::hasPid).count(); + } - return (int) authors.stream().filter(AuthorMerger::hasPid).count(); - } + private static int authorsSize(List authors) { + if (authors == null) + return 0; + return authors.size(); + } - private static int authorsSize(List authors) { - if (authors == null) - return 0; - return authors.size(); - } + private static Double sim(Author a, Author b) { - private static Double sim(Author a, Author b) { + final Person pa = parse(a); + final Person pb = parse(b); - final Person pa = parse(a); - final Person pb = parse(b); + // if both are accurate (e.g. they have name and surname) + if (pa.isAccurate() & pb.isAccurate()) { + return new JaroWinkler().score(normalize(pa.getSurnameString()), normalize(pb.getSurnameString())) * 0.5 + + new JaroWinkler().score(normalize(pa.getNameString()), normalize(pb.getNameString())) * 0.5; + } else { + return new JaroWinkler() + .score(normalize(pa.getNormalisedFullname()), normalize(pb.getNormalisedFullname())); + } + } - // if both are accurate (e.g. they have name and surname) - if (pa.isAccurate() & pb.isAccurate()) { - return new JaroWinkler().score(normalize(pa.getSurnameString()), normalize(pb.getSurnameString())) * 0.5 - + new JaroWinkler().score(normalize(pa.getNameString()), normalize(pb.getNameString())) * 0.5; - } else { - return new JaroWinkler() - .score(normalize(pa.getNormalisedFullname()), normalize(pb.getNormalisedFullname())); - } - } + private static boolean hasPid(Author a) { + if (a == null || a.getPid() == null || a.getPid().isEmpty()) + return false; + return a.getPid().stream().anyMatch(p -> p != null && StringUtils.isNotBlank(p.getValue())); + } - private static boolean hasPid(Author a) { - if (a == null || a.getPid() == null || a.getPid().isEmpty()) - return false; - return a.getPid().stream().anyMatch(p -> p != null && StringUtils.isNotBlank(p.getValue())); - } + private static Person parse(Author author) { + if (StringUtils.isNotBlank(author.getSurname())) { + return new Person(author.getSurname() + ", " + author.getName(), false); + } else { + if (StringUtils.isNotBlank(author.getFullname())) + return new Person(author.getFullname(), false); + else + return new Person("", false); + } + } - private static Person parse(Author author) { - if (StringUtils.isNotBlank(author.getSurname())) { - return new Person(author.getSurname() + ", " + author.getName(), false); - } else { - if (StringUtils.isNotBlank(author.getFullname())) - return new Person(author.getFullname(), false); - else - return new Person("", false); - } - } + public static String normalize(final String s) { + String[] normalized = nfd(s) + .toLowerCase() + // do not compact the regexes in a single expression, would cause StackOverflowError + // in case + // of large input strings + .replaceAll("(\\W)+", " ") + .replaceAll("(\\p{InCombiningDiacriticalMarks})+", " ") + .replaceAll("(\\p{Punct})+", " ") + .replaceAll("(\\d)+", " ") + .replaceAll("(\\n)+", " ") + .trim() + .split(" "); - public static String normalize(final String s) { - String[] normalized = nfd(s) - .toLowerCase() - // do not compact the regexes in a single expression, would cause StackOverflowError - // in case - // of large input strings - .replaceAll("(\\W)+", " ") - .replaceAll("(\\p{InCombiningDiacriticalMarks})+", " ") - .replaceAll("(\\p{Punct})+", " ") - .replaceAll("(\\d)+", " ") - .replaceAll("(\\n)+", " ") - .trim() - .split(" "); + Arrays.sort(normalized); - Arrays.sort(normalized); + return String.join(" ", normalized); + } - return String.join(" ", normalized); - } - - private static String nfd(final String s) { - return Normalizer.normalize(s, Normalizer.Form.NFD); - } + private static String nfd(final String s) { + return Normalizer.normalize(s, Normalizer.Form.NFD); + } } From 511a98dd800d2697134b0c01f36f7b09f2caa8f1 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 1 Dec 2023 16:21:53 +0100 Subject: [PATCH 127/148] fixed doiboost process workflow, removed references to the ProcessORCID step --- .../eu/dnetlib/dhp/doiboost/process/oozie_app/workflow.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/process/oozie_app/workflow.xml b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/process/oozie_app/workflow.xml index 8f28d706d..84cbc5232 100644 --- a/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/process/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-doiboost/src/main/resources/eu/dnetlib/dhp/doiboost/process/oozie_app/workflow.xml @@ -84,7 +84,6 @@ ${wf:conf('resumeFrom') eq 'Skip'} ${wf:conf('resumeFrom') eq 'PreprocessMag'} ${wf:conf('resumeFrom') eq 'PreprocessUW'} - ${wf:conf('resumeFrom') eq 'ProcessORCID'} ${wf:conf('resumeFrom') eq 'CreateDOIBoost'} ${wf:conf('resumeFrom') eq 'GenerateActionSet'} From 3caf6ff27ef91aa6b684fdb91ea4d7e1274f9b3d Mon Sep 17 00:00:00 2001 From: Sandro La Bruzzo Date: Fri, 1 Dec 2023 16:33:56 +0100 Subject: [PATCH 128/148] Extracted the correct original type to pass to instanceTypeMapping in Crossref Mapping --- .../scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala index 53576fc10..91118c636 100644 --- a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala +++ b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala @@ -372,7 +372,9 @@ case object Crossref2Oaf { objectType, mappingCrossrefSubType.getOrElse(objectSubType, "0038 Other literature type") ) - mappingResult(result, json, cOBJCategory, objectSubType) + + val originalType = if (mappingCrossrefSubType.contains(objectType)) objectType else objectSubType + mappingResult(result, json, cOBJCategory, originalType) if (result == null || result.getId == null) return List() From 178a14c491ea24847d02e3eb2295a08198b37d9b Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Sun, 3 Dec 2023 13:31:58 +0100 Subject: [PATCH 129/148] code formatting --- .../main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala index 91118c636..ee857e2c4 100644 --- a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala +++ b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala @@ -373,7 +373,7 @@ case object Crossref2Oaf { mappingCrossrefSubType.getOrElse(objectSubType, "0038 Other literature type") ) - val originalType = if (mappingCrossrefSubType.contains(objectType)) objectType else objectSubType + val originalType = if (mappingCrossrefSubType.contains(objectType)) objectType else objectSubType mappingResult(result, json, cOBJCategory, originalType) if (result == null || result.getId == null) return List() From 2a233a89aa49c2ec59c95ba86d256ab46eb3db9b Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Sun, 3 Dec 2023 13:32:52 +0100 Subject: [PATCH 130/148] [graph grouping] added isLookupUrl to the workflow definition, passed to the grouping spark aciton --- .../eu/dnetlib/dhp/oa/graph/group/oozie_app/workflow.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/group/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/group/oozie_app/workflow.xml index 190788c9d..2f44bea9d 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/group/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/oa/graph/group/oozie_app/workflow.xml @@ -12,6 +12,10 @@ graphOutputPath path of the output graph + + isLookupUrl + the URL address of the lookUp service + filterInvisible whether filter out invisible entities after merge @@ -98,6 +102,7 @@ --graphInputPath${graphBasePath} --checkpointPath${workingPath}/grouped_entities --outputPath${graphOutputPath} + --isLookupUrl${isLookupUrl} --filterInvisible${filterInvisible} From e6086efc53804cf3e29d29fd6b2f1b8f958f9130 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Sun, 3 Dec 2023 13:33:20 +0100 Subject: [PATCH 131/148] avoid NPEs in Vocabulary.getTermBySynonym --- .../java/eu/dnetlib/dhp/common/vocabulary/Vocabulary.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/common/vocabulary/Vocabulary.java b/dhp-common/src/main/java/eu/dnetlib/dhp/common/vocabulary/Vocabulary.java index 2ab23bda6..94e7f5221 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/common/vocabulary/Vocabulary.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/common/vocabulary/Vocabulary.java @@ -63,7 +63,10 @@ public class Vocabulary implements Serializable { } public VocabularyTerm getTermBySynonym(final String syn) { - return getTerm(synonyms.get(syn.toLowerCase())); + return Optional + .ofNullable(syn) + .map(s -> getTerm(synonyms.get(s.toLowerCase()))) + .orElse(null); } public Qualifier getTermAsQualifier(final String termId) { From 74b185d07b3655f6e964ce5ae0610695d1685903 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Sun, 3 Dec 2023 16:18:20 +0100 Subject: [PATCH 132/148] avoid NPEs --- .../dhp/schema/oaf/utils/GraphCleaningFunctions.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java index 7cb32eab5..c65f36de9 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java @@ -943,17 +943,17 @@ public class GraphCleaningFunctions extends CleaningFunctions { private static Qualifier getMetaResourceType(final List instances, final VocabularyGroup vocs) { if (vocs.vocabularyExists(OPENAIRE_META_RESOURCE_TYPE)) { - Optional instanceTypeMapping = instances + Optional itm = instances .stream() .flatMap( i -> Optional.ofNullable(i.getInstanceTypeMapping()).map(Collection::stream).orElse(Stream.empty())) .filter(t -> OPENAIRE_COAR_RESOURCE_TYPES_3_1.equals(t.getVocabularyName())) .findFirst(); - if (!instanceTypeMapping.isPresent()) { + if (!itm.isPresent() || Objects.isNull(itm.get().getTypeCode())) { return null; } else { - final String typeCode = instanceTypeMapping.get().getTypeCode(); + final String typeCode = itm.get().getTypeCode(); return Optional .ofNullable(vocs.lookupTermBySynonym(OPENAIRE_META_RESOURCE_TYPE, typeCode)) .orElseThrow( From 7c3041b276c2c27f235f68e3ebdc4ae3b862dd5b Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Sun, 3 Dec 2023 16:49:49 +0100 Subject: [PATCH 133/148] avoid NPEs --- .../oaf/utils/GraphCleaningFunctions.java | 50 +++++++++++-------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java index c65f36de9..c25d67f25 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java @@ -941,28 +941,36 @@ public class GraphCleaningFunctions extends CleaningFunctions { } private static Qualifier getMetaResourceType(final List instances, final VocabularyGroup vocs) { + return Optional + .ofNullable(instances) + .map(ii -> { + if (vocs.vocabularyExists(OPENAIRE_META_RESOURCE_TYPE)) { + Optional itm = ii + .stream() + .filter(Objects::nonNull) + .flatMap( + i -> Optional + .ofNullable(i.getInstanceTypeMapping()) + .map(Collection::stream) + .orElse(Stream.empty())) + .filter(t -> OPENAIRE_COAR_RESOURCE_TYPES_3_1.equals(t.getVocabularyName())) + .findFirst(); - if (vocs.vocabularyExists(OPENAIRE_META_RESOURCE_TYPE)) { - Optional itm = instances - .stream() - .flatMap( - i -> Optional.ofNullable(i.getInstanceTypeMapping()).map(Collection::stream).orElse(Stream.empty())) - .filter(t -> OPENAIRE_COAR_RESOURCE_TYPES_3_1.equals(t.getVocabularyName())) - .findFirst(); - - if (!itm.isPresent() || Objects.isNull(itm.get().getTypeCode())) { - return null; - } else { - final String typeCode = itm.get().getTypeCode(); - return Optional - .ofNullable(vocs.lookupTermBySynonym(OPENAIRE_META_RESOURCE_TYPE, typeCode)) - .orElseThrow( - () -> new IllegalStateException("unable to find a synonym for '" + typeCode + "' in " + - OPENAIRE_META_RESOURCE_TYPE)); - } - } else { - throw new IllegalStateException("vocabulary '" + OPENAIRE_META_RESOURCE_TYPE + "' not available"); - } + if (!itm.isPresent() || Objects.isNull(itm.get().getTypeCode())) { + return null; + } else { + final String typeCode = itm.get().getTypeCode(); + return Optional + .ofNullable(vocs.lookupTermBySynonym(OPENAIRE_META_RESOURCE_TYPE, typeCode)) + .orElseThrow( + () -> new IllegalStateException("unable to find a synonym for '" + typeCode + "' in " + + OPENAIRE_META_RESOURCE_TYPE)); + } + } else { + throw new IllegalStateException("vocabulary '" + OPENAIRE_META_RESOURCE_TYPE + "' not available"); + } + }) + .orElse(null); } } From 3c3bdb83187d572c101447def9e5f75febe8efa0 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 5 Dec 2023 09:08:48 +0100 Subject: [PATCH 134/148] [bulktagging] fixed workflow parameters --- .../dnetlib/dhp/bulktag/SparkBulkTagJob.java | 2 +- .../dhp/bulktag/input_bulkTag_parameters.json | 5 ----- .../dhp/bulktag/oozie_app/workflow.xml | 21 ++++--------------- 3 files changed, 5 insertions(+), 23 deletions(-) diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java index f01063955..5d1b2b38d 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/bulktag/SparkBulkTagJob.java @@ -135,7 +135,7 @@ public class SparkBulkTagJob { ModelSupport.entityTypes .keySet() .parallelStream() - .filter(e -> ModelSupport.isResult(e)) + .filter(ModelSupport::isResult) .forEach(e -> { removeOutputDir(spark, outputPath + e.name()); ResultTagger resultTagger = new ResultTagger(); diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/input_bulkTag_parameters.json b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/input_bulkTag_parameters.json index d88904cd9..ce1a8ecab 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/input_bulkTag_parameters.json +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/input_bulkTag_parameters.json @@ -1,5 +1,4 @@ [ - { "paramName":"s", "paramLongName":"sourcePath", @@ -12,7 +11,6 @@ "paramDescription": "the json path associated to each selection field", "paramRequired": true }, - { "paramName": "out", "paramLongName": "outputPath", @@ -25,19 +23,16 @@ "paramDescription": "true if the spark session is managed, false otherwise", "paramRequired": false }, - { "paramName": "tg", "paramLongName": "taggingConf", "paramDescription": "this parameter is intended for testing purposes only. It is a possible tagging configuration obtained via the XQUERY. Intended to be removed", "paramRequired": false }, - { "paramName": "bu", "paramLongName": "baseURL", "paramDescription": "this parameter is to specify the api to be queried (beta or production)", "paramRequired": false } - ] \ No newline at end of file diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml index 56eed4703..0d4d1f046 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml @@ -12,21 +12,10 @@ outputPath the output path - - - postgresURL - the url of the postgress server to query + baseURL + the community API base URL - - postgresUser - the username to access the postgres db - - - postgresPassword - the postgres password - - @@ -104,7 +93,7 @@ yarn-cluster cluster - bulkTagging-publication + bulkTagging-result eu.dnetlib.dhp.bulktag.SparkBulkTagJob dhp-enrichment-${projectVersion}.jar @@ -120,14 +109,12 @@ --sourcePath${sourcePath}/ --outputPath${outputPath}/ --pathMap${pathMap} - --production${production} + --baseURL${baseURL} - - \ No newline at end of file From c5b72531301c4317ebc63261e1148b10c1c6f240 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 5 Dec 2023 09:13:33 +0100 Subject: [PATCH 135/148] [community_organization propagation] fixed workflow parameters --- .../oozie_app/workflow.xml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml index d5d75fbc1..dfa762ac6 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml @@ -4,11 +4,14 @@ sourcePath the source path - outputPath the output path + + baseURL + the community API base URL + @@ -99,13 +102,12 @@ --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.dynamicAllocation.maxExecutors=${spark2MaxExecutors} --sourcePath${sourcePath}/relation --outputPath${workingDir}/preparedInfo/resultCommunityList --hive_metastore_uris${hive_metastore_uris} - --production${production} + --baseURL${baseURL} @@ -115,7 +117,7 @@ yarn cluster - community2resultfromorganization-Publication + community2resultfromorganization eu.dnetlib.dhp.resulttocommunityfromorganization.SparkResultToCommunityFromOrganizationJob dhp-enrichment-${projectVersion}.jar From 321922772b71c70e074eb97efb46d920f1031752 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 5 Dec 2023 16:37:04 +0100 Subject: [PATCH 136/148] added serialization for the new fields imported for the Irish tender --- .../oa/provision/utils/XmlRecordFactory.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java index 3b810ba8c..4d9d9c341 100644 --- a/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java +++ b/dhp-workflows/dhp-graph-provision/src/main/java/eu/dnetlib/dhp/oa/provision/utils/XmlRecordFactory.java @@ -434,6 +434,24 @@ public class XmlRecordFactory implements Serializable { XmlSerializationUtils .asXmlElement("processingchargecurrency", r.getProcessingchargecurrency().getValue())); } + if (r.getIsGreen() != null) { + metadata + .add(XmlSerializationUtils.asXmlElement("isgreen", Boolean.toString(r.getIsGreen()))); + } + if (r.getOpenAccessColor() != null) { + metadata + .add(XmlSerializationUtils.asXmlElement("openaccesscolor", r.getOpenAccessColor().toString())); + } + if (r.getIsInDiamondJournal() != null) { + metadata + .add( + XmlSerializationUtils + .asXmlElement("isindiamondjournal", Boolean.toString(r.getIsInDiamondJournal()))); + } + if (r.getPubliclyFunded() != null) { + metadata + .add(XmlSerializationUtils.asXmlElement("publiclyfunded", Boolean.toString(r.getPubliclyFunded()))); + } } switch (type) { From 431c6bb08a2713df4cb1dbe0abeb60011c409349 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Wed, 6 Dec 2023 11:06:46 +0100 Subject: [PATCH 137/148] [dedup] added isLookupUrl to the graph consistency workflow definition, required now by the entity grouping phase --- .../dnetlib/dhp/oa/dedup/consistency/oozie_app/workflow.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/workflow.xml b/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/workflow.xml index 5e2fc0a01..306229e79 100644 --- a/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-dedup-openaire/src/main/resources/eu/dnetlib/dhp/oa/dedup/consistency/oozie_app/workflow.xml @@ -16,6 +16,10 @@ filterInvisible whether filter out invisible entities after merge + + isLookupUrl + the URL address of the lookUp service + sparkDriverMemory heap memory for driver process @@ -128,6 +132,7 @@ --graphInputPath${graphBasePath} --checkpointPath${workingPath}/grouped_entities --outputPath${graphOutputPath} + --isLookupUrl${isLookupUrl} --filterInvisible${filterInvisible} From 259c69e446c855a3f3da8cfec532f2ac0e2ff87e Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Wed, 6 Dec 2023 19:41:53 +0100 Subject: [PATCH 138/148] [orcid enrichment] fixed workflow definition --- .../dhp/enrich/orcid/oozie_app/workflow.xml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/enrich/orcid/oozie_app/workflow.xml b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/enrich/orcid/oozie_app/workflow.xml index a4330b966..ce117b5e9 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/enrich/orcid/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-graph-mapper/src/main/resources/eu/dnetlib/dhp/enrich/orcid/oozie_app/workflow.xml @@ -38,16 +38,12 @@ --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --orcidPath - ${orcidPath} - --targetPath - ${targetPath} - --graphPath - ${graphPath} - --master - yarn + --orcidPath${orcidPath} + --targetPath${targetPath} + --graphPath${graphPath} + --masteryarn - + From c0cde53bf647d00bf5d633ef0ef906b4984d35b5 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 7 Dec 2023 10:08:35 +0100 Subject: [PATCH 139/148] [bulktagging] setting first step of bulktaggin as the copy of the entities and relations not involved in the tagging' --- .../resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml index 0d4d1f046..03373eda0 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/bulktag/oozie_app/workflow.xml @@ -29,7 +29,7 @@ - + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] From 336fb31d8778cbfbb75a555165b55ee48518a9ba Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Thu, 7 Dec 2023 10:27:25 +0100 Subject: [PATCH 140/148] [community_result_propagation] adjusting starting poit of workflow --- .../resulttocommunityfromorganization/oozie_app/workflow.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml index dfa762ac6..e342bce23 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromorganization/oozie_app/workflow.xml @@ -25,7 +25,7 @@ - + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] From c381bacee01b7f5a3918e0e77d4a55d8e969c04b Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Thu, 7 Dec 2023 14:07:11 +0100 Subject: [PATCH 141/148] [enrichment] passing the community API base URL --- .../resulttocommunityfromproject/oozie_app/workflow.xml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/oozie_app/workflow.xml b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/oozie_app/workflow.xml index 21cc2d887..d0784c94d 100644 --- a/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-enrichment/src/main/resources/eu/dnetlib/dhp/resulttocommunityfromproject/oozie_app/workflow.xml @@ -4,11 +4,14 @@ sourcePath the source path - outputPath the output path + + baseURL + the community API base URL + @@ -104,7 +107,7 @@ --sourcePath${sourcePath}/relation --outputPath${workingDir}/preparedInfo/resultCommunityList - --production${production} + --baseURL${baseURL} From 70eb1796b20c1001d4c49655e72dbbe0c791252a Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Thu, 7 Dec 2023 14:08:04 +0100 Subject: [PATCH 142/148] logging typo --- .../resulttocommunityfromproject/PrepareResultCommunitySet.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java index 7fed2606b..467e11a96 100644 --- a/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java +++ b/dhp-workflows/dhp-enrichment/src/main/java/eu/dnetlib/dhp/resulttocommunityfromproject/PrepareResultCommunitySet.java @@ -53,7 +53,7 @@ public class PrepareResultCommunitySet { log.info("outputPath: {}", outputPath); final String baseURL = parser.get("baseURL"); - log.info("baseUEL: {}", baseURL); + log.info("baseURL: {}", baseURL); final CommunityEntityMap projectsMap = Utils.getCommunityProjects(baseURL); From cb71a7936bb98f498693fe75335c60947f5cbef6 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Thu, 7 Dec 2023 23:09:54 +0100 Subject: [PATCH 143/148] [graph cleaning] avoid stack overflow error when navigating Oaf objects declaring an Enum --- .../dhp/oa/graph/clean/OafCleaner.java | 1 + .../graph/clean/CleanGraphSparkJobTest.java | 32 +++++++++++++++++++ .../dhp/oa/graph/clean/graph/orp/orp.json | 1 + 3 files changed, 34 insertions(+) create mode 100644 dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/graph/orp/orp.json diff --git a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafCleaner.java b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafCleaner.java index 102a1fa85..4cfec6fe7 100644 --- a/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafCleaner.java +++ b/dhp-workflows/dhp-graph-mapper/src/main/java/eu/dnetlib/dhp/oa/graph/clean/OafCleaner.java @@ -59,6 +59,7 @@ public class OafCleaner implements Serializable { private static boolean isPrimitive(Object o) { return Objects.isNull(o) || o.getClass().isPrimitive() + || o.getClass().isEnum() || o instanceof Class || o instanceof Integer || o instanceof Double diff --git a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleanGraphSparkJobTest.java b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleanGraphSparkJobTest.java index a1a4c29cd..4ae3f82c2 100644 --- a/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleanGraphSparkJobTest.java +++ b/dhp-workflows/dhp-graph-mapper/src/test/java/eu/dnetlib/dhp/oa/graph/clean/CleanGraphSparkJobTest.java @@ -690,6 +690,38 @@ public class CleanGraphSparkJobTest { } + @Test + void testClean_ORP() throws Exception { + final String prefix = "gcube "; + + new CleanGraphSparkJob( + args( + "/eu/dnetlib/dhp/oa/graph/input_clean_graph_parameters.json", + new String[] { + "--inputPath", graphInputPath + "/orp", + "--outputPath", graphOutputPath + "/orp", + "--isLookupUrl", "lookupurl", + "--graphTableClassName", OtherResearchProduct.class.getCanonicalName(), + "--deepClean", "true", + "--contextId", "sobigdata", + "--verifyParam", "gCube ", + "--masterDuplicatePath", dsMasterDuplicatePath, + "--country", "NL", + "--verifyCountryParam", "10.17632", + "--collectedfrom", "NARCIS", + "--hostedBy", Objects + .requireNonNull( + getClass() + .getResource("/eu/dnetlib/dhp/oa/graph/clean/hostedBy")) + .getPath() + })).run(false, isLookUpService); + + Dataset orp = read(spark, graphOutputPath + "/orp", OtherResearchProduct.class); + + assertEquals(1, orp.count()); + + } + @Test void testCleanCfHbSparkJob() throws Exception { diff --git a/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/graph/orp/orp.json b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/graph/orp/orp.json new file mode 100644 index 000000000..85045e361 --- /dev/null +++ b/dhp-workflows/dhp-graph-mapper/src/test/resources/eu/dnetlib/dhp/oa/graph/clean/graph/orp/orp.json @@ -0,0 +1 @@ +{"collectedfrom":[{"key":"10|openaire____::fffd45256148b1de3114788f21179083","value":"JAIRO","dataInfo":null}],"dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"0.8","inferenceprovenance":"dedup-result-decisiontree-v3","provenanceaction":{"classid":"sysimport:dedup","classname":"sysimport:dedup","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}},"lastupdatetimestamp":1701811731058,"id":"50|dedup_wf_001::5ff833ef4a38d08e840935d5c52fc479","originalId":["oai:irdb.nii.ac.jp:01292:0003046643","50|jairo_______::5ff833ef4a38d08e840935d5c52fc479","oai:irdb.nii.ac.jp:0108/00008334","50|jairo_______::f542b980618630c53c3f8052588dca25"],"pid":[],"dateofcollection":"2023-07-12T02:35:49.772Z","dateoftransformation":"2023-07-12T13:51:55.356Z","extraInfo":[],"oaiprovenance":{"originDescription":{"harvestDate":"2023-07-12T02:35:49.772Z","altered":true,"baseURL":"https%3A%2F%2Firdb.nii.ac.jp%2Foai","identifier":"oai:irdb.nii.ac.jp:01292:0003046643","datestamp":"2023-07-08T10:54:57Z","metadataNamespace":""}},"measures":[{"id":"influence","unit":[{"key":"score","value":"3.244804E-9","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:bip","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"key":"class","value":"C5","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:bip","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}]},{"id":"popularity","unit":[{"key":"score","value":"6.0055183E-10","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:bip","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"key":"class","value":"C5","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:bip","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}]},{"id":"influence_alt","unit":[{"key":"score","value":"0","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:bip","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"key":"class","value":"C5","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:bip","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}]},{"id":"popularity_alt","unit":[{"key":"score","value":"0.0","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:bip","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"key":"class","value":"C5","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:bip","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}]},{"id":"impulse","unit":[{"key":"score","value":"0","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:bip","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"key":"class","value":"C5","dataInfo":{"invisible":false,"inferred":true,"deletedbyinference":false,"trust":"","inferenceprovenance":"update","provenanceaction":{"classid":"measure:bip","classname":"Inferred by OpenAIRE","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}]}],"processingchargeamount":null,"processingchargecurrency":null,"author":[{"fullname":"Shimazaki, Kazushi","name":"Kazushi","surname":"Shimazaki","rank":2,"pid":[],"affiliation":null}],"resulttype":{"classid":"other","classname":"other","schemeid":"dnet:result_typologies","schemename":"dnet:result_typologies"},"metaResourceType":null,"language":{"classid":"jpn","classname":"Japanese","schemeid":"dnet:languages","schemename":"dnet:languages"},"country":[],"subject":[{"value":"Reinforced concrete structure","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"damage level","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"柱","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"shear span ratio","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"損傷レベル","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"column","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"鉄筋コンクリート構造","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"残存軸耐力","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Residual Axis Load Capacity","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"せん断スパン比","qualifier":{"classid":"keyword","classname":"keyword","schemeid":"dnet:subject_classification_typologies","schemename":"dnet:subject_classification_typologies"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"title":[{"value":"Eximental study odual axis loadity of reinforced concrete column","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"RC造柱の地震後の残存軸耐力に関する研究","qualifier":{"classid":"main title","classname":"main title","schemeid":"dnet:dataCite_title","schemename":"dnet:dataCite_title"},"dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"relevantdate":[],"description":[{"value":"Shear failure of columns may lead to collapse for existence buildings．After a severe earthquake，some buildings collapsed by loss of axial capacity of the columns，however， some buildings have stayed even many columns were fractured by shear．For the examinaton of building collapse，it is required to investigate the axial load capacity of the columns after shear failure and to analyze the building performance using the pertinent analytical model for the columns．This paper investigates the analytical model for the collapse analysis of a building based on Yoshimura’s concept of the failure surface contraction． The reduction rate was obtained as a function of the maximum deformation，shear margin ratio，and so on，using the regression analydid of experimental results． The tested results are explained by this formula with good agreement．","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"Article","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"研究報告書ページ(113)-(118)","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"学術フロンティア研究プロジェクト(2005年度～2009年度)　2008年度　研究成果報告書","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"dateofacceptance":{"value":"2008-03-31","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"publisher":{"value":"神奈川大学","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"embargoenddate":null,"source":[],"fulltext":[{"value":"https://kanagawa-u.repo.nii.ac.jp/record/4315/files/2008-28.pdf","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},{"value":"http://klibredb.lib.kanagawa-u.ac.jp/dspace/bitstream/10487/10662/1/2008-28.pdf","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"format":[{"value":"application/pdf","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}}],"contributor":[],"resourcetype":{"classid":"UNKNOWN","classname":"Unknown","schemeid":"dnet:dataCite_resource","schemename":"dnet:dataCite_resource"},"coverage":[],"bestaccessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes"},"context":[],"externalReference":[],"instance":[{"license":null,"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes","openAccessRoute":null},"instancetype":{"classid":"0000","classname":"Unknown","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"instanceTypeMapping":[{"originalType":"Research Paper","typeCode":null,"typeLabel":null,"vocabularyName":"openaire::coar_resource_types_3_1"}],"hostedby":{"key":"10|openaire____::fffd45256148b1de3114788f21179083","value":"JAIRO","dataInfo":null},"url":["https://kanagawa-u.repo.nii.ac.jp/records/4315"],"distributionlocation":"","collectedfrom":{"key":"10|openaire____::fffd45256148b1de3114788f21179083","value":"JAIRO","dataInfo":null},"pid":[],"alternateIdentifier":[],"dateofacceptance":{"value":"2008-03-31","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":{"classid":"0002","classname":"nonPeerReviewed","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"measures":null,"fulltext":"https://kanagawa-u.repo.nii.ac.jp/record/4315/files/2008-28.pdf"},{"license":null,"accessright":{"classid":"OPEN","classname":"Open Access","schemeid":"dnet:access_modes","schemename":"dnet:access_modes","openAccessRoute":null},"instancetype":{"classid":"0000","classname":"Unknown","schemeid":"dnet:publication_resource","schemename":"dnet:publication_resource"},"instanceTypeMapping":[{"originalType":"Article","typeCode":"http://purl.org/coar/resource_type/c_2df8fbb1","typeLabel":"research article","vocabularyName":"openaire::coar_resource_types_3_1"},{"originalType":"http://purl.org/coar/resource_type/c_2df8fbb1","typeCode":"Article","typeLabel":"Article","vocabularyName":"openaire::user_resource_types"}],"hostedby":{"key":"10|openaire____::fffd45256148b1de3114788f21179083","value":"JAIRO","dataInfo":null},"url":["http://hdl.handle.net/10487/10662"],"distributionlocation":"","collectedfrom":{"key":"10|openaire____::fffd45256148b1de3114788f21179083","value":"JAIRO","dataInfo":null},"pid":[],"alternateIdentifier":[],"dateofacceptance":{"value":"2008-03-31","dataInfo":{"invisible":false,"inferred":false,"deletedbyinference":false,"trust":"0.9","inferenceprovenance":"","provenanceaction":{"classid":"sysimport:crosswalk:repository","classname":"Harvested","schemeid":"dnet:provenanceActions","schemename":"dnet:provenanceActions"}}},"processingchargeamount":null,"processingchargecurrency":null,"refereed":{"classid":"0002","classname":"nonPeerReviewed","schemeid":"dnet:review_levels","schemename":"dnet:review_levels"},"measures":null,"fulltext":"http://klibredb.lib.kanagawa-u.ac.jp/dspace/bitstream/10487/10662/1/2008-28.pdf"}],"eoscifguidelines":[],"openAccessColor":"bronze","publiclyFunded":false,"contactperson":[],"contactgroup":[],"tool":[],"isGreen":true,"isInDiamondJournal":false} \ No newline at end of file From aba95ed1d17a1683729ebddf2afff785bcb5e138 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Fri, 8 Dec 2023 17:06:19 +0100 Subject: [PATCH 144/148] code formatting --- .../oaf/utils/GraphCleaningFunctions.java | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java index 2c1bc48e3..e6ae37908 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java @@ -16,7 +16,6 @@ import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; -import eu.dnetlib.dhp.common.vocabulary.VocabularyTerm; import org.apache.commons.lang3.StringUtils; import com.github.sisyphsu.dateparser.DateParserUtils; @@ -24,6 +23,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup; +import eu.dnetlib.dhp.common.vocabulary.VocabularyTerm; import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.common.ModelSupport; import eu.dnetlib.dhp.schema.oaf.*; @@ -416,10 +416,11 @@ public class GraphCleaningFunctions extends CleaningFunctions { .replaceAll(NAME_CLEANING_REGEX, " ")); if (vocs.vocabularyExists(DNET_PUBLISHERS)) { - vocs.find(DNET_PUBLISHERS) - .map(voc -> voc.getTermBySynonym(r.getPublisher().getValue())) - .map(VocabularyTerm::getName) - .ifPresent(publisher -> r.getPublisher().setValue(publisher)); + vocs + .find(DNET_PUBLISHERS) + .map(voc -> voc.getTermBySynonym(r.getPublisher().getValue())) + .map(VocabularyTerm::getName) + .ifPresent(publisher -> r.getPublisher().setValue(publisher)); } } } @@ -582,10 +583,11 @@ public class GraphCleaningFunctions extends CleaningFunctions { } if (Objects.nonNull(i.getLicense()) && Objects.nonNull(i.getLicense().getValue())) { - vocs.find(DNET_LICENSES) - .map(voc -> voc.getTermBySynonym(i.getLicense().getValue())) - .map(VocabularyTerm::getId) - .ifPresent(license -> i.getLicense().setValue(license)); + vocs + .find(DNET_LICENSES) + .map(voc -> voc.getTermBySynonym(i.getLicense().getValue())) + .map(VocabularyTerm::getId) + .ifPresent(license -> i.getLicense().setValue(license)); } // from the script from Dimitris From 7e8eff40c1660d5d277fdc36ff87d2e264e6549d Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 12 Dec 2023 08:54:15 +0100 Subject: [PATCH 145/148] [graph provision] added tests for the new model fields --- .../dhp/oa/provision/XmlIndexingJobTest.java | 34 ++- .../oa/provision/XmlRecordFactoryTest.java | 8 +- .../eu/dnetlib/dhp/oa/provision/fields.xml | 242 ++++++++++-------- .../dnetlib/dhp/oa/provision/publication.json | 10 +- .../conf/exploreTestConfig/managed-schema | 18 +- .../solr/conf/testConfig/managed-schema | 6 + 6 files changed, 183 insertions(+), 135 deletions(-) diff --git a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlIndexingJobTest.java b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlIndexingJobTest.java index 6f1956578..47f0ec8c3 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlIndexingJobTest.java +++ b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlIndexingJobTest.java @@ -4,6 +4,7 @@ package eu.dnetlib.dhp.oa.provision; import java.io.IOException; import java.io.StringReader; import java.net.URI; +import java.util.Map; import org.apache.commons.io.IOUtils; import org.apache.hadoop.io.Text; @@ -11,6 +12,7 @@ import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrInputField; import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaSparkContext; @@ -30,6 +32,8 @@ import eu.dnetlib.dhp.oa.provision.utils.ISLookupClient; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService; +import static org.junit.jupiter.api.Assertions.assertEquals; + @ExtendWith(MockitoExtension.class) public class XmlIndexingJobTest extends SolrTest { @@ -101,14 +105,34 @@ public class XmlIndexingJobTest extends SolrTest { new XmlIndexingJob(spark, inputPath, FORMAT, batchSize, XmlIndexingJob.OutputFormat.SOLR, null) .run(isLookupClient); - Assertions.assertEquals(0, miniCluster.getSolrClient().commit().getStatus()); + assertEquals(0, miniCluster.getSolrClient().commit().getStatus()); QueryResponse rsp = miniCluster.getSolrClient().query(new SolrQuery().add(CommonParams.Q, "*:*")); - Assertions - .assertEquals( + assertEquals( nRecord, rsp.getResults().getNumFound(), "the number of indexed records should be equal to the number of input records"); + + + rsp = miniCluster.getSolrClient().query(new SolrQuery().add(CommonParams.Q, "isgreen:true")); + assertEquals( + 0, rsp.getResults().getNumFound(), + "the number of indexed records having isgreen = true"); + + rsp = miniCluster.getSolrClient().query(new SolrQuery().add(CommonParams.Q, "openaccesscolor:bronze")); + assertEquals( + 0, rsp.getResults().getNumFound(), + "the number of indexed records having openaccesscolor = bronze"); + + rsp = miniCluster.getSolrClient().query(new SolrQuery().add(CommonParams.Q, "isindiamondjournal:true")); + assertEquals( + 0, rsp.getResults().getNumFound(), + "the number of indexed records having isindiamondjournal = true"); + + rsp = miniCluster.getSolrClient().query(new SolrQuery().add(CommonParams.Q, "publiclyfunded:true")); + assertEquals( + 0, rsp.getResults().getNumFound(), + "the number of indexed records having publiclyfunded = publiclyfunded"); } @Test @@ -126,7 +150,7 @@ public class XmlIndexingJobTest extends SolrTest { .map(s -> new SAXReader().read(new StringReader(s)).valueOf(ID_XPATH)) .distinct() .count(); - Assertions.assertEquals(nRecord, xmlIdUnique, "IDs should be unique among input records"); + assertEquals(nRecord, xmlIdUnique, "IDs should be unique among input records"); final String outputPath = workingDir.resolve("outputPath").toAbsolutePath().toString(); new XmlIndexingJob(spark, inputPath, FORMAT, batchSize, XmlIndexingJob.OutputFormat.HDFS, outputPath) @@ -142,7 +166,7 @@ public class XmlIndexingJobTest extends SolrTest { }, Encoders.STRING()) .distinct() .count(); - Assertions.assertEquals(xmlIdUnique, docIdUnique, "IDs should be unique among the output records"); + assertEquals(xmlIdUnique, docIdUnique, "IDs should be unique among the output records"); } diff --git a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java index 88bffd0e7..097d36d05 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java +++ b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java @@ -32,7 +32,7 @@ public class XmlRecordFactoryTest { .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); @Test - public void testXMLRecordFactory() throws IOException, DocumentException { + void testXMLRecordFactory() throws IOException, DocumentException { final ContextMapper contextMapper = new ContextMapper(); @@ -93,6 +93,12 @@ public class XmlRecordFactoryTest { "https://osf.io/preprints/socarxiv/7vgtu/download", doc.valueOf("//*[local-name() = 'result']/fulltext[1]")); + assertEquals("true", doc.valueOf("//*[local-name() = 'result']/isgreen/text()")); + assertEquals("bronze", doc.valueOf("//*[local-name() = 'result']/openaccesscolor/text()")); + assertEquals("true", doc.valueOf("//*[local-name() = 'result']/isindiamondjournal/text()")); + assertEquals("true", doc.valueOf("//*[local-name() = 'result']/publiclyfunded/text()")); + + System.out.println(doc.asXML()); } @Test diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/fields.xml b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/fields.xml index 0bf588a57..513ead1dd 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/fields.xml +++ b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/fields.xml @@ -1,116 +1,134 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/publication.json b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/publication.json index 4ba20292c..a89ec62d5 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/publication.json +++ b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/publication.json @@ -1048,8 +1048,8 @@ "schemename": "dnet:dataCite_resource" }, "refereed": { - "classid": "0001", - "classname": "peerReviewed", + "classid": "0002", + "classname": "nonPeerReviewed", "schemeid": "dnet:review_levels", "schemename": "dnet:review_levels" }, @@ -1949,5 +1949,9 @@ "schemename": "dnet:provenanceActions" } } - } + }, + "isGreen": true, + "openAccessColor": "bronze", + "isInDiamondJournal": true, + "publiclyFunded": true } \ No newline at end of file diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/exploreTestConfig/managed-schema b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/exploreTestConfig/managed-schema index 39c811f83..28a439619 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/exploreTestConfig/managed-schema +++ b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/exploreTestConfig/managed-schema @@ -219,20 +219,6 @@ - - - - - - - - - - - - - - @@ -368,6 +354,10 @@ + + + + diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/testConfig/managed-schema b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/testConfig/managed-schema index 977e0b2d7..962e0cc6f 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/testConfig/managed-schema +++ b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/testConfig/managed-schema @@ -365,7 +365,13 @@ + + + + + + From 84d54643cf55d86d42cea236569c84a1b2c339bc Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 12 Dec 2023 09:57:00 +0100 Subject: [PATCH 146/148] [cleaning] allow enriched orcids to pass the cleaning, rule out non-orcid author pids --- .../dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java index e6ae37908..8700ea527 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java @@ -689,6 +689,7 @@ public class GraphCleaningFunctions extends CleaningFunctions { .filter(Objects::nonNull) .filter(p -> Objects.nonNull(p.getQualifier())) .filter(p -> StringUtils.isNotBlank(p.getValue())) + .filter(p -> StringUtils.contains(StringUtils.lowerCase(p.getQualifier().getClassid()), ORCID)) .map(p -> { // hack to distinguish orcid from orcid_pending String pidProvenance = getProvenance(p.getDataInfo()); @@ -698,7 +699,8 @@ public class GraphCleaningFunctions extends CleaningFunctions { .toLowerCase() .contains(ModelConstants.ORCID)) { if (pidProvenance - .equals(ModelConstants.SYSIMPORT_CROSSWALK_ENTITYREGISTRY)) { + .equals(ModelConstants.SYSIMPORT_CROSSWALK_ENTITYREGISTRY) || + pidProvenance.equals("ORCID_ENRICHMENT")) { p.getQualifier().setClassid(ModelConstants.ORCID); } else { p.getQualifier().setClassid(ModelConstants.ORCID_PENDING); From 98cce5bfb22c5d761fbf7044834959efdec4e31e Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 12 Dec 2023 09:59:05 +0100 Subject: [PATCH 147/148] code formatting --- .../dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java index 8700ea527..0124e96fc 100644 --- a/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java +++ b/dhp-common/src/main/java/eu/dnetlib/dhp/schema/oaf/utils/GraphCleaningFunctions.java @@ -689,7 +689,9 @@ public class GraphCleaningFunctions extends CleaningFunctions { .filter(Objects::nonNull) .filter(p -> Objects.nonNull(p.getQualifier())) .filter(p -> StringUtils.isNotBlank(p.getValue())) - .filter(p -> StringUtils.contains(StringUtils.lowerCase(p.getQualifier().getClassid()), ORCID)) + .filter( + p -> StringUtils + .contains(StringUtils.lowerCase(p.getQualifier().getClassid()), ORCID)) .map(p -> { // hack to distinguish orcid from orcid_pending String pidProvenance = getProvenance(p.getDataInfo()); From ff924215b8e457b3e05e98792db0c6b3bdd0fa0d Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 12 Dec 2023 11:21:30 +0100 Subject: [PATCH 148/148] [graph provision] added tests for new peerreviewed field --- .../provision/IndexRecordTransformerTest.java | 20 ++++++++++++++++++- .../dhp/oa/provision/XmlIndexingJobTest.java | 7 ++++++- .../oa/provision/XmlRecordFactoryTest.java | 2 -- .../eu/dnetlib/dhp/oa/provision/fields.xml | 3 +++ .../conf/exploreTestConfig/managed-schema | 1 + .../solr/conf/testConfig/managed-schema | 1 + 6 files changed, 30 insertions(+), 4 deletions(-) diff --git a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/IndexRecordTransformerTest.java b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/IndexRecordTransformerTest.java index ce593cf07..e07ba1b4e 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/IndexRecordTransformerTest.java +++ b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/IndexRecordTransformerTest.java @@ -78,6 +78,22 @@ public class IndexRecordTransformerTest { testRecordTransformation(record); } + @Test + void testPeerReviewed() throws IOException, TransformerException { + + final XmlRecordFactory xmlRecordFactory = new XmlRecordFactory(contextMapper, false, + XmlConverterJob.schemaLocation); + + final Publication p = load("publication.json", Publication.class); + + final JoinedEntity je = new JoinedEntity<>(p); + final String record = xmlRecordFactory.build(je); + assertNotNull(record); + SolrInputDocument solrDoc = testRecordTransformation(record); + + assertEquals("true", solrDoc.get("peerreviewed").getValue()); + } + @Test public void testRiunet() throws IOException, TransformerException { @@ -184,7 +200,7 @@ public class IndexRecordTransformerTest { } } - private void testRecordTransformation(final String record) throws IOException, TransformerException { + private SolrInputDocument testRecordTransformation(final String record) throws IOException, TransformerException { final String fields = IOUtils.toString(getClass().getResourceAsStream("fields.xml")); final String xslt = IOUtils.toString(getClass().getResourceAsStream("layoutToRecordTransformer.xsl")); @@ -200,6 +216,8 @@ public class IndexRecordTransformerTest { Assertions.assertNotNull(xmlDoc); System.out.println(xmlDoc); + + return solrDoc; } private T load(final String fileName, final Class clazz) throws IOException { diff --git a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlIndexingJobTest.java b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlIndexingJobTest.java index 47f0ec8c3..b62acbac3 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlIndexingJobTest.java +++ b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlIndexingJobTest.java @@ -132,7 +132,12 @@ public class XmlIndexingJobTest extends SolrTest { rsp = miniCluster.getSolrClient().query(new SolrQuery().add(CommonParams.Q, "publiclyfunded:true")); assertEquals( 0, rsp.getResults().getNumFound(), - "the number of indexed records having publiclyfunded = publiclyfunded"); + "the number of indexed records having publiclyfunded = true"); + + rsp = miniCluster.getSolrClient().query(new SolrQuery().add(CommonParams.Q, "peerreviewed:true")); + assertEquals( + 0, rsp.getResults().getNumFound(), + "the number of indexed records having peerreviewed = true"); } @Test diff --git a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java index 097d36d05..de69795f8 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java +++ b/dhp-workflows/dhp-graph-provision/src/test/java/eu/dnetlib/dhp/oa/provision/XmlRecordFactoryTest.java @@ -97,8 +97,6 @@ public class XmlRecordFactoryTest { assertEquals("bronze", doc.valueOf("//*[local-name() = 'result']/openaccesscolor/text()")); assertEquals("true", doc.valueOf("//*[local-name() = 'result']/isindiamondjournal/text()")); assertEquals("true", doc.valueOf("//*[local-name() = 'result']/publiclyfunded/text()")); - - System.out.println(doc.asXML()); } @Test diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/fields.xml b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/fields.xml index 513ead1dd..ba3fa83df 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/fields.xml +++ b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/fields.xml @@ -74,6 +74,9 @@ + + + diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/exploreTestConfig/managed-schema b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/exploreTestConfig/managed-schema index 28a439619..4e85ca3be 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/exploreTestConfig/managed-schema +++ b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/exploreTestConfig/managed-schema @@ -358,6 +358,7 @@ + diff --git a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/testConfig/managed-schema b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/testConfig/managed-schema index 962e0cc6f..e191c6223 100644 --- a/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/testConfig/managed-schema +++ b/dhp-workflows/dhp-graph-provision/src/test/resources/eu/dnetlib/dhp/oa/provision/solr/conf/testConfig/managed-schema @@ -371,6 +371,7 @@ +