From 7a8e79eff8b4b64eb27b6c39a4ba5a94a880980b Mon Sep 17 00:00:00 2001 From: "fabio.sinibaldi" Date: Tue, 3 Dec 2019 15:58:35 +0100 Subject: [PATCH] more tests --- .../publishing/ckan2zenodo/clients/GCat.java | 4 +- .../ckan2zenodo/model/CkanItemDescriptor.java | 8 +- .../publishing/ckan2zenodo/model/Mapping.java | 6 +- .../org/gcube/tests/TransformationTests.java | 37 +-- .../src/test/resources/FSKXModel.json | 269 ++++++++++++++++++ .../src/test/resources/ResearchObject.json | 127 +++++++++ 6 files changed, 428 insertions(+), 23 deletions(-) create mode 100644 ckan2zenodo-library/src/test/resources/FSKXModel.json create mode 100644 ckan2zenodo-library/src/test/resources/ResearchObject.json diff --git a/ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/clients/GCat.java b/ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/clients/GCat.java index 35811a7..87a4c3c 100644 --- a/ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/clients/GCat.java +++ b/ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/clients/GCat.java @@ -9,11 +9,11 @@ public class GCat { public static CkanItemDescriptor getByID(String itemName) throws MalformedURLException { - return new CkanItemDescriptor(itemName,new Item().read(itemName)); + return new CkanItemDescriptor(new Item().read(itemName)); } public static void updateItem(CkanItemDescriptor toUpdate) throws MalformedURLException { - new Item().update(toUpdate.getItemName(), toUpdate.getContent()); + new Item().update(toUpdate.getName(), toUpdate.getContent()); } } diff --git a/ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/CkanItemDescriptor.java b/ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/CkanItemDescriptor.java index d2b101f..19e600d 100644 --- a/ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/CkanItemDescriptor.java +++ b/ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/CkanItemDescriptor.java @@ -29,10 +29,8 @@ public class CkanItemDescriptor { private static final String AUTHOR="$.author"; private static final String MAINTAINER="$.maintainer"; private static final String ITEM_URL="$.extras[?(@.key=='Item URL')].value"; + private static final String NAME="$.name"; - @NonNull - @Getter - private String itemName; @NonNull @Getter private String content; @@ -53,6 +51,10 @@ public class CkanItemDescriptor { return values.get(0); } + public String getName() { + return getDocument().read(NAME); + } + public String getTitle() { return getDocument().read(TITLE); } diff --git a/ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/Mapping.java b/ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/Mapping.java index 798f83f..4263595 100644 --- a/ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/Mapping.java +++ b/ckan2zenodo-library/src/main/java/org/gcube/data/publishing/ckan2zenodo/model/Mapping.java @@ -5,9 +5,11 @@ import java.util.HashMap; import lombok.Getter; import lombok.NonNull; import lombok.RequiredArgsConstructor; +import lombok.Setter; @RequiredArgsConstructor @Getter +@Setter public class Mapping { @NonNull @@ -16,5 +18,7 @@ public class Mapping { private String target; @NonNull private HashMap valueMapping; - + + private String regexp; + } diff --git a/ckan2zenodo-library/src/test/java/org/gcube/tests/TransformationTests.java b/ckan2zenodo-library/src/test/java/org/gcube/tests/TransformationTests.java index 433a47b..d1ffa05 100644 --- a/ckan2zenodo-library/src/test/java/org/gcube/tests/TransformationTests.java +++ b/ckan2zenodo-library/src/test/java/org/gcube/tests/TransformationTests.java @@ -2,15 +2,16 @@ package org.gcube.tests; import java.io.IOException; import java.util.Collections; +import java.util.List; import org.gcube.data.publishing.ckan2zenodo.Fixer; import org.gcube.data.publishing.ckan2zenodo.Transformer; import org.gcube.data.publishing.ckan2zenodo.model.CkanItemDescriptor; +import org.gcube.data.publishing.ckan2zenodo.model.Mapping; import org.gcube.data.publishing.ckan2zenodo.model.zenodo.ZenodoDeposition; import org.junit.BeforeClass; import org.junit.Test; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; public class TransformationTests { @@ -24,17 +25,11 @@ public class TransformationTests { @Test - public void transform() throws IOException { - - String json=TestCommons.convertStreamToString(this.getClass().getResourceAsStream("/simpleItem.json")); - Transformer defaultTransformer=new Transformer(Collections.EMPTY_LIST); - CkanItemDescriptor desc=new CkanItemDescriptor("sampleontable_in_prevre", json); - System.out.println("Going to transform : "+desc.getContent()); - System.out.println("Result : "); - ZenodoDeposition dep=defaultTransformer.transform(desc, null); - System.out.println(dep); - System.out.println("As JSON : "); - System.out.println(Fixer.fixSending(mapper.writeValueAsString(dep))); + public void transform() throws Exception { + Transformer basic=new Transformer(Collections.EMPTY_LIST); + readAndTransform("/simpleItem.json",basic); + readAndTransform("/FSKXModel.json",basic); + readAndTransform("/ResearchObject.json",basic); } @@ -42,15 +37,23 @@ public class TransformationTests { @Test public void transformWithMappings() throws IOException { - String json=TestCommons.convertStreamToString(this.getClass().getResourceAsStream("/simpleItem.json")); - Transformer defaultTransformer=new Transformer(Collections.EMPTY_LIST); - CkanItemDescriptor desc=new CkanItemDescriptor("sampleontable_in_prevre", json); + + + } + + private static final void readAndTransform(String jsonFile, Transformer transformer) throws Exception { + try{ + String json=TestCommons.convertStreamToString(TransformationTests.class.getResourceAsStream(jsonFile)); + CkanItemDescriptor desc=new CkanItemDescriptor(json); System.out.println("Going to transform : "+desc.getContent()); System.out.println("Result : "); - ZenodoDeposition dep=defaultTransformer.transform(desc, null); + ZenodoDeposition dep=transformer.transform(desc, null); System.out.println(dep); System.out.println("As JSON : "); System.out.println(Fixer.fixSending(mapper.writeValueAsString(dep))); - + }catch(Throwable t) { + throw new Exception("Errors with json file "+jsonFile,t); + } } + } diff --git a/ckan2zenodo-library/src/test/resources/FSKXModel.json b/ckan2zenodo-library/src/test/resources/FSKXModel.json new file mode 100644 index 0000000..c9e959b --- /dev/null +++ b/ckan2zenodo-library/src/test/resources/FSKXModel.json @@ -0,0 +1,269 @@ +{ + "rating": 0.0, + "license_title": "Creative Commons Attribution-NonCommercial 4.0", + "maintainer": "", + "relationships_as_object": [], + "private": true, + "maintainer_email": "", + "num_tags": 11, + "id": "38bec1bc-0743-4cb1-bc8c-fea2fa314bcd", + "metadata_created": "2019-09-13T10:43:02.056378", + "metadata_modified": "2019-09-26T14:11:16.537679", + "author": "thomas_schueler", + "author_email": "thomas.schueler@bfr.bund.de", + "acquire_url": "", + "state": "active", + "version": "", + "license_id": "Creative Commons Attribution-NonCommercial 4.0", + "type": "dataset", + "resources": [ + { + "mimetype": null, + "cache_url": null, + "hash": "", + "description": "execute model in VRE", + "name": "execute immediately", + "format": "URL", + "url": "http://data.d4science.org/analytics/get/AGINFRAplusDev?dim=https://data.d4science.org/shub/E_eTBnYkN2YVIxWExlc1FGbWMrWldWVjZ3QnlMTmVuVVVwNUkvTDQzWW50TU55eXdqMmdWMHdHakpMVnB3R2QyMA==", + "datastore_active": false, + "cache_last_updated": null, + "package_id": "38bec1bc-0743-4cb1-bc8c-fea2fa314bcd", + "created": "2019-09-13T10:43:08.790133", + "state": "active", + "mimetype_inner": null, + "last_modified": null, + "position": 0, + "revision_id": "2a91358d-3c59-4196-9254-b039df38fd56", + "url_type": null, + "id": "9ae33ff9-bebb-49f3-bb70-c0dd02304c7d", + "resource_type": null, + "size": null + }, + { + "mimetype": null, + "cache_url": null, + "hash": "", + "description": "Change simulation parameters before running the model", + "name": "execute with new simulation parameters", + "format": "URL", + "url": "http://data.d4science.org/knime/get/AGINFRAplusDev?pm:file_ID=copy_url_of_file_here", + "datastore_active": false, + "cache_last_updated": null, + "package_id": "38bec1bc-0743-4cb1-bc8c-fea2fa314bcd", + "created": "2019-09-13T10:43:09.248507", + "state": "active", + "mimetype_inner": null, + "last_modified": null, + "position": 1, + "revision_id": "2a91358d-3c59-4196-9254-b039df38fd56", + "url_type": null, + "id": "42b78314-a0a9-42d9-8214-088ae45d65d2", + "resource_type": null, + "size": null + }], + "num_resources": 2, + "tags": [ + { + "vocabulary_id": null, + "state": "active", + "display_name": "FSKX", + "id": "2f2d2f8a-5fd3-4e3a-88e7-2c70a1e6d41d", + "name": "FSKX" + }, + { + "vocabulary_id": null, + "state": "active", + "display_name": "Lettuce", + "id": "e6cc2e69-22c2-42b5-8a08-dbabfac2f8e2", + "name": "Lettuce" + }, + { + "vocabulary_id": null, + "state": "active", + "display_name": "Meat preparations of meat offals blood animal fats fresh chilled or frozen salted in brine", + "id": "d59cfef6-76d1-4914-9a04-0b649ba4f805", + "name": "Meat preparations of meat offals blood animal fats fresh chilled or frozen salted in brine" + }, + { + "vocabulary_id": null, + "state": "active", + "display_name": "Monoammonium glutamate", + "id": "eb3c1c78-7bdb-419c-b5c9-9079f341c430", + "name": "Monoammonium glutamate" + }, + { + "vocabulary_id": null, + "state": "active", + "display_name": "R 3", + "id": "c4bd4a30-b052-42a5-a332-6a5d458ef9ca", + "name": "R 3" + }, + { + "vocabulary_id": null, + "state": "active", + "display_name": "Salmonella Daarle", + "id": "c7a63c98-ba69-44ce-8e37-a123df122417", + "name": "Salmonella Daarle" + }, + { + "vocabulary_id": null, + "state": "active", + "display_name": "Tomatoes", + "id": "b6e034fa-24c5-4cea-87b8-03f4a8afc4b1", + "name": "Tomatoes" + }, + { + "vocabulary_id": null, + "state": "active", + "display_name": "human consumer adult", + "id": "6ae4d20c-a847-4ea1-b059-72ebeef0b2e6", + "name": "human consumer adult" + }, + { + "vocabulary_id": null, + "state": "active", + "display_name": "human consumer men", + "id": "de758955-1287-4938-9ca3-7d8562b16b88", + "name": "human consumer men" + }, + { + "vocabulary_id": null, + "state": "active", + "display_name": "human consumer no age specification", + "id": "700657d3-4d08-4abb-8a87-b1b9083d3003", + "name": "human consumer no age specification" + }, + { + "vocabulary_id": null, + "state": "active", + "display_name": "norovirus Norwalk like virus", + "id": "3be8bc88-b97f-4027-a774-67e83c08cefb", + "name": "norovirus Norwalk like virus" + }], + "groups": [], + "creator_user_id": "adc74fb3-06b9-4c77-9648-d444c7a8c56a", + "relationships_as_subject": [], + "organization": + { + "description": "This Virtual Research Environment is conceived to provide AGINFRAplus consortium members with a working environment to experience with AGINFRAplus services and develop new ones.", + "created": "2018-10-01T15:47:48.334870", + "title": "AGINFRAplusDev", + "name": "aginfraplusdev", + "is_organization": true, + "state": "active", + "image_url": "https://services.d4science.org/image/layout_set_logo?img_id=143596548", + "revision_id": "691b0a49-da44-4051-a367-04219dc48d45", + "type": "organization", + "id": "10df5d21-fe6f-4c68-a78b-d5521be85e22", + "approval_status": "approved" + }, + "name": "49f6919f-2403-4bdf-80f0-b4b3be2ab686", + "isopen": false, + "url": "", + "notes": "

description of a system using mathematical concepts and language. The process of developing a mathematical model is termed mathematical modeling. Mathematical models are used in the natural sciences (such as physics, biology, earth science, chemistry) and engineering disciplines (such as computer science, electrical engineering), as well as in the social sciences (such as economics, psychology, sociology, political science).

Objective is to test, whether all metadata are displayed correctly

", + "owner_org": "10df5d21-fe6f-4c68-a78b-d5521be85e22", + "extras": [ + { + "key": "FSKX_model_parameters:Input Parameter", + "value": "Temperature( T): 4.0 \u00b0C( Double)" + }, + { + "key": "FSKX_model_parameters:Input Parameter", + "value": "Dimensionless quantity( aw): 0.974 [aw]( Double)" + }, + { + "key": "FSKX_model_parameters:Output Parameter", + "value": "Time( Time): h( Double)" + }, + { + "key": "FSKX_model_parameters:Output Parameter", + "value": "Number_Content(count/mass)( value): ln(objects/g)( Double)" + }, + { + "key": "FSKX_model_scope:Hazard", + "value": "norovirus Norwalk like virus " + }, + { + "key": "FSKX_model_scope:Hazard", + "value": "Salmonella Daarle" + }, + { + "key": "FSKX_model_scope:Hazard", + "value": "Monoammonium glutamate" + }, + { + "key": "FSKX_model_scope:Population", + "value": "human consumer no age specification" + }, + { + "key": "FSKX_model_scope:Population", + "value": "human consumer adult" + }, + { + "key": "FSKX_model_scope:Population", + "value": "human consumer men" + }, + { + "key": "FSKX_model_scope:Product", + "value": "Lettuce" + }, + { + "key": "FSKX_model_scope:Product", + "value": "Tomatoes" + }, + { + "key": "FSKX_model_scope:Product", + "value": "Meat preparations of meat offals blood animal fats fresh chilled or frozen salted in brine " + }, + { + "key": "Item URL", + "value": "http://data.d4science.org/ctlg/AGINFRAplusDev/49f6919f-2403-4bdf-80f0-b4b3be2ab686" + }, + { + "key": "Model Author", + "value": "Mosley, Steve, mosley@nyu.org" + }, + { + "key": "Model Creator", + "value": "Romanov, Natalia, black_widow@marvel.com" + }, + { + "key": "Model Creator", + "value": "Parker, Peter, peter.parker@parker.com" + }, + { + "key": "Model ID", + "value": "Toy_Model_Generic_03" + }, + { + "key": "Model Language", + "value": "R 3" + }, + { + "key": "ReadMe", + "value": "This model is made available in the FSK-ML format, i.e. as .fskx file. To execute the model or to perform model-based predictions it is recommended to use the software FSK-Lab. FSK-Lab is an open-source extension of the open-source data analytics platform KNIME. To install FSK-Lab follow the installation instructions available at: https://foodrisklabs.bfr.bund.de/fsk-lab_de/. Once FSK-Lab is installed a new KNIME workflow should be created and the FSKX Reader node should be dragged into it. This FSKX Reader node can be configured to read in the given .fskx file. To perform a model-based prediction connect the out-port of the FSKX Reader node with the FSK Simulation Configurator JS node to adjust if necessary input parameters and store this into a user defined simulation setting, After that connect the output port with the input of a FSK Runner node that perform the simulation and look at the results at the node's outport." + }, + { + "key": "Reference Description", + "value": "Quantitative Risk Assessment in Food Establishments: Evaluating Strategies and Behavior on the Risk Associated with Fats in Foods DOI: 10.5072/zenodo.168114" + }, + { + "key": "Reference Description", + "value": "Norwalk virus: How infectious is it? DOI: 10.1002/zenodo.211137" + }, + { + "key": "Reference Description", + "value": "Dose Response Models For Infectious Gastronomies DOI: 10.1111/j.153xxx924.1999.tb01143.x" + }, + { + "key": "Related Identifier", + "value": "myLink" + }, + { + "key": "system:type", + "value": "FSKXModel" + }], + "ratings_count": 0, + "title": "Toy Model for Testing Purposes", + "revision_id": "999791f0-aaab-454c-b72f-a1c1ded2fba7" + } \ No newline at end of file diff --git a/ckan2zenodo-library/src/test/resources/ResearchObject.json b/ckan2zenodo-library/src/test/resources/ResearchObject.json new file mode 100644 index 0000000..3dd57c7 --- /dev/null +++ b/ckan2zenodo-library/src/test/resources/ResearchObject.json @@ -0,0 +1,127 @@ +{ + "rating": 0.0, + "license_title": "Creative Commons Attribution 4.0", + "maintainer": "Fabio Sinibaldi", + "relationships_as_object": [], + "private": true, + "maintainer_email": "fabio.sinibaldi@isti.cnr.it", + "num_tags": 4, + "id": "54938132-299e-46f7-a84a-494fcd75ad8f", + "metadata_created": "2019-12-03T11:16:47.872092", + "metadata_modified": "2019-12-03T11:20:02.571739", + "author": "Candela Leonardo", + "author_email": "leonardo.candela@isti.cnr.it", + "acquire_url": "", + "state": "active", + "version": "1", + "license_id": "CC-BY-4.0", + "type": "dataset", + "resources": [ + { + "mimetype": null, + "cache_url": null, + "hash": "", + "description": "Curated by: World Health Organization (WHO)\r\n\r\nAs part of their core goal for better health information worldwide, the World Health Organization makes their data on global health publicly available through the Global Health Observatory (GHO). The GHO acts as a portal with which to access and analyze health situations and important themes.\r\n\r\nThe various data sets are organized according to themes, such as mortality, health systems, communicable and non-communicable diseases, medicines and vaccines, health risks, and so on. The WHO\u2019s health statistics are to go-to source for global health information and is also used in the work of the US Centers for Disease Control and Prevention.", + "name": "Sample dataset Global Health Observatory data", + "format": "HTML", + "url": "https://www.ncdc.noaa.gov/cdo-web/datatools/lcd", + "datastore_active": false, + "cache_last_updated": null, + "package_id": "54938132-299e-46f7-a84a-494fcd75ad8f", + "created": "2019-12-03T11:20:02.594436", + "state": "active", + "mimetype_inner": null, + "last_modified": null, + "position": 0, + "revision_id": "9dd5e97c-8c39-4198-a34e-c56ae0631d17", + "url_type": null, + "id": "8caa68cb-6880-4683-9849-1aad02d8cc11", + "resource_type": null, + "size": null + }], + "num_resources": 1, + "tags": [ + { + "vocabulary_id": null, + "state": "active", + "display_name": "AgriFood", + "id": "d91b8753-1da4-4dea-a62e-7a9caea4326f", + "name": "AgriFood" + }, + { + "vocabulary_id": null, + "state": "active", + "display_name": "Sample", + "id": "7b0fbf46-07eb-46a0-9859-1269f792ff74", + "name": "Sample" + }, + { + "vocabulary_id": null, + "state": "active", + "display_name": "abruzzi", + "id": "5cf9a91a-c74e-4efe-b788-8d4cea220d58", + "name": "abruzzi" + }, + { + "vocabulary_id": null, + "state": "active", + "display_name": "acid rain", + "id": "5c715bef-7d5e-4af5-8567-318711218aea", + "name": "acid rain" + }], + "groups": [], + "creator_user_id": "1be839d2-acc2-4cc5-ac71-f174218c41af", + "relationships_as_subject": [], + "organization": + { + "description": "This Virtual Research Environment is conceived to provide AGINFRAplus consortium members with a working environment to experience with AGINFRAplus services and develop new ones.", + "created": "2018-10-01T15:47:48.334870", + "title": "AGINFRAplusDev", + "name": "aginfraplusdev", + "is_organization": true, + "state": "active", + "image_url": "https://services.d4science.org/image/layout_set_logo?img_id=143596548", + "revision_id": "691b0a49-da44-4051-a367-04219dc48d45", + "type": "organization", + "id": "10df5d21-fe6f-4c68-a78b-d5521be85e22", + "approval_status": "approved" + }, + "name": "a_sample_item_for_zenodo", + "isopen": true, + "url": "", + "notes": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Mauris venenatis \r\nullamcorper commodo. Duis id aliquet sapien. Mauris imperdiet turpis sit \r\namet tempor faucibus. Maecenas elementum, leo eu rutrum finibus, nisl orci \r\nlobortis elit, sed rhoncus tortor risus non est. Aliquam erat volutpat. Sed \r\nid quam et urna fringilla egestas sit amet vitae magna. Praesent nec augue \r\nvitae libero faucibus auctor in sit amet elit. Sed nec tristique eros, a \r\nfeugiat ex. Proin interdum ultrices luctus. Interdum et malesuada fames ac \r\nante ipsum primis in faucibus.", + "owner_org": "10df5d21-fe6f-4c68-a78b-d5521be85e22", + "extras": [ + { + "key": "Agricultural concept", + "value": "abruzzi" + }, + { + "key": "Agricultural concept", + "value": "acid rain" + }, + { + "key": "Author", + "value": "Verdi, Luca, luca.verdi@acme.org" + }, + { + "key": "Author", + "value": "Rossi, Paolo, paolo.rossi@acme.org, orcid.org/0000-1111-2222-3333" + }, + { + "key": "Item URL", + "value": "http://data.d4science.org/ctlg/AGINFRAplusDev/a_sample_item_for_zenodo" + }, + { + "key": "spatial", + "value": "{\"type\": \"Polygon\", \"coordinates\": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0], [-180.0, 90.0], [-180.0, -90.0]]]}" + }, + { + "key": "system:type", + "value": "ResearchObject" + }], + "license_url": "https://creativecommons.org/licenses/by/4.0/", + "ratings_count": 0, + "title": "A sample item for Zenodo", + "revision_id": "66e8337c-9d1b-4526-9a76-b1662035fd1c" +} \ No newline at end of file