update crossref mapping to be runnable separately as a single datasource outside doiboost
Sandro La Bruzzo
1 month ago
parent
5281f010a5
commit
58dbe71d39
@ -0,0 +1,26 @@
|
||||
[
|
||||
{
|
||||
"paramName": "m",
|
||||
"paramLongName": "master",
|
||||
"paramDescription": "the master name",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "s",
|
||||
"paramLongName": "sourcePath",
|
||||
"paramDescription": "The base path of Crossref DUMP",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "t",
|
||||
"paramLongName": "targetPath",
|
||||
"paramDescription": "The target path",
|
||||
"paramRequired": false
|
||||
},
|
||||
{
|
||||
"paramName": "i",
|
||||
"paramLongName": "isLookupUrl",
|
||||
"paramDescription": "the Information System Service LookUp URL",
|
||||
"paramRequired": true
|
||||
}
|
||||
]
|
||||
@ -0,0 +1,940 @@
|
||||
[
|
||||
{
|
||||
"id": "100007630",
|
||||
"uri": "http://dx.doi.org/10.13039/100007630",
|
||||
"name": "College of Engineering and Informatics, National University of Ireland, Galway",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100007731",
|
||||
"uri": "http://dx.doi.org/10.13039/100007731",
|
||||
"name": "Endo International",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100008099",
|
||||
"uri": "http://dx.doi.org/10.13039/100008099",
|
||||
"name": "Food Safety Authority of Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100008124",
|
||||
"uri": "http://dx.doi.org/10.13039/100008124",
|
||||
"name": "Department of Jobs, Enterprise and Innovation",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100009098",
|
||||
"uri": "http://dx.doi.org/10.13039/100009098",
|
||||
"name": "Department of Foreign Affairs and Trade, Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100009099",
|
||||
"uri": "http://dx.doi.org/10.13039/100009099",
|
||||
"name": "Irish Aid",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100009770",
|
||||
"uri": "http://dx.doi.org/10.13039/100009770",
|
||||
"name": "National University of Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100009985",
|
||||
"uri": "http://dx.doi.org/10.13039/100009985",
|
||||
"name": "Parkinson's Association of Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100010399",
|
||||
"uri": "http://dx.doi.org/10.13039/100010399",
|
||||
"name": "European Society of Cataract and Refractive Surgeons",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100010414",
|
||||
"uri": "http://dx.doi.org/10.13039/100010414",
|
||||
"name": "Health Research Board",
|
||||
"synonym": [
|
||||
"501100001590"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "100010546",
|
||||
"uri": "http://dx.doi.org/10.13039/100010546",
|
||||
"name": "Deparment of Children and Youth Affairs, Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100010993",
|
||||
"uri": "http://dx.doi.org/10.13039/100010993",
|
||||
"name": "Irish Nephrology Society",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100011062",
|
||||
"uri": "http://dx.doi.org/10.13039/100011062",
|
||||
"name": "Asian Spinal Cord Network",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100011096",
|
||||
"uri": "http://dx.doi.org/10.13039/100011096",
|
||||
"name": "Jazz Pharmaceuticals",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100011396",
|
||||
"uri": "http://dx.doi.org/10.13039/100011396",
|
||||
"name": "Irish College of General Practitioners",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100012734",
|
||||
"uri": "http://dx.doi.org/10.13039/100012734",
|
||||
"name": "Department for Culture, Heritage and the Gaeltacht, Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100012754",
|
||||
"uri": "http://dx.doi.org/10.13039/100012754",
|
||||
"name": "Horizon Pharma",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100012891",
|
||||
"uri": "http://dx.doi.org/10.13039/100012891",
|
||||
"name": "Medical Research Charities Group",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100012919",
|
||||
"uri": "http://dx.doi.org/10.13039/100012919",
|
||||
"name": "Epilepsy Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100012920",
|
||||
"uri": "http://dx.doi.org/10.13039/100012920",
|
||||
"name": "GLEN",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100012921",
|
||||
"uri": "http://dx.doi.org/10.13039/100012921",
|
||||
"name": "Royal College of Surgeons in Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100013029",
|
||||
"uri": "http://dx.doi.org/10.13039/100013029",
|
||||
"name": "Iris O'Brien Foundation",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100013206",
|
||||
"uri": "http://dx.doi.org/10.13039/100013206",
|
||||
"name": "Food Institutional Research Measure",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100013381",
|
||||
"uri": "http://dx.doi.org/10.13039/100013381",
|
||||
"name": "Irish Phytochemical Food Network",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100013433",
|
||||
"uri": "http://dx.doi.org/10.13039/100013433",
|
||||
"name": "Transport Infrastructure Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100013461",
|
||||
"uri": "http://dx.doi.org/10.13039/100013461",
|
||||
"name": "Arts and Disability Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100013548",
|
||||
"uri": "http://dx.doi.org/10.13039/100013548",
|
||||
"name": "Filmbase",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100013917",
|
||||
"uri": "http://dx.doi.org/10.13039/100013917",
|
||||
"name": "Society for Musicology in Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100014251",
|
||||
"uri": "http://dx.doi.org/10.13039/100014251",
|
||||
"name": "Humanities in the European Research Area",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100014364",
|
||||
"uri": "http://dx.doi.org/10.13039/100014364",
|
||||
"name": "National Children's Research Centre",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100014384",
|
||||
"uri": "http://dx.doi.org/10.13039/100014384",
|
||||
"name": "Amarin Corporation",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100014902",
|
||||
"uri": "http://dx.doi.org/10.13039/100014902",
|
||||
"name": "Irish Association for Cancer Research",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100015023",
|
||||
"uri": "http://dx.doi.org/10.13039/100015023",
|
||||
"name": "Ireland Funds",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100015037",
|
||||
"uri": "http://dx.doi.org/10.13039/100015037",
|
||||
"name": "Simon Cumbers Media Fund",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100015319",
|
||||
"uri": "http://dx.doi.org/10.13039/100015319",
|
||||
"name": "Sport Ireland Institute",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100015320",
|
||||
"uri": "http://dx.doi.org/10.13039/100015320",
|
||||
"name": "Paralympics Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100015442",
|
||||
"uri": "http://dx.doi.org/10.13039/100015442",
|
||||
"name": "Global Brain Health Institute",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100015776",
|
||||
"uri": "http://dx.doi.org/10.13039/100015776",
|
||||
"name": "Health and Social Care Board",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100015992",
|
||||
"uri": "http://dx.doi.org/10.13039/100015992",
|
||||
"name": "St. Luke's Institute of Cancer Research",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100017897",
|
||||
"uri": "http://dx.doi.org/10.13039/100017897",
|
||||
"name": "Friedreich\u2019s Ataxia Research Alliance Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100018064",
|
||||
"uri": "http://dx.doi.org/10.13039/100018064",
|
||||
"name": "Department of Tourism, Culture, Arts, Gaeltacht, Sport and Media",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100018172",
|
||||
"uri": "http://dx.doi.org/10.13039/100018172",
|
||||
"name": "Department of the Environment, Climate and Communications",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100018175",
|
||||
"uri": "http://dx.doi.org/10.13039/100018175",
|
||||
"name": "Dairy Processing Technology Centre",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100018270",
|
||||
"uri": "http://dx.doi.org/10.13039/100018270",
|
||||
"name": "Health Service Executive",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100018529",
|
||||
"uri": "http://dx.doi.org/10.13039/100018529",
|
||||
"name": "Alkermes",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100018542",
|
||||
"uri": "http://dx.doi.org/10.13039/100018542",
|
||||
"name": "Irish Endocrine Society",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100018754",
|
||||
"uri": "http://dx.doi.org/10.13039/100018754",
|
||||
"name": "An Roinn Sl\u00e1inte",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100018998",
|
||||
"uri": "http://dx.doi.org/10.13039/100018998",
|
||||
"name": "Irish Research eLibrary",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100019428",
|
||||
"uri": "http://dx.doi.org/10.13039/100019428",
|
||||
"name": "Nabriva Therapeutics",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100019637",
|
||||
"uri": "http://dx.doi.org/10.13039/100019637",
|
||||
"name": "Horizon Therapeutics",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100020174",
|
||||
"uri": "http://dx.doi.org/10.13039/100020174",
|
||||
"name": "Health Research Charities Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100020202",
|
||||
"uri": "http://dx.doi.org/10.13039/100020202",
|
||||
"name": "UCD Foundation",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100020233",
|
||||
"uri": "http://dx.doi.org/10.13039/100020233",
|
||||
"name": "Ireland Canada University Foundation",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100022943",
|
||||
"uri": "http://dx.doi.org/10.13039/100022943",
|
||||
"name": "National Cancer Registry Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001581",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001581",
|
||||
"name": "Arts Council of Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001582",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001582",
|
||||
"name": "Centre for Ageing Research and Development in Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001583",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001583",
|
||||
"name": "Cystinosis Foundation Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001584",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001584",
|
||||
"name": "Department of Agriculture, Food and the Marine, Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001586",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001586",
|
||||
"name": "Department of Education and Skills, Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001587",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001587",
|
||||
"name": "Economic and Social Research Institute",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001588",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001588",
|
||||
"name": "Enterprise Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001589",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001589",
|
||||
"name": "Environmental Protection Agency",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001591",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001591",
|
||||
"name": "Heritage Council",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001592",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001592",
|
||||
"name": "Higher Education Authority",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001593",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001593",
|
||||
"name": "Irish Cancer Society",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001594",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001594",
|
||||
"name": "Irish Heart Foundation",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001595",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001595",
|
||||
"name": "Irish Hospice Foundation",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001596",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001596",
|
||||
"name": "Irish Research Council for Science, Engineering and Technology",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001597",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001597",
|
||||
"name": "Irish Research Council for the Humanities and Social Sciences",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001598",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001598",
|
||||
"name": "Mental Health Commission",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001600",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001600",
|
||||
"name": "Research and Education Foundation, Sligo General Hospital",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001601",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001601",
|
||||
"name": "Royal Irish Academy",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001603",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001603",
|
||||
"name": "Sustainable Energy Authority of Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001604",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001604",
|
||||
"name": "Teagasc",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001627",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001627",
|
||||
"name": "Marine Institute",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001628",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001628",
|
||||
"name": "Central Remedial Clinic",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001629",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001629",
|
||||
"name": "Royal Dublin Society",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001630",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001630",
|
||||
"name": "Dublin Institute for Advanced Studies",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001631",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001631",
|
||||
"name": "University College Dublin",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001633",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001633",
|
||||
"name": "National University of Ireland, Maynooth",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001634",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001634",
|
||||
"name": "University of Galway",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001635",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001635",
|
||||
"name": "University of Limerick",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001636",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001636",
|
||||
"name": "University College Cork",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001637",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001637",
|
||||
"name": "Trinity College Dublin",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001638",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001638",
|
||||
"name": "Dublin City University",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100002081",
|
||||
"uri": "http://dx.doi.org/10.13039/501100002081",
|
||||
"name": "Irish Research Council",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100002736",
|
||||
"uri": "http://dx.doi.org/10.13039/501100002736",
|
||||
"name": "Covidien",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100002755",
|
||||
"uri": "http://dx.doi.org/10.13039/501100002755",
|
||||
"name": "Brennan and Company",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100002919",
|
||||
"uri": "http://dx.doi.org/10.13039/501100002919",
|
||||
"name": "Cork Institute of Technology",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100002959",
|
||||
"uri": "http://dx.doi.org/10.13039/501100002959",
|
||||
"name": "Dublin City Council",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100003036",
|
||||
"uri": "http://dx.doi.org/10.13039/501100003036",
|
||||
"name": "Perrigo Company Charitable Foundation",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100003037",
|
||||
"uri": "http://dx.doi.org/10.13039/501100003037",
|
||||
"name": "Elan",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100003496",
|
||||
"uri": "http://dx.doi.org/10.13039/501100003496",
|
||||
"name": "HeyStaks Technologies",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100003553",
|
||||
"uri": "http://dx.doi.org/10.13039/501100003553",
|
||||
"name": "Gaelic Athletic Association",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100003840",
|
||||
"uri": "http://dx.doi.org/10.13039/501100003840",
|
||||
"name": "Irish Institute of Clinical Neuroscience",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100003956",
|
||||
"uri": "http://dx.doi.org/10.13039/501100003956",
|
||||
"name": "Aspect Medical Systems",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100004162",
|
||||
"uri": "http://dx.doi.org/10.13039/501100004162",
|
||||
"name": "Meath Foundation",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100004210",
|
||||
"uri": "http://dx.doi.org/10.13039/501100004210",
|
||||
"name": "Our Lady's Children's Hospital, Crumlin",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100004321",
|
||||
"uri": "http://dx.doi.org/10.13039/501100004321",
|
||||
"name": "Shire",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100004981",
|
||||
"uri": "http://dx.doi.org/10.13039/501100004981",
|
||||
"name": "Athlone Institute of Technology",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100006518",
|
||||
"uri": "http://dx.doi.org/10.13039/501100006518",
|
||||
"name": "Department of Communications, Energy and Natural Resources, Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100006553",
|
||||
"uri": "http://dx.doi.org/10.13039/501100006553",
|
||||
"name": "Collaborative Centre for Applied Nanotechnology",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100006759",
|
||||
"uri": "http://dx.doi.org/10.13039/501100006759",
|
||||
"name": "CLARITY Centre for Sensor Web Technologies",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100009246",
|
||||
"uri": "http://dx.doi.org/10.13039/501100009246",
|
||||
"name": "Technological University Dublin",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100009269",
|
||||
"uri": "http://dx.doi.org/10.13039/501100009269",
|
||||
"name": "Programme of Competitive Forestry Research for Development",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100009315",
|
||||
"uri": "http://dx.doi.org/10.13039/501100009315",
|
||||
"name": "Cystinosis Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100010808",
|
||||
"uri": "http://dx.doi.org/10.13039/501100010808",
|
||||
"name": "Geological Survey of Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100011030",
|
||||
"uri": "http://dx.doi.org/10.13039/501100011030",
|
||||
"name": "Alimentary Glycoscience Research Cluster",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100011031",
|
||||
"uri": "http://dx.doi.org/10.13039/501100011031",
|
||||
"name": "Alimentary Health",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100011103",
|
||||
"uri": "http://dx.doi.org/10.13039/501100011103",
|
||||
"name": "Rann\u00eds",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100012354",
|
||||
"uri": "http://dx.doi.org/10.13039/501100012354",
|
||||
"name": "Inland Fisheries Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100014384",
|
||||
"uri": "http://dx.doi.org/10.13039/501100014384",
|
||||
"name": "X-Bolt Orthopaedics",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100014710",
|
||||
"uri": "http://dx.doi.org/10.13039/501100014710",
|
||||
"name": "PrecisionBiotics Group",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100014827",
|
||||
"uri": "http://dx.doi.org/10.13039/501100014827",
|
||||
"name": "Dormant Accounts Fund",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100016041",
|
||||
"uri": "http://dx.doi.org/10.13039/501100016041",
|
||||
"name": "St Vincents Anaesthesia Foundation",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100017501",
|
||||
"uri": "http://dx.doi.org/10.13039/501100017501",
|
||||
"name": "FotoNation",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100018641",
|
||||
"uri": "http://dx.doi.org/10.13039/501100018641",
|
||||
"name": "Dairy Research Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100018839",
|
||||
"uri": "http://dx.doi.org/10.13039/501100018839",
|
||||
"name": "Irish Centre for High-End Computing",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100019905",
|
||||
"uri": "http://dx.doi.org/10.13039/501100019905",
|
||||
"name": "Galway University Foundation",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100020036",
|
||||
"uri": "http://dx.doi.org/10.13039/501100020036",
|
||||
"name": "Dystonia Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100020221",
|
||||
"uri": "http://dx.doi.org/10.13039/501100020221",
|
||||
"name": "Irish Motor Neurone Disease Association",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100020270",
|
||||
"uri": "http://dx.doi.org/10.13039/501100020270",
|
||||
"name": "Advanced Materials and Bioengineering Research",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100020403",
|
||||
"uri": "http://dx.doi.org/10.13039/501100020403",
|
||||
"name": "Irish Composites Centre",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100020425",
|
||||
"uri": "http://dx.doi.org/10.13039/501100020425",
|
||||
"name": "Irish Thoracic Society",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100021102",
|
||||
"uri": "http://dx.doi.org/10.13039/501100021102",
|
||||
"name": "Waterford Institute of Technology",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100021110",
|
||||
"uri": "http://dx.doi.org/10.13039/501100021110",
|
||||
"name": "Irish MPS Society",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100021525",
|
||||
"uri": "http://dx.doi.org/10.13039/501100021525",
|
||||
"name": "Insight SFI Research Centre for Data Analytics",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100021694",
|
||||
"uri": "http://dx.doi.org/10.13039/501100021694",
|
||||
"name": "Elan Pharma International",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100021838",
|
||||
"uri": "http://dx.doi.org/10.13039/501100021838",
|
||||
"name": "Royal College of Physicians of Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100022542",
|
||||
"uri": "http://dx.doi.org/10.13039/501100022542",
|
||||
"name": "Breakthrough Cancer Research",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100022610",
|
||||
"uri": "http://dx.doi.org/10.13039/501100022610",
|
||||
"name": "Breast Cancer Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100022728",
|
||||
"uri": "http://dx.doi.org/10.13039/501100022728",
|
||||
"name": "Munster Technological University",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100022729",
|
||||
"uri": "http://dx.doi.org/10.13039/501100022729",
|
||||
"name": "Institute of Technology, Tralee",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100023273",
|
||||
"uri": "http://dx.doi.org/10.13039/501100023273",
|
||||
"name": "HRB Clinical Research Facility Galway",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100023378",
|
||||
"uri": "http://dx.doi.org/10.13039/501100023378",
|
||||
"name": "Lauritzson Foundation",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100023551",
|
||||
"uri": "http://dx.doi.org/10.13039/501100023551",
|
||||
"name": "Cystic Fibrosis Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100023970",
|
||||
"uri": "http://dx.doi.org/10.13039/501100023970",
|
||||
"name": "Tyndall National Institute",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100024094",
|
||||
"uri": "http://dx.doi.org/10.13039/501100024094",
|
||||
"name": "Raidi\u00f3 Teilif\u00eds \u00c9ireann",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100024242",
|
||||
"uri": "http://dx.doi.org/10.13039/501100024242",
|
||||
"name": "Synthesis and Solid State Pharmaceutical Centre",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100024313",
|
||||
"uri": "http://dx.doi.org/10.13039/501100024313",
|
||||
"name": "Irish Rugby Football Union",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100007490",
|
||||
"uri": "http://dx.doi.org/10.13039/100007490",
|
||||
"name": "Bausch and Lomb Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100007819",
|
||||
"uri": "http://dx.doi.org/10.13039/100007819",
|
||||
"name": "Allergan",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100010547",
|
||||
"uri": "http://dx.doi.org/10.13039/100010547",
|
||||
"name": "Irish Youth Justice Service",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100012733",
|
||||
"uri": "http://dx.doi.org/10.13039/100012733",
|
||||
"name": "National Parks and Wildlife Service",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100015278",
|
||||
"uri": "http://dx.doi.org/10.13039/100015278",
|
||||
"name": "Pfizer Healthcare Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100017144",
|
||||
"uri": "http://dx.doi.org/10.13039/100017144",
|
||||
"name": "Shell E and P Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "100022895",
|
||||
"uri": "http://dx.doi.org/10.13039/100022895",
|
||||
"name": "Health Research Institute, University of Limerick",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100001599",
|
||||
"uri": "http://dx.doi.org/10.13039/501100001599",
|
||||
"name": "National Council for Forest Research and Development",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100006554",
|
||||
"uri": "http://dx.doi.org/10.13039/501100006554",
|
||||
"name": "IDA Ireland",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100011626",
|
||||
"uri": "http://dx.doi.org/10.13039/501100011626",
|
||||
"name": "Energy Policy Research Centre, Economic and Social Research Institute",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100014531",
|
||||
"uri": "http://dx.doi.org/10.13039/501100014531",
|
||||
"name": "Physical Education and Sport Sciences Department, University of Limerick",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100014745",
|
||||
"uri": "http://dx.doi.org/10.13039/501100014745",
|
||||
"name": "APC Microbiome Institute",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100014826",
|
||||
"uri": "http://dx.doi.org/10.13039/501100014826",
|
||||
"name": "ADAPT - Centre for Digital Content Technology",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100020570",
|
||||
"uri": "http://dx.doi.org/10.13039/501100020570",
|
||||
"name": "College of Medicine, Nursing and Health Sciences, National University of Ireland, Galway",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100020871",
|
||||
"uri": "http://dx.doi.org/10.13039/501100020871",
|
||||
"name": "Bernal Institute, University of Limerick",
|
||||
"synonym": []
|
||||
},
|
||||
{
|
||||
"id": "501100023852",
|
||||
"uri": "http://dx.doi.org/10.13039/501100023852",
|
||||
"name": "Moore Institute for Research in the Humanities and Social Studies, University of Galway",
|
||||
"synonym": []
|
||||
}
|
||||
]
|
||||
@ -0,0 +1,44 @@
|
||||
<configuration>
|
||||
<property>
|
||||
<name>jobTracker</name>
|
||||
<value>yarnRM</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>nameNode</name>
|
||||
<value>hdfs://nameservice1</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>oozie.use.system.libpath</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>oozie.action.sharelib.for.spark</name>
|
||||
<value>spark2</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>oozie.launcher.mapreduce.user.classpath.first</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>spark2ExtraListeners</name>
|
||||
<value>com.cloudera.spark.lineage.NavigatorAppListener</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>spark2SqlQueryExecutionListeners</name>
|
||||
<value>com.cloudera.spark.lineage.NavigatorQueryListener</value>
|
||||
</property>
|
||||
<property>
|
||||
<name>spark2YarnHistoryServerAddress</name>
|
||||
<value>http://iis-cdh5-test-gw.ocean.icm.edu.pl:18089 </value>
|
||||
</property>
|
||||
<property>
|
||||
<name>spark2EventLogDir</name>
|
||||
<value>/user/spark/spark2ApplicationHistory</value>
|
||||
</property>
|
||||
|
||||
|
||||
|
||||
</configuration>
|
||||
@ -0,0 +1,54 @@
|
||||
<workflow-app name="generate_crossref_Datasource" xmlns="uri:oozie:workflow:0.5">
|
||||
<parameters>
|
||||
<property>
|
||||
<name>sourcePath</name>
|
||||
<description>The base path of MAG DUMP CSV Tables</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>targetPath</name>
|
||||
<description>The base path of MAG DUMP CSV Tables</description>
|
||||
</property>
|
||||
<property>
|
||||
<name>isLookupUrl</name>
|
||||
<description>The Information service Lookup URL</description>
|
||||
</property>
|
||||
</parameters>
|
||||
|
||||
<start to="generateOAF"/>
|
||||
|
||||
<kill name="Kill">
|
||||
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
|
||||
</kill>
|
||||
|
||||
|
||||
|
||||
|
||||
<action name="generateOAF">
|
||||
<spark xmlns="uri:oozie:spark-action:0.2">
|
||||
<master>yarn</master>
|
||||
<mode>cluster</mode>
|
||||
<name>Crossref TO OAF</name>
|
||||
<class>eu.dnetlib.dhp.collection.crossref.SparkMapDumpIntoOAF</class>
|
||||
<jar>dhp-aggregation-${projectVersion}.jar</jar>
|
||||
<spark-opts>
|
||||
--executor-memory=${sparkExecutorMemory}
|
||||
--executor-cores=${sparkExecutorCores}
|
||||
--driver-memory=${sparkDriverMemory}
|
||||
--conf spark.executor.memoryOverhead=2g
|
||||
--conf spark.sql.shuffle.partitions=3000
|
||||
--conf spark.extraListeners=${spark2ExtraListeners}
|
||||
--conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners}
|
||||
--conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress}
|
||||
--conf spark.eventLog.dir=${nameNode}${spark2EventLogDir}
|
||||
</spark-opts>
|
||||
<arg>--sourcePath</arg><arg>${sourcePath}</arg>
|
||||
<arg>--targetPath</arg><arg>${targetPath}</arg>
|
||||
<arg>--isLookupUrl</arg><arg>${isLookupUrl}</arg>
|
||||
<arg>--master</arg><arg>yarn</arg>
|
||||
</spark>
|
||||
<ok to="End"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
||||
<end name="End"/>
|
||||
</workflow-app>
|
||||
@ -1,106 +0,0 @@
|
||||
package eu.dnetlib.dhp.collection.crossref
|
||||
|
||||
import eu.dnetlib.dhp.application.ArgumentApplicationParser
|
||||
import org.apache.commons.io.IOUtils
|
||||
import org.apache.hadoop.io.{IntWritable, Text}
|
||||
import org.apache.spark.SparkConf
|
||||
import org.apache.spark.sql.expressions.Aggregator
|
||||
import org.apache.spark.sql.{Dataset, Encoder, SaveMode, SparkSession}
|
||||
import org.json4s
|
||||
import org.json4s.DefaultFormats
|
||||
import org.json4s.jackson.JsonMethods.parse
|
||||
import org.slf4j.{Logger, LoggerFactory}
|
||||
|
||||
object CrossrefDataset {
|
||||
|
||||
val logger: Logger = LoggerFactory.getLogger(CrossrefDataset.getClass)
|
||||
|
||||
def to_item(input: String): CrossrefDT = {
|
||||
|
||||
implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats
|
||||
lazy val json: json4s.JValue = parse(input)
|
||||
val ts: Long = (json \ "indexed" \ "timestamp").extract[Long]
|
||||
val doi: String = DoiBoostMappingUtil.normalizeDoi((json \ "DOI").extract[String])
|
||||
CrossrefDT(doi, input, ts)
|
||||
|
||||
}
|
||||
|
||||
def main(args: Array[String]): Unit = {
|
||||
|
||||
val conf: SparkConf = new SparkConf()
|
||||
val parser = new ArgumentApplicationParser(
|
||||
IOUtils.toString(
|
||||
CrossrefDataset.getClass.getResourceAsStream(
|
||||
"/eu/dnetlib/dhp/doiboost/crossref_to_dataset_params.json"
|
||||
)
|
||||
)
|
||||
)
|
||||
parser.parseArgument(args)
|
||||
val spark: SparkSession =
|
||||
SparkSession
|
||||
.builder()
|
||||
.config(conf)
|
||||
.appName(CrossrefDataset.getClass.getSimpleName)
|
||||
.master(parser.get("master"))
|
||||
.getOrCreate()
|
||||
import spark.implicits._
|
||||
|
||||
val crossrefAggregator = new Aggregator[CrossrefDT, CrossrefDT, CrossrefDT] with Serializable {
|
||||
|
||||
override def zero: CrossrefDT = null
|
||||
|
||||
override def reduce(b: CrossrefDT, a: CrossrefDT): CrossrefDT = {
|
||||
if (b == null)
|
||||
return a
|
||||
if (a == null)
|
||||
return b
|
||||
|
||||
if (a.timestamp > b.timestamp) {
|
||||
return a
|
||||
}
|
||||
b
|
||||
}
|
||||
|
||||
override def merge(a: CrossrefDT, b: CrossrefDT): CrossrefDT = {
|
||||
if (b == null)
|
||||
return a
|
||||
if (a == null)
|
||||
return b
|
||||
|
||||
if (a.timestamp > b.timestamp) {
|
||||
return a
|
||||
}
|
||||
b
|
||||
}
|
||||
|
||||
override def bufferEncoder: Encoder[CrossrefDT] = implicitly[Encoder[CrossrefDT]]
|
||||
|
||||
override def outputEncoder: Encoder[CrossrefDT] = implicitly[Encoder[CrossrefDT]]
|
||||
|
||||
override def finish(reduction: CrossrefDT): CrossrefDT = reduction
|
||||
}
|
||||
|
||||
val workingPath: String = parser.get("workingPath")
|
||||
|
||||
val main_ds: Dataset[CrossrefDT] = spark.read.load(s"$workingPath/crossref_ds").as[CrossrefDT]
|
||||
|
||||
val update =
|
||||
spark.createDataset(
|
||||
spark.sparkContext
|
||||
.sequenceFile(s"$workingPath/index_update", classOf[IntWritable], classOf[Text])
|
||||
.map(i => CrossrefImporter.decompressBlob(i._2.toString))
|
||||
.map(i => to_item(i))
|
||||
)
|
||||
|
||||
main_ds
|
||||
.union(update)
|
||||
.groupByKey(_.doi)
|
||||
.agg(crossrefAggregator.toColumn)
|
||||
.map(s => s._2)
|
||||
.write
|
||||
.mode(SaveMode.Overwrite)
|
||||
.save(s"$workingPath/crossref_ds_updated")
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,61 @@
|
||||
package eu.dnetlib.dhp.collection.crossref
|
||||
|
||||
import eu.dnetlib.dhp.application.AbstractScalaApplication
|
||||
import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup
|
||||
import eu.dnetlib.dhp.schema.oaf.{Oaf, Publication, Dataset => OafDataset}
|
||||
import eu.dnetlib.dhp.utils.ISLookupClientFactory
|
||||
import org.apache.spark.sql._
|
||||
import org.slf4j.{Logger, LoggerFactory}
|
||||
|
||||
class SparkMapDumpIntoOAF(propertyPath: String, args: Array[String], log: Logger)
|
||||
extends AbstractScalaApplication(propertyPath, args, log: Logger) {
|
||||
|
||||
/** Here all the spark applications runs this method
|
||||
* where the whole logic of the spark node is defined
|
||||
*/
|
||||
override def run(): Unit = {
|
||||
val sourcePath = parser.get("sourcePath")
|
||||
log.info("sourcePath: {}", sourcePath)
|
||||
val targetPath = parser.get("targetPath")
|
||||
log.info("targetPath: {}", targetPath)
|
||||
val isLookupUrl: String = parser.get("isLookupUrl")
|
||||
log.info("isLookupUrl: {}", isLookupUrl)
|
||||
val isLookupService = ISLookupClientFactory.getLookUpService(isLookupUrl)
|
||||
val vocabularies = VocabularyGroup.loadVocsFromIS(isLookupService)
|
||||
require(vocabularies != null)
|
||||
transformCrossref(spark, sourcePath, targetPath, vocabularies)
|
||||
|
||||
}
|
||||
|
||||
def transformCrossref(
|
||||
spark: SparkSession,
|
||||
sourcePath: String,
|
||||
targetPath: String,
|
||||
vocabularies: VocabularyGroup
|
||||
): Unit = {
|
||||
import spark.implicits._
|
||||
val dump = spark.read.text(sourcePath).as[String]
|
||||
dump
|
||||
.flatMap(s => Crossref2Oaf.convert(s, vocabularies))
|
||||
.write
|
||||
.mode(SaveMode.Overwrite)
|
||||
.partitionBy("oafType")
|
||||
.option("compression", "gzip")
|
||||
.text(targetPath)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
object SparkMapDumpIntoOAF {
|
||||
|
||||
def main(args: Array[String]): Unit = {
|
||||
val logger: Logger = LoggerFactory.getLogger(SparkMapDumpIntoOAF.getClass)
|
||||
|
||||
new SparkMapDumpIntoOAF(
|
||||
log = logger,
|
||||
args = args,
|
||||
propertyPath = "/eu/dnetlib/dhp/collection/crossref/convert_crossref_dump_to_oaf_params.json"
|
||||
).initialize().run()
|
||||
}
|
||||
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,41 @@
|
||||
package eu.dnetlib.dhp.collection.crossref
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper
|
||||
import eu.dnetlib.dhp.aggregation.AbstractVocabularyTest
|
||||
import org.apache.spark.sql.SparkSession
|
||||
import org.junit.jupiter.api.{BeforeEach, Test}
|
||||
import org.junit.jupiter.api.extension.ExtendWith
|
||||
import org.mockito.junit.jupiter.MockitoExtension
|
||||
import org.slf4j.{Logger, LoggerFactory}
|
||||
|
||||
import scala.io.Source
|
||||
|
||||
@ExtendWith(Array(classOf[MockitoExtension]))
|
||||
class CrossrefMappingTest extends AbstractVocabularyTest {
|
||||
|
||||
val logger: Logger = LoggerFactory.getLogger(Crossref2Oaf.getClass)
|
||||
val mapper = new ObjectMapper()
|
||||
|
||||
@BeforeEach
|
||||
def setUp(): Unit = {
|
||||
super.setUpVocabulary()
|
||||
}
|
||||
|
||||
@Test
|
||||
def testMapping(): Unit = {
|
||||
val spark = SparkSession.builder().master("local[*]").appName("TransformCrossref").getOrCreate()
|
||||
|
||||
val s = new SparkMapDumpIntoOAF(null, null, null)
|
||||
import spark.implicits._
|
||||
|
||||
s.transformCrossref(
|
||||
spark,
|
||||
sourcePath = "/home/sandro/Downloads/crossref",
|
||||
targetPath = "/home/sandro/Downloads/crossref_transformed",
|
||||
vocabularies = vocabularies
|
||||
)
|
||||
|
||||
print(spark.read.text("/home/sandro/Downloads/crossref_transformed").count)
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue