|
|
|
@ -27,24 +27,19 @@ def testValidJSON(x):
|
|
|
|
|
except:
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
conf = SparkConf().setAppName('fairshare').setMaster('local[*]')
|
|
|
|
|
conf = SparkConf().setAppName('Create JSON Dump').setMaster('local[*]')
|
|
|
|
|
sc = SparkContext(conf=conf)
|
|
|
|
|
spark = SparkSession.builder.config(conf=conf).getOrCreate()
|
|
|
|
|
|
|
|
|
|
deleteDir("datasources")
|
|
|
|
|
# RAW DUMPS PATH
|
|
|
|
|
fairsharing_path = "../../data/raw/fairsharing_dump_api_02_2022.json"
|
|
|
|
|
opendoar_path = "../../data/raw/openDoar.tsv"
|
|
|
|
|
re3data_path = "../../data/raw/re3data.tsv"
|
|
|
|
|
roar_path = "../../data/raw/export_roar_CSV.csv"
|
|
|
|
|
|
|
|
|
|
fairsharing_path = "../data/raw/fairsharing_dump_api_02_2022.json"
|
|
|
|
|
opendoar_path = "../data/raw/openDoar.tsv"
|
|
|
|
|
re3data_path = "../data/raw/re3data.tsv"
|
|
|
|
|
roar_path = "../data/raw/export_roar_CSV.csv"
|
|
|
|
|
output_path = "../../data/interim/datasources"
|
|
|
|
|
|
|
|
|
|
# #FAIRSHARING V1.0
|
|
|
|
|
# #FILE STRUCTURE: Full name of record | Short name | FAIRsharing URL | Homepage URL of the resource | Country Field | Subjects
|
|
|
|
|
# fairsharing = sc.textFile("/Users/miconis/Desktop/Fairsharing dedup/registries_analysis/data/raw/FAIRsharingDBrec_summary20210304.csv")
|
|
|
|
|
# fairsharing = fairsharing.filter(lambda x: " FAIRsharing URL " not in x.split("|")).map(lambda x: dict(fullname=x.split("|")[0], shortname=x.split("|")[1], fairshare_url=x.split("|")[2], homepage_url=x.split("|")[3], country=x.split("|")[4], subjects=x.split("|")[5]))
|
|
|
|
|
# fairsharing = fairsharing.map(lambda x: dict(id=x['fairshare_url'], originalId=x['fairshare_url'], websiteurl=x['homepage_url'], name=x['fullname'], alternativeNames=[x['shortname']], collectedfrom="FAIRsharing"))
|
|
|
|
|
#
|
|
|
|
|
# print "FAIRSHARING:" + str(fairsharing.count())
|
|
|
|
|
deleteDir(output_path)
|
|
|
|
|
|
|
|
|
|
#FAIRSHARING (file structure: )
|
|
|
|
|
fairsharing = sc.textFile(fairsharing_path).filter(testValidJSON).map(json.loads)
|
|
|
|
@ -73,4 +68,4 @@ print "ROAR:" + str(roar.count())
|
|
|
|
|
|
|
|
|
|
all_ds = fairsharing.union(opendoar).union(re3data).union(roar)
|
|
|
|
|
|
|
|
|
|
all_ds.map(json.dumps).saveAsTextFile("datasources")
|
|
|
|
|
all_ds.map(json.dumps).saveAsTextFile(output_path)
|
|
|
|
|