added opendoar dump; refactored dump scripts
parent
85d2dc6cb7
commit
a055b4247e
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@ -0,0 +1,21 @@
|
||||
import config
|
||||
import requests
|
||||
import json
|
||||
import time
|
||||
|
||||
DATE = time.strftime('%Y-%m-%d')
|
||||
|
||||
with open('../../data/raw/opendoar_dump_%s.json' % DATE, 'w') as file_out:
|
||||
size = 100
|
||||
offset = 0
|
||||
while(True):
|
||||
url = 'https://v2.sherpa.ac.uk/cgi/retrieve/cgi/retrieve?item-type=repository&api-key=%s&format=Json&limit=%s&offset=%s' % (config.opendoar_apikey, size, offset)
|
||||
print(url)
|
||||
|
||||
response = requests.request("GET", url)
|
||||
file_out.writelines('\n'.join([json.dumps(record) for record in response.json()['items']]))
|
||||
file_out.write('\n')
|
||||
|
||||
if len(response.json()['items']) < size:
|
||||
break
|
||||
offset += size
|
Loading…
Reference in New Issue