You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
74 lines
3.3 KiB
Python
74 lines
3.3 KiB
Python
# -*- coding: utf-8 -*-
|
|
import click
|
|
import logging
|
|
from pathlib import Path
|
|
from dotenv import find_dotenv, load_dotenv
|
|
import pandas as pd
|
|
import ast
|
|
import os
|
|
|
|
@click.command()
|
|
@click.argument('input_filepath', type=click.Path(exists=True))
|
|
@click.argument('output_filepath', type=click.Path())
|
|
def main(input_filepath, output_filepath):
|
|
""" Runs data processing scripts to turn raw data from (../raw) into
|
|
cleaned data ready to be analyzed (saved in ../processed).
|
|
"""
|
|
logger = logging.getLogger(__name__)
|
|
logger.info('Making final data set from raw data')
|
|
logger.info('Loading the zipped dataset')
|
|
df = pd.read_csv(os.path.join(input_filepath, 'initial_info_whole_20210322.tsv.gz'), compression='gzip', sep='\t', header=0,
|
|
names=['orcid', 'claimed','verified_email', 'verified_primary_email',
|
|
'given_names', 'family_name', 'biography', 'other_names', 'urls',
|
|
'primary_email', 'other_emails', 'keywords', 'external_ids', 'education',
|
|
'employment', 'n_works', 'works_source', 'activation_date', 'last_update_date',
|
|
'n_doi', 'n_arxiv', 'n_pmc', 'n_other_pids'], encoding = 'utf-8')
|
|
|
|
logger.info('Loading list columns')
|
|
logger.info('... other_names')
|
|
df['other_names'] = df[df.other_names.notna()]['other_names'].apply(lambda x: ast.literal_eval(x))
|
|
|
|
logger.info('... keywords')
|
|
df['keywords'] = df[df.keywords.notna()]['keywords'].apply(lambda x: ast.literal_eval(x))
|
|
|
|
logger.info('... urls')
|
|
df['urls'] = df[df.urls.notna()]['urls'].apply(lambda x: ast.literal_eval(x))
|
|
|
|
logger.info('... other_emails')
|
|
df['other_emails'] = df[df.other_emails.notna()]['other_emails'].apply(lambda x: ast.literal_eval(x))
|
|
|
|
logger.info('... education')
|
|
df['education'] = df[df.education.notna()]['education'].apply(lambda x: ast.literal_eval(x))
|
|
|
|
logger.info('... employment')
|
|
df['employment'] = df[df.employment.notna()]['employment'].apply(lambda x: ast.literal_eval(x))
|
|
|
|
logger.info('... external_ids')
|
|
df['external_ids'] = df[df.external_ids.notna()]['external_ids'].apply(lambda x: ast.literal_eval(x))
|
|
|
|
logger.info('... works_source')
|
|
df['works_source'] = df[df.works_source.notna()]['works_source'].apply(lambda x: ast.literal_eval(x))
|
|
|
|
logger.info('Integrating labels from ORCID found in OpenAIRE')
|
|
openaire_orcid = pd.read_csv(os.path.join(input_filepath, 'orcid_openaire.txt'), header=None, names=['orcid'])
|
|
df['label'] = df.orcid.isin(openaire_orcid['orcid'])
|
|
df['label'] = df['label'].astype(int)
|
|
|
|
logger.info('Serializing the dataset in ../processed')
|
|
df.to_pickle(os.path.join(output_filepath, 'dataset.pkl'))
|
|
|
|
logger.info('DONE!')
|
|
|
|
if __name__ == '__main__':
|
|
log_fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
|
logging.basicConfig(level=logging.INFO, format=log_fmt)
|
|
|
|
# not used in this stub but often useful for finding various files
|
|
project_dir = Path(__file__).resolve().parents[2]
|
|
|
|
# find .env automagically by walking up directories until it's found, then
|
|
# load up the .env entries as environment variables
|
|
load_dotenv(find_dotenv())
|
|
|
|
main()
|