351 lines
13 KiB
Python
351 lines
13 KiB
Python
from collections import defaultdict
|
|
from collections import Counter
|
|
|
|
import Levenshtein
|
|
|
|
from sklearn.feature_extraction.text import CountVectorizer
|
|
from sklearn.metrics.pairwise import cosine_similarity
|
|
|
|
from functions_cluster import *
|
|
from create_input_cluster import *
|
|
|
|
|
|
|
|
specific = [k for k in categ_dicts if categ_dicts[k] == 'Specific']
|
|
|
|
def best_sim_score(light_raw, candidate_num, pairs_list, m, simU, simG):
|
|
"""
|
|
Finds the best match between a 'key word' and several legal names from the OpenAIRE database.
|
|
---> corrects special cases in the main map that follows
|
|
|
|
Args:
|
|
light_raw
|
|
l2 candidate_num: number of candidates.
|
|
l3 pairs_list: List of pairs. (s, x, score)
|
|
l4 m: mult
|
|
|
|
Returns:
|
|
List: Resulting list containing OpenAIRE names and their similarity scores.
|
|
"""
|
|
|
|
vectorizer = CountVectorizer()
|
|
univ_num = light_raw.lower().count('univ')
|
|
result = []
|
|
best = []
|
|
s = light_raw
|
|
|
|
for j in range(len(pairs_list)):
|
|
x = pairs_list[j][1]
|
|
|
|
if [x, pairs_list[j][2]] in result:
|
|
continue
|
|
|
|
if m[pairs_list[j][0]] == 1:
|
|
|
|
if is_contained('univ', x.lower()) and pairs_list[j][2] > simU:
|
|
result.append([x, pairs_list[j][2]])
|
|
elif pairs_list[j][2] > simG:
|
|
result.append([x, pairs_list[j][2]])
|
|
|
|
elif pairs_list[j][2] >= 0.98: # and (is_contained("univ", x.lower()) or is_contained("college", x.lower()) or is_contained("center", x.lower()) or is_contained("schule", x.lower())): # If the similarity score of a pair (s,x) was 1, we store it to results list
|
|
result.append([pairs_list[j][1], 1])
|
|
|
|
else:
|
|
try:
|
|
if not is_contained("univ", x.lower()):
|
|
continue # Skip if x does not contain "university" or "univ"
|
|
|
|
# if (is_contained('hosp', x.lower()) and not is_contained('hosp', s)) or (not is_contained('hosp', x.lower()) and is_contained('hosp', s)) or (is_contained('hopital', x.lower()) and not is_contained('hopital', s)) or (not is_contained('hopital', x.lower()) and is_contained('hopital', s)):
|
|
# continue
|
|
s_vector = vectorizer.fit_transform([s]).toarray() #Else we compute the similarity of s with the original affiiation name
|
|
x_vector = vectorizer.transform([x]).toarray()
|
|
|
|
# Compute similarity between the vectors
|
|
similarity = cosine_similarity(x_vector, s_vector)[0][0]
|
|
if similarity > 0.1:
|
|
similarity_l = 1 - Levenshtein.distance(x, pairs_list[j][0]) / max(len(x), len(pairs_list[j][0]))
|
|
|
|
best.append([x, similarity, similarity_l]) #(similarity+similarity2)/2])
|
|
except:
|
|
KeyError
|
|
|
|
if best:
|
|
# max_numbers = defaultdict(float)
|
|
|
|
|
|
# Assuming best is a list of three-element lists
|
|
# Each element is (string, number1, number2)
|
|
max_numbers = defaultdict(float)
|
|
for item in best:
|
|
string, number1, number2 = item # Unpack the three elements
|
|
max_numbers[string] = max(max_numbers[string], number1)
|
|
|
|
reduced_best = [[string, number1, number2] for string, number1, number2 in best if number1 == max_numbers[string]]
|
|
|
|
# Sort by number1 decreasingly and then by number2 in descending order
|
|
reduced_best.sort(key=lambda x: (x[1], x[2]), reverse=True)
|
|
|
|
result = result + reduced_best
|
|
|
|
univ_list = []
|
|
other_list = []
|
|
|
|
for r in result:
|
|
if is_contained('univ', r[0]):
|
|
univ_list.append(r)
|
|
else:
|
|
other_list.append(r)
|
|
|
|
limit = min(univ_num, candidate_num)
|
|
|
|
if len(univ_list) > limit:
|
|
result = univ_list[:limit] + other_list
|
|
|
|
result_dict = {}
|
|
pairs_dict = {}
|
|
|
|
|
|
for l in pairs_list:
|
|
pairs_dict[l[1]] = l[2]
|
|
|
|
|
|
for p in result:
|
|
result_dict[p[0]] = pairs_dict[p[0]]
|
|
|
|
|
|
result_dict_list = [[y[0], result_dict[y[0]]] for y in result]
|
|
|
|
return result_dict_list
|
|
|
|
|
|
|
|
def Aff_Ids(input, dix_org, dix_mult, dix_city_ror, dix_country_ror, simU, simG):
|
|
|
|
"""
|
|
Matches affiliations in DataFrame 'DF' with names from dictionary 'dix_org' and their ROR_ids based on similarity scores.
|
|
|
|
Args:
|
|
m (int): The number of DOIs to check.
|
|
DF (DataFrame): The input DataFrame containing affiliation data.
|
|
dix_org (dict): A dictionary of names of organizations and their ROR_ids.
|
|
simU (float): Similarity threshold for universities.
|
|
simG (float): Similarity threshold for non-universities.
|
|
|
|
Returns:
|
|
DataFrame: The final DataFrame with matched affiliations and their corresponding similarity scores.
|
|
"""
|
|
df_list = input[1]
|
|
light_aff = input[0]
|
|
vectorizer = CountVectorizer()
|
|
|
|
dix = {} # will store indeces and legalnames of organizations of the DOI { i : [legalname1, legalname2,...]}
|
|
#pairs = []
|
|
result = {}
|
|
pairs = []
|
|
|
|
|
|
def get_keywords(filtered_list):
|
|
# Extract the "keywords" values from the dictionaries in filtered_list
|
|
keywords_list = [entry["keywords"] for entry in filtered_list]
|
|
|
|
return keywords_list
|
|
keywords = get_keywords(df_list)
|
|
|
|
|
|
for k,s in enumerate(keywords):
|
|
similar_k = []
|
|
pairs_k = []
|
|
|
|
if s in dix_org:
|
|
similarity = 1
|
|
similar_k.append(similarity)
|
|
|
|
pairs_k.append((s,s,similarity,dix_org[s]))
|
|
pairs.append((s,s,similarity,dix_org[s]))
|
|
|
|
|
|
if k not in dix:
|
|
dix[k] = [s]
|
|
else:
|
|
dix[k].append(s)
|
|
else:
|
|
|
|
for x in dix_org:
|
|
if is_contained(s, x):
|
|
|
|
x_vector = vectorizer.fit_transform([x]).toarray()
|
|
s_vector = vectorizer.transform([s]).toarray()
|
|
|
|
# Compute similarity between the vectors
|
|
similarity = cosine_similarity(x_vector, s_vector)[0][0]
|
|
if similarity > min(simU, simG):
|
|
if (is_contained('univ', s) and is_contained('univ', x)) and similarity > simU:
|
|
similar_k.append(similarity)
|
|
pairs_k.append((s,x,similarity,dix_org[x]))
|
|
pairs.append((s,x,similarity,dix_org[x]))
|
|
|
|
|
|
if k not in dix:
|
|
dix[k] = [x]
|
|
else:
|
|
dix[k].append(x)
|
|
elif (not is_contained('univ', s) and not is_contained('univ', x)) and similarity > simG:
|
|
similar_k.append(similarity)
|
|
pairs_k.append((s,x,similarity,dix_org[x]))
|
|
pairs.append((s,x,similarity,dix_org[x]))
|
|
|
|
|
|
if k not in dix:
|
|
dix[k] = [x]
|
|
else:
|
|
dix[k].append(x)
|
|
|
|
elif is_contained(x, s):
|
|
if (is_contained('univ', s) and is_contained('univ', x)):
|
|
|
|
s_vector = vectorizer.fit_transform([s]).toarray()
|
|
x_vector = vectorizer.transform([x]).toarray()
|
|
|
|
# Compute similarity between the vectors
|
|
similarity = cosine_similarity(s_vector, x_vector)[0][0]
|
|
if similarity > simU: #max(0.82,sim):
|
|
similar_k.append(similarity)
|
|
pairs_k.append((s,x,similarity,dix_org[x]))
|
|
pairs.append((s,x,similarity,dix_org[x]))
|
|
|
|
if k not in dix:
|
|
dix[k] = [x]
|
|
else:
|
|
dix[k].append(x)
|
|
elif not is_contained('univ', s) and not is_contained('univ', x):
|
|
|
|
s_vector = vectorizer.fit_transform([s]).toarray()
|
|
x_vector = vectorizer.transform([x]).toarray()
|
|
|
|
# Compute similarity between the vectors
|
|
similarity = cosine_similarity(s_vector, x_vector)[0][0]
|
|
if similarity > simG: #max(0.82,sim):
|
|
similar_k.append(similarity)
|
|
pairs_k.append((s,x,similarity,dix_org[x]))
|
|
pairs.append((s,x,similarity,dix_org[x]))
|
|
|
|
if k not in dix:
|
|
dix[k] = [x]
|
|
else:
|
|
dix[k].append(x)
|
|
|
|
result[k] = pairs_k
|
|
|
|
multi = index_multiple_matchings(list(set(pairs)))
|
|
# need_check = list(set([i for i in range(len(multi)) if list(multi.values())[i]>1]))
|
|
# print('here', multi)
|
|
# need_check_keys = [keywords[i] for i in range(len(keywords)) if multi[keywords[i]]>1]
|
|
need_check_keys = []
|
|
for i in range(len(keywords)):
|
|
try:
|
|
if multi[keywords[i]]>1:
|
|
need_check_keys.append(keywords[i])
|
|
except:
|
|
pass
|
|
|
|
best = best_sim_score(light_aff, len(keywords), pairs, multi, simU, simG)
|
|
matched_org = [x[0] for x in best]
|
|
# best_o = []
|
|
# best_s = []
|
|
# best_result = []
|
|
# for x in best:
|
|
# best_o.append([x[i][0] for i in range(len(x))])
|
|
# best_s.append([round(x[i][1],2) for i in range(len(x))])
|
|
# num_mathced = [len(best_s[i]) for i in range(len(need_check))]
|
|
ids = [dix_org[x[0]] for x in best]
|
|
for i,x in enumerate(matched_org):
|
|
# id_list = []
|
|
if dix_mult[x] == 'unique':
|
|
if 'institu' in x and 'univ' in x:
|
|
if dix_city_ror[x][0] not in light_aff and dix_country_ror[x][0] not in light_aff:
|
|
pass
|
|
|
|
else:
|
|
ids[i] = dix_org[x]
|
|
|
|
|
|
if dix_mult[x] != 'unique':
|
|
if x in dix_city_ror:
|
|
match_found = False
|
|
|
|
for city in dix_city_ror[x]:
|
|
if city[0] in light_aff:
|
|
if city[0] not in x:
|
|
ids[i] = city[1]
|
|
match_found = True
|
|
break
|
|
else:
|
|
if light_aff.count(city[0]) >1:
|
|
ids[i] = city[1]
|
|
match_found = True
|
|
break
|
|
|
|
if not match_found:
|
|
for city in dix_city_ror[x]:
|
|
if city[0] in light_aff and city[0] not in x:
|
|
ids[i] = city[1]
|
|
break
|
|
|
|
if not match_found:
|
|
match_found2 = False
|
|
match_found3 = False
|
|
|
|
for country in dix_country_ror[x]:
|
|
if country[0] == 'united states' and (country[0] in light_aff or 'usa' in light_aff):
|
|
ids[i] = country[1]
|
|
match_found2 = True
|
|
match_found3 = True
|
|
break
|
|
|
|
if country[0] == 'united kingdom' and (country[0] in light_aff or 'uk' in light_aff):
|
|
ids[i] = country[1]
|
|
match_found2 = True
|
|
match_found3 = True
|
|
break
|
|
|
|
elif country[0] in light_aff:
|
|
|
|
if country[0] not in x:
|
|
ids[i] = country[1]
|
|
match_found2 = True
|
|
match_found3 = True
|
|
break
|
|
|
|
if not match_found3:
|
|
for country in dix_country_ror[x]:
|
|
if country[0] in light_aff and country[0] in x:
|
|
ids[i] = country[1]
|
|
match_found2 = True
|
|
break
|
|
if not match_found2:
|
|
if 'univ' in x:
|
|
ids[i] = dix_org[x]
|
|
|
|
else:
|
|
for sp in specific:
|
|
if sp in x:
|
|
ids[i] = dix_org[x]
|
|
|
|
|
|
results = [[x[0],x[1], ids[i]] for i,x in enumerate(best)]
|
|
# results_upd = []
|
|
|
|
# for r in results:
|
|
# if 'ror.org' in r[2]:
|
|
# if dix_status[r[2]][0] == 'active':
|
|
# results_upd.append([r[0],r[1], 'ROR', r[2], 'active'])
|
|
# else:
|
|
# if dix_status[r[2]][1] == '':
|
|
# results_upd.append([r[0],r[1], 'ROR', r[2], dix_status[r[2]][0]])
|
|
# else:
|
|
|
|
# results_upd.append([r[0],r[1], 'ROR', r[2], dix_status[r[2]][0]])
|
|
|
|
# results_upd.append([r[0],r[1], 'ROR', dix_status[r[2]][1], 'active'])
|
|
|
|
return results |