D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

importer

This commit is contained in:
Michele Artini 2023-07-03 12:15:37 +02:00
parent 60b7d82d47
commit 6f873b1e45
2 changed files with 33 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityService.java
View File

@ -51,8 +51,6 @@ import eu.dnetlib.openaire.exporter.model.context.ConceptSummary;
public class CommunityService {
// TODO: Verificare Tickets: #8835, #8854, #6483, #3259, #3494
// L'IMPORT DI ALCUNI PROGETTI FALLISCE perche' non hanno openaireID
// L'IMPORT DI ALCUNE ORGS DAL PROVISION WF FALLISCE perche' il communityID non esiste
@Autowired
private DbCommunityRepository dbCommunityRepository;

36
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/importer/CommunityImporterService.java
View File

@ -19,12 +19,14 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.dom4j.DocumentHelper;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
import eu.dnetlib.miscutils.functional.hash.Hashing;
import eu.dnetlib.openaire.community.CommunityService;
import eu.dnetlib.openaire.community.model.DbOrganization;
import eu.dnetlib.openaire.community.repository.DbOrganizationRepository;
@ -105,6 +107,9 @@ public class CommunityImporterService {
@Autowired
private CommunityService service;
@Autowired
private JdbcTemplate jdbcTemplate;
private static final Log log = LogFactory.getLog(CommunityImporterService.class);
public List<DbOrganization> importPropagationOrganizationsFromProfile(final String xml, final boolean simulation) throws Exception {
@ -141,10 +146,28 @@ public class CommunityImporterService {
final List<CommunityContentprovider> datasources =
getCommunityInfo(context, CONTENTPROVIDERS_ID_SUFFIX, c -> asCommunityDataprovider(context.getId(), c));
// TODO: REMOVE THIS FILTER ?
final List<CommunityProject> projects =
getCommunityInfo(context, PROJECTS_ID_SUFFIX, c -> asCommunityProject(context.getId(), c))
.stream()
.map(p -> {
if (p.getOpenaireId() == null) {
if (p.getFunder().equalsIgnoreCase("EC")) {
final String ns = findNamespaceForECProject(p.getGrantId());
if (ns != null) {
p.setOpenaireId(ns + "::" + Hashing.md5(p.getGrantId()));
} else {
log.error("EC project not in the db: " + p.getGrantId());
}
} else if (p.getFunder().equalsIgnoreCase("NSF")) {
p.setOpenaireId("nsf_________::" + Hashing.md5(p.getGrantId()));
} else if (p.getFunder().equalsIgnoreCase("NIH")) {
p.setOpenaireId("nih_________::" + Hashing.md5(p.getGrantId()));
} else {
log.warn("Openaire ID is missing, funder: " + p.getFunder());
}
}
return p;
})
.filter(p -> p.getOpenaireId() != null)
.collect(Collectors.toList());
@ -173,7 +196,9 @@ public class CommunityImporterService {
service.addCommunityContentProvidersList(context.getId(), datasources);
service.addCommunityOrganizationList(context.getId(), orgs);
service.addSubCommunityList(subs);
} catch (final Exception e) {
} catch (
final Exception e) {
throw new RuntimeException("Error importing community: " + context.getId(), e);
}
}
@ -278,7 +303,6 @@ public class CommunityImporterService {
final Map<String, List<Param>> p = c.getParams();
final CommunityOrganization o = new CommunityOrganization();
o.setCommunityId(id);
// o.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
o.setName(firstValue(p, CORGANIZATION_NAME));
o.setLogo_url(getDecodedUrl(firstValue(p, CORGANIZATION_LOGOURL)));
o.setWebsite_url(getDecodedUrl(firstValue(p, CORGANIZATION_WEBSITEURL)));
@ -306,6 +330,12 @@ public class CommunityImporterService {
return list;
}
private String findNamespaceForECProject(final String code) {
final List<String> list =
jdbcTemplate.queryForList("SELECT substr(id, 1, 12) from projects where code = ? and id like 'corda%'", String.class, code);
return list.isEmpty() ? null : list.get(0);
}
private static String getDecodedUrl(final String encoded_url) {
if (encoded_url == null) { return encoded_url; }
return new String(Base64.getDecoder().decode(encoded_url));