D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request 'task9462' (#18) from task9462 into master 

Reviewed-on: #18
This commit is contained in:
Michele Artini 2024-03-13 09:22:51 +01:00
parent 90bb2effc9 7c672594c1
commit 5a8e8f2ade
10 changed files with 718 additions and 599 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

858
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityApiController.java
View File

File diff suppressed because it is too large Load Diff

169
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityService.java
View File

@ -3,6 +3,7 @@ package eu.dnetlib.openaire.community;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedHashSet; import java.util.LinkedHashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -74,21 +75,17 @@ public class CommunityService {
public List<CommunitySummary> listCommunities() { public List<CommunitySummary> listCommunities() {
return dbCommunityRepository.findAll() return dbCommunityRepository.findAll()
.stream() .stream()
.map(CommunityMappingUtils::toCommunitySummary) .map(CommunityMappingUtils::toCommunitySummary)
.collect(Collectors.toList()); .collect(Collectors.toList());
} }
@Transactional @Transactional
public CommunityDetails newCommunity(final CommunityDetails details) throws CommunityException { public CommunityDetails newCommunity(final CommunityDetails details) throws CommunityException {
if (StringUtils.isBlank(details.getId())) { if (StringUtils.isBlank(details.getId())) { throw new CommunityException("Empty Id"); }
throw new CommunityException("Empty Id"); if (dbCommunityRepository.existsById(details.getId())) { throw new CommunityException("Community already exists: " + details.getId()); }
} else if (dbCommunityRepository.existsById(details.getId())) { details.setCreationDate(LocalDateTime.now());
throw new CommunityException("Community already exists: " + details.getId()); return saveCommunity(details);
} else {
details.setCreationDate(LocalDateTime.now());
return saveCommunity(details);
}
} }
@ -115,23 +112,23 @@ public class CommunityService {
@Transactional @Transactional
public Page<CommunityProject> getCommunityProjects(final String id, public Page<CommunityProject> getCommunityProjects(final String id,
final String funder, final String funder,
final String filter, final String filter,
final int page, final int page,
final int size, final int size,
final String orderBy) throws CommunityException { final String orderBy) throws CommunityException {
if (StringUtils.isBlank(id)) { throw new CommunityException("Empty ID"); } if (StringUtils.isBlank(id)) { throw new CommunityException("Empty ID"); }
try { try {
final Sort sort; final Sort sort;
if (StringUtils.isBlank(orderBy)) { if (StringUtils.isBlank(orderBy)) {
sort = Sort.by("projectName"); sort = Sort.by("projectName");
} else if (orderBy.equalsIgnoreCase("funder")) { } else if ("funder".equalsIgnoreCase(orderBy)) {
sort = Sort.by("projectFunder").and(Sort.by("projectName")); sort = Sort.by("projectFunder").and(Sort.by("projectName"));
} else if (orderBy.equalsIgnoreCase("grantId")) { } else if ("grantId".equalsIgnoreCase(orderBy)) {
sort = Sort.by("projectCode"); sort = Sort.by("projectCode");
} else if (orderBy.equalsIgnoreCase("acronym")) { } else if ("acronym".equalsIgnoreCase(orderBy)) {
sort = Sort.by("projectAcronym"); sort = Sort.by("projectAcronym");
} else if (orderBy.equalsIgnoreCase("openaireId")) { } else if ("openaireId".equalsIgnoreCase(orderBy)) {
sort = Sort.by("projectId"); sort = Sort.by("projectId");
} else { } else {
sort = Sort.by("projectName"); sort = Sort.by("projectName");
@ -140,10 +137,9 @@ public class CommunityService {
final PageRequest pageable = PageRequest.of(page, size, sort); final PageRequest pageable = PageRequest.of(page, size, sort);
if (StringUtils.isAllBlank(filter, funder)) { if (StringUtils.isAllBlank(filter, funder)) {
return dbProjectRepository.findByCommunity(id, pageable).map(CommunityMappingUtils::toCommunityProject); return dbProjectRepository.findByCommunity(id, pageable).map(CommunityMappingUtils::toCommunityProject);
} else {
final Specification<DbProject> projSpec = prepareProjectSpec(id, funder, filter);
return dbProjectRepository.findAll(projSpec, pageable).map(CommunityMappingUtils::toCommunityProject);
} }
final Specification<DbProject> projSpec = prepareProjectSpec(id, funder, filter);
return dbProjectRepository.findAll(projSpec, pageable).map(CommunityMappingUtils::toCommunityProject);
} catch (final Throwable e) { } catch (final Throwable e) {
log.error(e); log.error(e);
throw new CommunityException(e); throw new CommunityException(e);
@ -191,8 +187,8 @@ public class CommunityService {
public void addCommunityProjects(final String id, final CommunityProject... projects) throws CommunityException { public void addCommunityProjects(final String id, final CommunityProject... projects) throws CommunityException {
try { try {
final List<DbProject> list = Arrays.stream(projects) final List<DbProject> list = Arrays.stream(projects)
.map(p -> CommunityMappingUtils.toDbProject(id, p)) .map(p -> CommunityMappingUtils.toDbProject(id, p))
.collect(Collectors.toList()); .collect(Collectors.toList());
dbProjectRepository.saveAll(list); dbProjectRepository.saveAll(list);
} catch (final Throwable e) { } catch (final Throwable e) {
@ -204,56 +200,75 @@ public class CommunityService {
@Transactional @Transactional
public void removeCommunityProjects(final String id, final String... ids) { public void removeCommunityProjects(final String id, final String... ids) {
final List<DbProjectPK> list = Arrays.stream(ids) final List<DbProjectPK> list = Arrays.stream(ids)
.map(projectId -> new DbProjectPK(id, projectId)) .map(projectId -> new DbProjectPK(id, projectId))
.collect(Collectors.toList()); .collect(Collectors.toList());
dbProjectRepository.deleteAllById(list); dbProjectRepository.deleteAllById(list);
} }
public List<CommunityContentprovider> getCommunityContentproviders(final String id) { public List<CommunityContentprovider> getCommunityDatasources(final String id) {
return dbDatasourceRepository.findByCommunity(id) return dbDatasourceRepository.findByCommunity(id)
.stream() .stream()
.map(CommunityMappingUtils::toCommunityContentprovider) .map(CommunityMappingUtils::toCommunityContentprovider)
.collect(Collectors.toList()); .collect(Collectors.toList());
}
public List<CommunityContentprovider> getCommunityDatasourcesWithDeposit(final String id, final boolean deposit) {
return dbDatasourceRepository.findByCommunityAndDeposit(id, deposit)
.stream()
.map(CommunityMappingUtils::toCommunityContentprovider)
.collect(Collectors.toList());
} }
@Transactional @Transactional
public void addCommunityContentProviders(final String id, final CommunityContentprovider... contentproviders) { public CommunityContentprovider updateCommunityDatasourcesDeposit(final String id, final String dsId, final Boolean deposit, final String message) {
return dbDatasourceRepository.findById(new DbDatasourcePK(id, dsId))
.map(ds -> {
ds.setDeposit(deposit != null ? deposit : false);
ds.setMessage(message);
return ds;
})
.map(CommunityMappingUtils::toCommunityContentprovider)
.orElseThrow(() -> new ResourceNotFoundException("Community and/or Datasource not found"));
}
@Transactional
public void addCommunityDatasources(final String id, final CommunityContentprovider... contentproviders) {
final List<DbDatasource> list = Arrays.stream(contentproviders) final List<DbDatasource> list = Arrays.stream(contentproviders)
.map(cp -> CommunityMappingUtils.toDbDatasource(id, cp)) .map(cp -> CommunityMappingUtils.toDbDatasource(id, cp))
.collect(Collectors.toList()); .collect(Collectors.toList());
dbDatasourceRepository.saveAll(list); dbDatasourceRepository.saveAll(list);
} }
@Transactional @Transactional
public void removeCommunityContentProviders(final String id, final String... ids) { public void removeCommunityDatasources(final String id, final String... ids) {
final List<DbDatasourcePK> list = Arrays.stream(ids) final List<DbDatasourcePK> list = Arrays.stream(ids)
.map(dsId -> new DbDatasourcePK(id, dsId)) .map(dsId -> new DbDatasourcePK(id, dsId))
.collect(Collectors.toList()); .collect(Collectors.toList());
dbDatasourceRepository.deleteAllById(list); dbDatasourceRepository.deleteAllById(list);
} }
@Transactional @Transactional
public void removeCommunityOrganizations(final String id, final String... orgNames) { public void removeCommunityOrganizations(final String id, final String... orgNames) {
final List<DbSupportOrgPK> list = Arrays.stream(orgNames) final List<DbSupportOrgPK> list = Arrays.stream(orgNames)
.map(name -> new DbSupportOrgPK(id, name)) .map(name -> new DbSupportOrgPK(id, name))
.collect(Collectors.toList()); .collect(Collectors.toList());
dbSupportOrgRepository.deleteAllById(list); dbSupportOrgRepository.deleteAllById(list);
} }
@Transactional @Transactional
public List<CommunityOrganization> getCommunityOrganizations(final String id) { public List<CommunityOrganization> getCommunityOrganizations(final String id) {
return dbSupportOrgRepository.findByCommunity(id) return dbSupportOrgRepository.findByCommunity(id)
.stream() .stream()
.map(CommunityMappingUtils::toCommunityOrganization) .map(CommunityMappingUtils::toCommunityOrganization)
.collect(Collectors.toList()); .collect(Collectors.toList());
} }
@Transactional @Transactional
public void addCommunityOrganizations(final String id, final CommunityOrganization... orgs) { public void addCommunityOrganizations(final String id, final CommunityOrganization... orgs) {
final List<DbSupportOrg> list = Arrays.stream(orgs) final List<DbSupportOrg> list = Arrays.stream(orgs)
.map(o -> CommunityMappingUtils.toDbSupportOrg(id, o)) .map(o -> CommunityMappingUtils.toDbSupportOrg(id, o))
.collect(Collectors.toList()); .collect(Collectors.toList());
dbSupportOrgRepository.saveAll(list); dbSupportOrgRepository.saveAll(list);
} }
@ -266,43 +281,43 @@ public class CommunityService {
@Transactional @Transactional
public List<SubCommunity> getSubCommunities(final String id) { public List<SubCommunity> getSubCommunities(final String id) {
return dbSubCommunityRepository.findByCommunity(id) return dbSubCommunityRepository.findByCommunity(id)
.stream() .stream()
.map(CommunityMappingUtils::toSubCommunity) .map(CommunityMappingUtils::toSubCommunity)
.collect(Collectors.toList()); .collect(Collectors.toList());
} }
@Transactional @Transactional
public void addSubCommunities(final String id, final SubCommunity... subs) { public void addSubCommunities(final String id, final SubCommunity... subs) {
final List<DbSubCommunity> list = Arrays.stream(subs) final List<DbSubCommunity> list = Arrays.stream(subs)
.map(s -> CommunityMappingUtils.toDbSubCommunity(id, s)) .map(s -> CommunityMappingUtils.toDbSubCommunity(id, s))
.collect(Collectors.toList()); .collect(Collectors.toList());
dbSubCommunityRepository.saveAll(list); dbSubCommunityRepository.saveAll(list);
} }
@Transactional @Transactional
public CommunityDetails addCommunitySubjects(final String id, final String... subjects) { public CommunityDetails addCommunitySubjects(final String id, final String... subjects) {
return modifyElementToArrayField(id, c -> c.getSubjects(), (c, subs) -> c.setSubjects(subs), false, subjects); return modifyElementToArrayField(id, DbCommunity::getSubjects, DbCommunity::setSubjects, false, subjects);
} }
public CommunityDetails removeCommunitySubjects(final String id, final String... subjects) { public CommunityDetails removeCommunitySubjects(final String id, final String... subjects) {
return modifyElementToArrayField(id, c -> c.getSubjects(), (c, subs) -> c.setSubjects(subs), true, subjects); return modifyElementToArrayField(id, DbCommunity::getSubjects, DbCommunity::setSubjects, true, subjects);
} }
public CommunityDetails addCommunityFOS(final String id, final String... foss) { public CommunityDetails addCommunityFOS(final String id, final String... foss) {
return modifyElementToArrayField(id, c -> c.getFos(), (c, fos) -> c.setFos(fos), false, foss); return modifyElementToArrayField(id, DbCommunity::getFos, DbCommunity::setFos, false, foss);
} }
public CommunityDetails removeCommunityFOS(final String id, final String... foss) { public CommunityDetails removeCommunityFOS(final String id, final String... foss) {
return modifyElementToArrayField(id, c -> c.getFos(), (c, fos) -> c.setFos(fos), true, foss); return modifyElementToArrayField(id, DbCommunity::getFos, DbCommunity::setFos, true, foss);
} }
public CommunityDetails addCommunitySDG(final String id, final String... sdgs) { public CommunityDetails addCommunitySDG(final String id, final String... sdgs) {
return modifyElementToArrayField(id, c -> c.getSdg(), (c, sdg) -> c.setSdg(sdg), false, sdgs); return modifyElementToArrayField(id, DbCommunity::getSdg, DbCommunity::setSdg, false, sdgs);
} }
public CommunityDetails removeCommunitySDG(final String id, final String... sdgs) { public CommunityDetails removeCommunitySDG(final String id, final String... sdgs) {
return modifyElementToArrayField(id, c -> c.getSdg(), (c, sdg) -> c.setSdg(sdg), true, sdgs); return modifyElementToArrayField(id, DbCommunity::getSdg, DbCommunity::setSdg, true, sdgs);
} }
@Transactional @Transactional
@ -342,25 +357,19 @@ public class CommunityService {
} }
public CommunityDetails removeCommunityZenodoCommunity(final String id, final String zenodoCommunity, final boolean isMain) { public CommunityDetails removeCommunityZenodoCommunity(final String id, final String zenodoCommunity, final boolean isMain) {
if (isMain) { if (isMain) { return updateElementToSimpleField(id, DbCommunity::setMainZenodoCommunity, null); }
return updateElementToSimpleField(id, (c, val) -> c.setMainZenodoCommunity(val), null); return modifyElementToArrayField(id, DbCommunity::getOtherZenodoCommunities, DbCommunity::setOtherZenodoCommunities, true, zenodoCommunity);
} else {
return modifyElementToArrayField(id, c -> c.getOtherZenodoCommunities(), (c, arr) -> c.setOtherZenodoCommunities(arr), true, zenodoCommunity);
}
} }
public CommunityDetails addCommunityZenodoCommunity(final String id, final String zenodoCommunity, final boolean isMain) { public CommunityDetails addCommunityZenodoCommunity(final String id, final String zenodoCommunity, final boolean isMain) {
if (isMain) { if (isMain) { return updateElementToSimpleField(id, DbCommunity::setMainZenodoCommunity, zenodoCommunity); }
return updateElementToSimpleField(id, (c, val) -> c.setMainZenodoCommunity(val), zenodoCommunity); return modifyElementToArrayField(id, DbCommunity::getOtherZenodoCommunities, DbCommunity::setOtherZenodoCommunities, false, zenodoCommunity);
} else {
return modifyElementToArrayField(id, c -> c.getOtherZenodoCommunities(), (c, arr) -> c.setOtherZenodoCommunities(arr), false, zenodoCommunity);
}
} }
@Transactional @Transactional
private CommunityDetails updateElementToSimpleField(final String id, private CommunityDetails updateElementToSimpleField(final String id,
final BiConsumer<DbCommunity, String> setter, final BiConsumer<DbCommunity, String> setter,
final String value) { final String value) {
final DbCommunity dbEntry = dbCommunityRepository.findById(id).orElseThrow(() -> new ResourceNotFoundException("Community not found: " + id)); final DbCommunity dbEntry = dbCommunityRepository.findById(id).orElseThrow(() -> new ResourceNotFoundException("Community not found: " + id));
setter.accept(dbEntry, value); setter.accept(dbEntry, value);
dbEntry.setLastUpdateDate(LocalDateTime.now()); dbEntry.setLastUpdateDate(LocalDateTime.now());
@ -370,19 +379,17 @@ public class CommunityService {
@Transactional @Transactional
private CommunityDetails modifyElementToArrayField(final String id, private CommunityDetails modifyElementToArrayField(final String id,
final Function<DbCommunity, String[]> getter, final Function<DbCommunity, String[]> getter,
final BiConsumer<DbCommunity, String[]> setter, final BiConsumer<DbCommunity, String[]> setter,
final boolean remove, final boolean remove,
final String... values) { final String... values) {
final DbCommunity dbEntry = dbCommunityRepository.findById(id).orElseThrow(() -> new ResourceNotFoundException("Community not found: " + id)); final DbCommunity dbEntry = dbCommunityRepository.findById(id).orElseThrow(() -> new ResourceNotFoundException("Community not found: " + id));
final Set<String> tmpList = new LinkedHashSet<>(); final Set<String> tmpList = new LinkedHashSet<>();
final String[] oldValues = getter.apply(dbEntry); final String[] oldValues = getter.apply(dbEntry);
if (oldValues != null) { if (oldValues != null) {
for (final String s : oldValues) { Collections.addAll(tmpList, oldValues);
tmpList.add(s);
}
} }
if (remove) { if (remove) {
tmpList.removeAll(Arrays.asList(values)); tmpList.removeAll(Arrays.asList(values));
@ -407,16 +414,16 @@ public class CommunityService {
@Transactional @Transactional
public Map<String, Set<String>> getPropagationOrganizationCommunityMap() { public Map<String, Set<String>> getPropagationOrganizationCommunityMap() {
return dbOrganizationRepository.findAll() return dbOrganizationRepository.findAll()
.stream() .stream()
.collect(Collectors.groupingBy(DbOrganization::getOrgId, Collectors.mapping(DbOrganization::getCommunity, Collectors.toSet()))); .collect(Collectors.groupingBy(DbOrganization::getOrgId, Collectors.mapping(DbOrganization::getCommunity, Collectors.toSet())));
} }
@Transactional @Transactional
public Set<String> getPropagationOrganizationsForCommunity(final String communityId) { public Set<String> getPropagationOrganizationsForCommunity(final String communityId) {
return dbOrganizationRepository.findByCommunity(communityId) return dbOrganizationRepository.findByCommunity(communityId)
.stream() .stream()
.map(DbOrganization::getOrgId) .map(DbOrganization::getOrgId)
.collect(Collectors.toSet()); .collect(Collectors.toSet());
} }
@Transactional @Transactional
@ -454,8 +461,8 @@ public class CommunityService {
final List<IISConfigurationEntry> res = new ArrayList<>(); final List<IISConfigurationEntry> res = new ArrayList<>();
res.add(dbCommunityRepository.findById(id) res.add(dbCommunityRepository.findById(id)
.map(CommunityMappingUtils::asIISConfigurationEntry) .map(CommunityMappingUtils::asIISConfigurationEntry)
.orElseThrow(() -> new ResourceNotFoundException("Community not found: " + id))); .orElseThrow(() -> new ResourceNotFoundException("Community not found: " + id)));
for (final DbSubCommunity subc : dbSubCommunityRepository.findByCommunity(id)) { for (final DbSubCommunity subc : dbSubCommunityRepository.findByCommunity(id)) {
res.add(CommunityMappingUtils.asIISConfigurationEntry(subc)); res.add(CommunityMappingUtils.asIISConfigurationEntry(subc));

160
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/importer/CommunityImporterService.java
View File

@ -6,6 +6,7 @@ import java.util.Arrays;
import java.util.Base64; import java.util.Base64;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry;
import java.util.function.Function; import java.util.function.Function;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
@ -115,23 +116,20 @@ public class CommunityImporterService {
public List<DbOrganization> importPropagationOrganizationsFromProfile(final String xml, final boolean simulation) throws Exception { public List<DbOrganization> importPropagationOrganizationsFromProfile(final String xml, final boolean simulation) throws Exception {
final String json = DocumentHelper.parseText(xml) final String json = DocumentHelper.parseText(xml)
.selectSingleNode("//NODE[@name='setPropagationOrganizationCommunityMap']//PARAM[@name='parameterValue']") .selectSingleNode("//NODE[@name='setPropagationOrganizationCommunityMap']//PARAM[@name='parameterValue']")
.getText(); .getText();
final List<DbOrganization> list = new ObjectMapper() final List<DbOrganization> list = new ObjectMapper()
.readValue(json, new TypeReference<Map<String, List<String>>>() {}) .readValue(json, new TypeReference<Map<String, List<String>>>() {})
.entrySet() .entrySet()
.stream()
.flatMap(e -> e.getValue()
.stream() .stream()
.map(community -> { .flatMap(e -> e.getValue()
if (e.getKey().contains("|")) { .stream()
return new DbOrganization(community, StringUtils.substringAfter(e.getKey(), "|")); .map(community -> {
} else { if (e.getKey().contains("|")) { return new DbOrganization(community, StringUtils.substringAfter(e.getKey(), "|")); }
return new DbOrganization(community, e.getKey()); return new DbOrganization(community, e.getKey());
} }))
})) .collect(Collectors.toList());
.collect(Collectors.toList());
if (!simulation) { if (!simulation) {
list.forEach(o -> { list.forEach(o -> {
@ -153,69 +151,69 @@ public class CommunityImporterService {
final CommunityDetails community = asCommunityDetails(context); final CommunityDetails community = asCommunityDetails(context);
final List<CommunityContentprovider> datasources = final List<CommunityContentprovider> datasources =
getCommunityInfo(context, CONTENTPROVIDERS_ID_SUFFIX, c -> asCommunityDataprovider(context.getId(), c)) getCommunityInfo(context, CONTENTPROVIDERS_ID_SUFFIX, c -> asCommunityDataprovider(context.getId(), c))
.stream() .stream()
.map(o -> { .map(o -> {
if (o.getOpenaireId() == null) { if (o.getOpenaireId() == null) {
log.warn("Openaire ID is missing, organization: " + o.getOfficialname()); log.warn("Openaire ID is missing, organization: " + o.getOfficialname());
} else if (o.getOpenaireId().contains("|")) { } else if (o.getOpenaireId().contains("|")) {
o.setOpenaireId(StringUtils.substringAfter(o.getOpenaireId(), "|")); o.setOpenaireId(StringUtils.substringAfter(o.getOpenaireId(), "|"));
} }
return o; return o;
}) })
.filter(o -> o.getOpenaireId() != null) .filter(o -> o.getOpenaireId() != null)
.collect(Collectors.toList()); .collect(Collectors.toList());
final List<CommunityProject> projects = final List<CommunityProject> projects =
getCommunityInfo(context, PROJECTS_ID_SUFFIX, c -> asCommunityProject(context.getId(), c)) getCommunityInfo(context, PROJECTS_ID_SUFFIX, c -> asCommunityProject(context.getId(), c))
.stream() .stream()
.map(p -> { .map(p -> {
if (p.getOpenaireId() == null) { if (p.getOpenaireId() == null) {
if (p.getFunder().equalsIgnoreCase("EC")) { if ("EC".equalsIgnoreCase(p.getFunder())) {
final String ns = findNamespaceForECProject(p.getGrantId()); final String ns = findNamespaceForECProject(p.getGrantId());
if (ns != null) { if (ns != null) {
p.setOpenaireId(ns + "::" + Hashing.md5(p.getGrantId())); p.setOpenaireId(ns + "::" + Hashing.md5(p.getGrantId()));
} else { } else {
log.warn("EC project not in the db: " + p.getGrantId()); log.warn("EC project not in the db: " + p.getGrantId());
}
} else if ("NSF".equalsIgnoreCase(p.getFunder())) {
p.setOpenaireId("nsf_________::" + Hashing.md5(p.getGrantId()));
} else if ("NIH".equalsIgnoreCase(p.getFunder())) {
p.setOpenaireId("nih_________::" + Hashing.md5(p.getGrantId()));
} else {
log.warn("Openaire ID is missing, funder: " + p.getFunder());
}
} else if (p.getOpenaireId().contains("|")) {
p.setOpenaireId(StringUtils.substringAfter(p.getOpenaireId(), "|"));
} }
} else if (p.getFunder().equalsIgnoreCase("NSF")) { return p;
p.setOpenaireId("nsf_________::" + Hashing.md5(p.getGrantId())); })
} else if (p.getFunder().equalsIgnoreCase("NIH")) { .filter(p -> p.getOpenaireId() != null)
p.setOpenaireId("nih_________::" + Hashing.md5(p.getGrantId())); .collect(Collectors.toList());
} else {
log.warn("Openaire ID is missing, funder: " + p.getFunder());
}
} else if (p.getOpenaireId().contains("|")) {
p.setOpenaireId(StringUtils.substringAfter(p.getOpenaireId(), "|"));
}
return p;
})
.filter(p -> p.getOpenaireId() != null)
.collect(Collectors.toList());
final List<CommunityOrganization> orgs = final List<CommunityOrganization> orgs =
getCommunityInfo(context, ORGANIZATION_ID_SUFFIX, c -> asCommunityOrganization(context.getId(), c)); getCommunityInfo(context, ORGANIZATION_ID_SUFFIX, c -> asCommunityOrganization(context.getId(), c));
final List<String> otherZenodoCommunities = final List<String> otherZenodoCommunities =
getCommunityInfo(context, ZENODOCOMMUNITY_ID_SUFFIX, c -> asZenodoCommunity(c)); getCommunityInfo(context, ZENODOCOMMUNITY_ID_SUFFIX, CommunityImporterService::asZenodoCommunity);
community.setOtherZenodoCommunities(otherZenodoCommunities); community.setOtherZenodoCommunities(otherZenodoCommunities);
final List<SubCommunity> subs = context.getCategories() final List<SubCommunity> subs = context.getCategories()
.entrySet() .entrySet()
.stream() .stream()
.filter(e -> !e.getKey().equals(context.getId() + CONTENTPROVIDERS_ID_SUFFIX)) .filter(e -> !(context.getId() + CONTENTPROVIDERS_ID_SUFFIX).equals(e.getKey()))
.filter(e -> !e.getKey().equals(context.getId() + PROJECTS_ID_SUFFIX)) .filter(e -> !(context.getId() + PROJECTS_ID_SUFFIX).equals(e.getKey()))
.filter(e -> !e.getKey().equals(context.getId() + ORGANIZATION_ID_SUFFIX)) .filter(e -> !(context.getId() + ORGANIZATION_ID_SUFFIX).equals(e.getKey()))
.filter(e -> !e.getKey().equals(context.getId() + ZENODOCOMMUNITY_ID_SUFFIX)) .filter(e -> !(context.getId() + ZENODOCOMMUNITY_ID_SUFFIX).equals(e.getKey()))
.map(e -> e.getValue()) .map(Entry::getValue)
.map(cat -> asSubCommunities(context.getId(), null, cat.getLabel(), cat.getConcepts())) .map(cat -> asSubCommunities(context.getId(), null, cat.getLabel(), cat.getConcepts()))
.flatMap(List::stream) .flatMap(List::stream)
.collect(Collectors.toList()); .collect(Collectors.toList());
service.saveCommunity(community); service.saveCommunity(community);
service.addCommunityProjects(context.getId(), projects.toArray(new CommunityProject[projects.size()])); service.addCommunityProjects(context.getId(), projects.toArray(new CommunityProject[projects.size()]));
service.addCommunityContentProviders(context.getId(), datasources.toArray(new CommunityContentprovider[datasources.size()])); service.addCommunityDatasources(context.getId(), datasources.toArray(new CommunityContentprovider[datasources.size()]));
service.addCommunityOrganizations(context.getId(), orgs.toArray(new CommunityOrganization[orgs.size()])); service.addCommunityOrganizations(context.getId(), orgs.toArray(new CommunityOrganization[orgs.size()]));
service.addSubCommunities(context.getId(), subs.toArray(new SubCommunity[subs.size()])); service.addSubCommunities(context.getId(), subs.toArray(new SubCommunity[subs.size()]));
} catch ( } catch (
@ -226,14 +224,14 @@ public class CommunityImporterService {
} }
private <R> List<R> getCommunityInfo(final Context context, final String idSuffix, final Function<Concept, R> mapping) private <R> List<R> getCommunityInfo(final Context context, final String idSuffix, final Function<Concept, R> mapping)
throws CommunityException { throws CommunityException {
if (context != null) { if (context != null) {
final Map<String, Category> categories = context.getCategories(); final Map<String, Category> categories = context.getCategories();
final Category category = categories.get(context.getId() + idSuffix); final Category category = categories.get(context.getId() + idSuffix);
if (category != null) { return category.getConcepts() if (category != null) { return category.getConcepts()
.stream() .stream()
.map(mapping) .map(mapping)
.collect(Collectors.toList()); } .collect(Collectors.toList()); }
} }
return Lists.newArrayList(); return Lists.newArrayList();
} }
@ -302,6 +300,8 @@ public class CommunityImporterService {
d.setOfficialname(firstValue(CCONTENTPROVIDER_OFFICIALNAME, p)); d.setOfficialname(firstValue(CCONTENTPROVIDER_OFFICIALNAME, p));
d.setEnabled(BooleanUtils.toBoolean(firstValue(CCONTENTPROVIDER_ENABLED, p))); d.setEnabled(BooleanUtils.toBoolean(firstValue(CCONTENTPROVIDER_ENABLED, p)));
d.setSelectioncriteria(SelectionCriteria.fromJson(firstValue(CCONTENTPROVIDER_SELCRITERIA, p))); d.setSelectioncriteria(SelectionCriteria.fromJson(firstValue(CCONTENTPROVIDER_SELCRITERIA, p)));
d.setDeposit(false);
d.setMessage(null);
return d; return d;
} }
@ -339,7 +339,7 @@ public class CommunityImporterService {
private String findNamespaceForECProject(final String code) { private String findNamespaceForECProject(final String code) {
final List<String> list = final List<String> list =
jdbcTemplate.queryForList("SELECT substr(id, 1, 12) from projects where code = ? and id like 'corda%'", String.class, code); jdbcTemplate.queryForList("SELECT substr(id, 1, 12) from projects where code = ? and id like 'corda%'", String.class, code);
return list.isEmpty() ? null : list.get(0); return list.isEmpty() ? null : list.get(0);
} }
@ -355,11 +355,11 @@ public class CommunityImporterService {
private static List<String> splitValues(final Stream<String> stream, final String separator) { private static List<String> splitValues(final Stream<String> stream, final String separator) {
return stream.map(s -> s.split(separator)) return stream.map(s -> s.split(separator))
.map(Arrays::asList) .map(Arrays::asList)
.flatMap(List::stream) .flatMap(List::stream)
.filter(StringUtils::isNotBlank) .filter(StringUtils::isNotBlank)
.map(StringUtils::trim) .map(StringUtils::trim)
.collect(Collectors.toList()); .collect(Collectors.toList());
} }
private static String firstValue(final String name, final List<Param> params) { private static String firstValue(final String name, final List<Param> params) {
@ -372,13 +372,13 @@ public class CommunityImporterService {
private static Stream<String> asValues(final String name, final List<Param> params) { private static Stream<String> asValues(final String name, final List<Param> params) {
return params == null ? Stream.empty() return params == null ? Stream.empty()
: params.stream() : params.stream()
.filter(p -> p != null) .filter(p -> p != null)
.filter(p -> StringUtils.isNotBlank(p.getName())) .filter(p -> StringUtils.isNotBlank(p.getName()))
.filter(p -> p.getName().trim().equals(name.trim())) .filter(p -> p.getName().trim().equals(name.trim()))
.map(Param::getValue) .map(Param::getValue)
.map(StringUtils::trim) .map(StringUtils::trim)
.distinct(); .distinct();
} }
protected DbOrganizationRepository getDbOrganizationRepository() { protected DbOrganizationRepository getDbOrganizationRepository() {

22
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/model/DbDatasource.java
View File

@ -40,6 +40,12 @@ public class DbDatasource implements Serializable {
@Column(name = "constraints") @Column(name = "constraints")
private SelectionCriteria constraints; private SelectionCriteria constraints;
@Column(name = "deposit")
private Boolean deposit;
@Column(name = "message")
private String message;
public DbDatasource() {} public DbDatasource() {}
public DbDatasource(final String community, final String dsId, final String dsName, final String dsOfficialName, final SelectionCriteria constraints) { public DbDatasource(final String community, final String dsId, final String dsName, final String dsOfficialName, final SelectionCriteria constraints) {
@ -98,4 +104,20 @@ public class DbDatasource implements Serializable {
this.constraints = constraints; this.constraints = constraints;
} }
public Boolean getDeposit() {
return deposit;
}
public void setDeposit(final Boolean deposit) {
this.deposit = deposit;
}
public String getMessage() {
return message;
}
public void setMessage(final String message) {
this.message = message;
}
} }

39
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/model/DepositionInfo.java Normal file
View File

@ -0,0 +1,39 @@
package eu.dnetlib.openaire.community.model;
import java.io.Serializable;
public class DepositionInfo implements Serializable {
private static final long serialVersionUID = 7538663287451167904L;
private String openaireId;
private Boolean deposit;
private String message;
public String getOpenaireId() {
return openaireId;
}
public void setOpenaireId(final String openaireId) {
this.openaireId = openaireId;
}
public Boolean getDeposit() {
return deposit;
}
public void setDeposit(final Boolean deposit) {
this.deposit = deposit;
}
public String getMessage() {
return message;
}
public void setMessage(final String message) {
this.message = message;
}
}

3
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/repository/DbDatasourceRepository.java
View File

@ -13,5 +13,8 @@ public interface DbDatasourceRepository extends JpaRepository<DbDatasource, DbDa
List<DbDatasource> findByCommunity(String community); List<DbDatasource> findByCommunity(String community);
List<DbDatasource> findByCommunityAndDeposit(String community, boolean deposit);
void deleteByCommunity(String id); void deleteByCommunity(String id);
} }

8
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/utils/CommunityMappingUtils.java
View File

@ -189,6 +189,8 @@ public class CommunityMappingUtils {
ccp.setOfficialname(dbEntry.getDsOfficialName()); ccp.setOfficialname(dbEntry.getDsOfficialName());
ccp.setSelectioncriteria(dbEntry.getConstraints()); ccp.setSelectioncriteria(dbEntry.getConstraints());
ccp.setEnabled(dbEntry.getEnabled() != null ? dbEntry.getEnabled() : true); ccp.setEnabled(dbEntry.getEnabled() != null ? dbEntry.getEnabled() : true);
ccp.setDeposit(dbEntry.getDeposit() != null ? dbEntry.getDeposit() : false);
ccp.setMessage(dbEntry.getMessage());
return ccp; return ccp;
} }
@ -200,6 +202,8 @@ public class CommunityMappingUtils {
ds.setDsOfficialName(provider.getOfficialname()); ds.setDsOfficialName(provider.getOfficialname());
ds.setConstraints(provider.getSelectioncriteria()); ds.setConstraints(provider.getSelectioncriteria());
ds.setEnabled(provider.isEnabled()); ds.setEnabled(provider.isEnabled());
ds.setDeposit(provider.getDeposit() != null ? provider.getDeposit() : false);
ds.setMessage(provider.getMessage());
return ds; return ds;
} }
@ -262,8 +266,8 @@ public class CommunityMappingUtils {
public static LocalDateTime asLocalDateTime(final Date date) { public static LocalDateTime asLocalDateTime(final Date date) {
return date.toInstant() return date.toInstant()
.atZone(ZoneId.systemDefault()) .atZone(ZoneId.systemDefault())
.toLocalDateTime(); .toLocalDateTime();
} }
private static String[] toStringArray(final List<String> list) { private static String[] toStringArray(final List<String> list) {

4
apps/dnet-exporter-api/src/main/resources/sql/community-schema.sql
View File

@ -44,8 +44,10 @@ CREATE TABLE community_datasources (
ds_id text NOT NULL, ds_id text NOT NULL,
ds_name text NOT NULL, ds_name text NOT NULL,
ds_officialname text NOT NULL, ds_officialname text NOT NULL,
enabled boolean NOT NULL DEFAULT true; enabled boolean NOT NULL DEFAULT true
constraints jsonb, constraints jsonb,
deposit boolean NOT NULL DEFAULT false,
message text,
PRIMARY KEY (community, ds_id) PRIMARY KEY (community, ds_id)
); );

10
apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/community/importer/CommunityImporterServiceTest.java
View File

@ -60,9 +60,9 @@ class CommunityImporterServiceTest {
// list.forEach(System.out::println); // list.forEach(System.out::println);
assertEquals(245, list.size()); assertEquals(245, list.size());
assertEquals(1, list.stream().filter(o -> o.getOrgId().equals("openorgs____::9dd5545aacd3d8019e00c3f837269746")).count()); assertEquals(1, list.stream().filter(o -> "openorgs____::9dd5545aacd3d8019e00c3f837269746".equals(o.getOrgId())).count());
assertEquals(2, list.stream().filter(o -> o.getOrgId().equals("openorgs____::d11f981828c485cd23d93f7f24f24db1")).count()); assertEquals(2, list.stream().filter(o -> "openorgs____::d11f981828c485cd23d93f7f24f24db1".equals(o.getOrgId())).count());
assertEquals(14, list.stream().filter(o -> o.getCommunity().equals("beopen")).count()); assertEquals(14, list.stream().filter(o -> "beopen".equals(o.getCommunity())).count());
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
@ -85,7 +85,7 @@ class CommunityImporterServiceTest {
Mockito.verify(service, Mockito.times(1)).saveCommunity(detailsCapture.capture()); Mockito.verify(service, Mockito.times(1)).saveCommunity(detailsCapture.capture());
Mockito.verify(service, Mockito.times(1)).addCommunityProjects(Mockito.anyString(), projectsCapture.capture()); Mockito.verify(service, Mockito.times(1)).addCommunityProjects(Mockito.anyString(), projectsCapture.capture());
Mockito.verify(service, Mockito.times(1)).addCommunityContentProviders(Mockito.anyString(), datasourcesCapture.capture()); Mockito.verify(service, Mockito.times(1)).addCommunityDatasources(Mockito.anyString(), datasourcesCapture.capture());
Mockito.verify(service, Mockito.times(1)).addCommunityOrganizations(Mockito.anyString(), orgsCapture.capture()); Mockito.verify(service, Mockito.times(1)).addCommunityOrganizations(Mockito.anyString(), orgsCapture.capture());
Mockito.verify(service, Mockito.times(1)).addSubCommunities(Mockito.anyString(), subCommunitiesCapture.capture()); Mockito.verify(service, Mockito.times(1)).addSubCommunities(Mockito.anyString(), subCommunitiesCapture.capture());
@ -132,7 +132,7 @@ class CommunityImporterServiceTest {
Mockito.verify(service, Mockito.times(1)).saveCommunity(detailsCapture.capture()); Mockito.verify(service, Mockito.times(1)).saveCommunity(detailsCapture.capture());
Mockito.verify(service, Mockito.times(1)).addCommunityProjects(Mockito.anyString(), projectsCapture.capture()); Mockito.verify(service, Mockito.times(1)).addCommunityProjects(Mockito.anyString(), projectsCapture.capture());
Mockito.verify(service, Mockito.times(1)).addCommunityContentProviders(Mockito.anyString(), datasourcesCapture.capture()); Mockito.verify(service, Mockito.times(1)).addCommunityDatasources(Mockito.anyString(), datasourcesCapture.capture());
Mockito.verify(service, Mockito.times(1)).addCommunityOrganizations(Mockito.anyString(), orgsCapture.capture()); Mockito.verify(service, Mockito.times(1)).addCommunityOrganizations(Mockito.anyString(), orgsCapture.capture());
Mockito.verify(service, Mockito.times(1)).addSubCommunities(Mockito.anyString(), subCommunitiesCapture.capture()); Mockito.verify(service, Mockito.times(1)).addSubCommunities(Mockito.anyString(), subCommunitiesCapture.capture());

44
libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/community/CommunityContentprovider.java
View File

@ -34,6 +34,12 @@ public class CommunityContentprovider {
@Schema(description = "content provider selection criteria", required = false) @Schema(description = "content provider selection criteria", required = false)
private SelectionCriteria selectioncriteria; private SelectionCriteria selectioncriteria;
@Schema(description = "suggested for deposition", required = false)
private Boolean deposit;
@Schema(description = "message for the deposition page", required = false)
private String message;
public String getOpenaireId() { public String getOpenaireId() {
return openaireId; return openaireId;
} }
@ -75,7 +81,7 @@ public class CommunityContentprovider {
} }
public SelectionCriteria getSelectioncriteria() { public SelectionCriteria getSelectioncriteria() {
return this.selectioncriteria; return selectioncriteria;
} }
public void setSelectioncriteria(final SelectionCriteria selectioncriteria) { public void setSelectioncriteria(final SelectionCriteria selectioncriteria) {
@ -101,17 +107,33 @@ public class CommunityContentprovider {
public String toString() { public String toString() {
final StringBuilder builder = new StringBuilder(); final StringBuilder builder = new StringBuilder();
builder.append("CommunityContentprovider [\n\topenaireId = ") builder.append("CommunityContentprovider [\n\topenaireId = ")
.append(openaireId) .append(openaireId)
.append(",\n\tcommunityId = ") .append(",\n\tcommunityId = ")
.append(communityId) .append(communityId)
.append(",\n\tname = ") .append(",\n\tname = ")
.append(name) .append(name)
.append(",\n\tofficialname = ") .append(",\n\tofficialname = ")
.append(officialname) .append(officialname)
.append(",\n\tselectioncriteria = ") .append(",\n\tselectioncriteria = ")
.append(selectioncriteria) .append(selectioncriteria)
.append("\n]"); .append("\n]");
return builder.toString(); return builder.toString();
} }
public Boolean getDeposit() {
return deposit;
}
public void setDeposit(final Boolean deposit) {
this.deposit = deposit;
}
public String getMessage() {
return message;
}
public void setMessage(final String message) {
this.message = message;
}
} }