D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

importers

This commit is contained in:
Michele Artini 2023-06-28 14:23:56 +02:00
parent 340a577430
commit 8ac49d1dab
9 changed files with 391 additions and 298 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClient.java
View File

@ -4,6 +4,7 @@ import java.io.IOException;
import java.util.List;
import java.util.Map;
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo;
import eu.dnetlib.openaire.exporter.model.context.Context;
@ -54,4 +55,6 @@ public interface ISClient {
@Deprecated
void updateConceptParamNoEscape(String id, String name, String value);
String getProfile(String profileId) throws ISLookUpException;
}

5
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java
View File

@ -374,4 +374,9 @@ public class ISClientImpl implements ISClient {
log.debug("dropped dsManager IS cache");
}
@Override
public String getProfile(final String profileId) throws ISLookUpException {
return isLookUpService.getResourceProfile(profileId);
}
}

300
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityImporterController.java
View File

@ -1,112 +1,36 @@
package eu.dnetlib.openaire.community;
import java.io.IOException;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import eu.dnetlib.openaire.common.ISClient;
import eu.dnetlib.openaire.community.db.CommunityService;
import eu.dnetlib.openaire.community.db.CommunityImporterService;
import eu.dnetlib.openaire.community.db.model.DbOrganization;
import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider;
import eu.dnetlib.openaire.exporter.model.community.CommunityDetails;
import eu.dnetlib.openaire.exporter.model.community.CommunityMembershipType;
import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization;
import eu.dnetlib.openaire.exporter.model.community.CommunityProject;
import eu.dnetlib.openaire.exporter.model.community.CommunityStatus;
import eu.dnetlib.openaire.exporter.model.community.CommunityType;
import eu.dnetlib.openaire.exporter.model.community.SubCommunity;
import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionCriteria;
import eu.dnetlib.openaire.exporter.model.context.Category;
import eu.dnetlib.openaire.exporter.model.context.Concept;
import eu.dnetlib.openaire.exporter.model.context.Context;
import eu.dnetlib.openaire.exporter.model.context.Param;
@RestController
public class CommunityImporterController {
private final static String pattern = "yyyy-MM-dd'T'hh:mm:ss";
// common
private final static String OPENAIRE_ID = "openaireId";
private final static String PIPE_SEPARATOR = "||";
private final static String ID_SEPARATOR = "::";
private final static String CSV_DELIMITER = ",";
private final static String CLABEL = "label";
// id suffixes
private final static String PROJECTS_ID_SUFFIX = ID_SEPARATOR + "projects";
private final static String CONTENTPROVIDERS_ID_SUFFIX = ID_SEPARATOR + "contentproviders";
private final static String ZENODOCOMMUNITY_ID_SUFFIX = ID_SEPARATOR + "zenodocommunities";
private final static String ORGANIZATION_ID_SUFFIX = ID_SEPARATOR + "organizations";
// community summary
private final static String CSUMMARY_DESCRIPTION = "description";
private final static String CSUMMARY_LOGOURL = "logourl";
private final static String CSUMMARY_STATUS = "status";
private final static String CSUMMARY_NAME = "name";
private final static String CSUMMARY_MANAGER = "manager";
private final static String CSUMMARY_ZENODOC = "zenodoCommunity";
// community profile
private final static String CPROFILE_SUBJECT = "subject";
private final static String CPROFILE_CREATIONDATE = "creationdate";
private final static String CPROFILE_FOS = "fos";
private final static String CPROFILE_SDG = "sdg";
private final static String CPROFILE_ADVANCED_CONSTRAINT = "advancedConstraints";
private final static String CPROFILE_REMOVE_CONSTRAINT = "removeConstraints";
// community project
private final static String CPROJECT_FUNDER = "funder";
private final static String CPROJECT_NUMBER = "CD_PROJECT_NUMBER";
private final static String CPROJECT_FULLNAME = "projectfullname";
private final static String CPROJECT_ACRONYM = "acronym";
// community content provider
private final static String CCONTENTPROVIDER_NAME = "name";
private final static String CCONTENTPROVIDER_OFFICIALNAME = "officialname";
private final static String CCONTENTPROVIDER_ENABLED = "enabled";
private final static String CCONTENTPROVIDERENABLED_DEFAULT = "true";
private final static String CCONTENTPROVIDER_SELCRITERIA = "selcriteria";
// community zenodo community
private final static String CZENODOCOMMUNITY_ID = "zenodoid";
// community organization
private final static String CORGANIZATION_NAME = "name";
private final static String CORGANIZATION_LOGOURL = "logourl";
private final static String CORGANIZATION_WEBSITEURL = "websiteurl";
private static final Log log = LogFactory.getLog(CommunityImporterController.class);
public final static Set<String> communityBlackList = Sets.newHashSet("fet-fp7", "fet-h2020");
@Autowired
private CommunityService service;
private CommunityImporterService importer;
@Autowired
private ISClient isClient;
@GetMapping("/community_importer/import")
@GetMapping("/community_importer/communities")
public List<String> importProfiles() throws CommunityException {
final Map<String, Context> contextMap = getContextMap();
@ -116,12 +40,19 @@ public class CommunityImporterController {
.collect(Collectors.toList());
list.forEach(id -> {
importCommunity(contextMap.get(id));
importer.importCommunity(contextMap.get(id));
});
return list;
}
@GetMapping("/community_importer/propagationOrgs")
public List<DbOrganization> importPropagationOrgs(@RequestParam final String profileId,
@RequestParam(required = false, defaultValue = "false") final boolean simulation) throws Exception {
final String xml = isClient.getProfile(profileId);
return importer.importPropagationOrganizationsFromProfile(xml, simulation);
}
private Map<String, Context> getContextMap() throws CommunityException {
try {
return isClient.getCommunityContextMap();
@ -130,209 +61,4 @@ public class CommunityImporterController {
}
}
public <R> List<R> getCommunityInfo(final Context context, final String idSuffix, final Function<Concept, R> mapping)
throws CommunityException {
if (context != null) {
final Map<String, Category> categories = context.getCategories();
final Category category = categories.get(context.getId() + idSuffix);
if (category != null) { return category.getConcepts()
.stream()
.map(mapping)
.collect(Collectors.toList()); }
}
return Lists.newArrayList();
}
private void importCommunity(final Context context) {
try {
final CommunityDetails community = asCommunityDetails(context);
final List<CommunityContentprovider> datasources =
getCommunityInfo(context, CONTENTPROVIDERS_ID_SUFFIX, c -> asCommunityDataprovider(context.getId(), c));
final List<CommunityProject> projects =
getCommunityInfo(context, PROJECTS_ID_SUFFIX, c -> asCommunityProject(context.getId(), c));
final List<CommunityOrganization> orgs =
getCommunityInfo(context, ORGANIZATION_ID_SUFFIX, c -> asCommunityOrganization(context.getId(), c));
final List<String> otherZenodoCimmunities =
getCommunityInfo(context, ZENODOCOMMUNITY_ID_SUFFIX, c -> asZenodoCommunity(c));
community.setOtherZenodoCommunities(otherZenodoCimmunities);
final List<SubCommunity> subs = context.getCategories()
.entrySet()
.stream()
.filter(e -> !e.getKey().equals(context.getId() + CONTENTPROVIDERS_ID_SUFFIX))
.filter(e -> !e.getKey().equals(context.getId() + PROJECTS_ID_SUFFIX))
.filter(e -> !e.getKey().equals(context.getId() + ORGANIZATION_ID_SUFFIX))
.filter(e -> !e.getKey().equals(context.getId() + ZENODOCOMMUNITY_ID_SUFFIX))
.map(e -> e.getValue())
.map(cat -> asSubCommunities(context.getId(), null, cat.getLabel(), cat.getConcepts()))
.flatMap(List::stream)
.collect(Collectors.toList());
service.saveCommunity(community);
service.addCommunityProjectList(context.getId(), projects);
service.addCommunityContentProvidersList(context.getId(), datasources);
service.addCommunityOrganizationList(context.getId(), orgs);
service.addSubCommunityList(subs);
} catch (final Exception e) {
throw new RuntimeException("Error importing community: " + context.getId(), e);
}
}
private static CommunityDetails asCommunityDetails(final Context c) {
final CommunityDetails details = new CommunityDetails();
details.setId(c.getId());
details.setShortName(c.getLabel());
details.setLastUpdateDate(convertToLocalDateTime(c.getLastUpdateDate()));
details.setCreationDate(convertToLocalDateTime(c.getCreationDate()));
details.setQueryId(c.getId() + PIPE_SEPARATOR + c.getLabel());
details.setType(CommunityType.valueOf(c.getType()));
details.setMembership(CommunityMembershipType.open);
// TODO: Considerare anche i campi claim
final Map<String, List<Param>> params = c.getParams();
if (params.containsKey(CSUMMARY_DESCRIPTION)) {
details.setDescription(asCsv(params.get(CSUMMARY_DESCRIPTION)));
}
if (params.containsKey(CSUMMARY_LOGOURL)) {
details.setLogoUrl(asCsv(params.get(CSUMMARY_LOGOURL)));
}
if (params.containsKey(CSUMMARY_STATUS)) {
details.setStatus(CommunityStatus.valueOf(firstValue(params, CSUMMARY_STATUS)));
}
if (params.containsKey(CSUMMARY_NAME)) {
details.setName(asCsv(params.get(CSUMMARY_NAME)));
} else {
details.setName(c.getLabel());
}
if (params.containsKey(CSUMMARY_ZENODOC)) {
details.setZenodoCommunity(asCsv(params.get(CSUMMARY_ZENODOC)));
}
if (params.containsKey(CPROFILE_SUBJECT)) {
details.setSubjects(splitValues(asValues(params.get(CPROFILE_SUBJECT)), CSV_DELIMITER));
}
if (params.containsKey(CPROFILE_FOS)) {
details.setFos(splitValues(asValues(params.get(CPROFILE_FOS)), CSV_DELIMITER));
}
if (params.containsKey(CPROFILE_SDG)) {
details.setSdg(splitValues(asValues(params.get(CPROFILE_SDG)), CSV_DELIMITER));
}
if (params.containsKey(CPROFILE_ADVANCED_CONSTRAINT)) {
// In the map the string is the serialization of the json representing the selection criteria so it is a valid json
details.setAdvancedConstraints(SelectionCriteria.fromJson(asCsv(params.get(CPROFILE_ADVANCED_CONSTRAINT))));
}
if (params.containsKey(CPROFILE_REMOVE_CONSTRAINT)) {
// In the map the string is the serialization of the json representing the selection criteria so it is a valid json
details.setRemoveConstraints(SelectionCriteria.fromJson(asCsv(params.get(CPROFILE_REMOVE_CONSTRAINT))));
}
if (params.containsKey(CPROFILE_CREATIONDATE)) {
try {
final Date d = org.apache.commons.lang3.time.DateUtils.parseDate(asCsv(params.get(CPROFILE_CREATIONDATE)), pattern);
details.setCreationDate(convertToLocalDateTime(d));
} catch (final Exception e) {
log.debug("Exception on date format: " + e.getMessage());
}
}
return details;
}
private static CommunityProject asCommunityProject(final String communityId, final Concept c) {
final Map<String, List<Param>> p = c.getParams();
final CommunityProject project = new CommunityProject();
project.setCommunityId(communityId);
project.setOpenaireId(firstValue(p, OPENAIRE_ID));
project.setFunder(firstValue(p, CPROJECT_FUNDER));
project.setGrantId(firstValue(p, CPROJECT_NUMBER));
project.setName(firstValue(p, CPROJECT_FULLNAME));
project.setAcronym(firstValue(p, CPROJECT_ACRONYM));
return project;
}
private static CommunityContentprovider asCommunityDataprovider(final String communityId, final Concept c) {
final Map<String, List<Param>> p = c.getParams();
final CommunityContentprovider d = new CommunityContentprovider();
d.setCommunityId(communityId);
d.setOpenaireId(firstValue(p, OPENAIRE_ID));
d.setName(firstValue(p, CCONTENTPROVIDER_NAME));
d.setOfficialname(firstValue(p, CCONTENTPROVIDER_OFFICIALNAME));
d.setSelectioncriteria(SelectionCriteria.fromJson(firstValue(p, CCONTENTPROVIDER_SELCRITERIA)));
return d;
}
private static CommunityOrganization asCommunityOrganization(final String id, final Concept c) {
final Map<String, List<Param>> p = c.getParams();
final CommunityOrganization o = new CommunityOrganization();
o.setCommunityId(id);
// o.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
o.setName(firstValue(p, CORGANIZATION_NAME));
o.setLogo_url(getDecodedUrl(firstValue(p, CORGANIZATION_LOGOURL)));
o.setWebsite_url(getDecodedUrl(firstValue(p, CORGANIZATION_WEBSITEURL)));
return o;
}
private static String asZenodoCommunity(final Concept c) {
return firstValue(c.getParams(), CZENODOCOMMUNITY_ID);
}
private static List<SubCommunity> asSubCommunities(final String communityId, final String parent, final String category, final List<Concept> concepts) {
final List<SubCommunity> list = new ArrayList<>();
for (final Concept c : concepts) {
final SubCommunity sc = new SubCommunity();
sc.setSubCommunityId(c.getId());
sc.setCommunityId(communityId);
sc.setParent(parent);
sc.setCategory(category);
sc.setLabel(c.getLabel());
sc.setParams(c.getParams());
list.add(sc);
list.addAll(asSubCommunities(communityId, c.getId(), category, c.getConcepts()));
}
return list;
}
private static String getDecodedUrl(final String encoded_url) {
if (encoded_url == null) { return encoded_url; }
return new String(Base64.getDecoder().decode(encoded_url));
}
private static List<String> splitValues(final Stream<String> stream, final String separator) {
return stream.map(s -> s.split(separator))
.map(Arrays::asList)
.flatMap(List::stream)
.filter(StringUtils::isNotBlank)
.map(StringUtils::trim)
.collect(Collectors.toList());
}
private static String firstValue(final Map<String, List<Param>> p, final String paramName) {
return asValues(p.get(paramName)).findFirst().orElse(null);
}
private static String asCsv(final List<Param> params) {
return asValues(params)
.collect(Collectors.joining(CSV_DELIMITER));
}
private static Stream<String> asValues(final List<Param> params) {
return params == null ? Stream.empty()
: params.stream()
.map(Param::getValue)
.map(StringUtils::trim)
.distinct();
}
private static LocalDateTime convertToLocalDateTime(final Date date) {
return date.toInstant()
.atZone(ZoneId.systemDefault())
.toLocalDateTime();
}
}

335
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/db/CommunityImporterService.java Normal file
View File

@ -0,0 +1,335 @@
package eu.dnetlib.openaire.community.db;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.transaction.Transactional;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.dom4j.DocumentHelper;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
import eu.dnetlib.openaire.community.db.model.DbOrganization;
import eu.dnetlib.openaire.community.db.repository.DbOrganizationRepository;
import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
import eu.dnetlib.openaire.exporter.model.community.CommunityClaimType;
import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider;
import eu.dnetlib.openaire.exporter.model.community.CommunityDetails;
import eu.dnetlib.openaire.exporter.model.community.CommunityMembershipType;
import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization;
import eu.dnetlib.openaire.exporter.model.community.CommunityProject;
import eu.dnetlib.openaire.exporter.model.community.CommunityStatus;
import eu.dnetlib.openaire.exporter.model.community.CommunityType;
import eu.dnetlib.openaire.exporter.model.community.SubCommunity;
import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionCriteria;
import eu.dnetlib.openaire.exporter.model.context.Category;
import eu.dnetlib.openaire.exporter.model.context.Concept;
import eu.dnetlib.openaire.exporter.model.context.Context;
import eu.dnetlib.openaire.exporter.model.context.Param;
@Service
public class CommunityImporterService {
private final static String pattern = "yyyy-MM-dd'T'hh:mm:ss";
// common
private final static String OPENAIRE_ID = "openaireId";
private final static String PIPE_SEPARATOR = "||";
private final static String ID_SEPARATOR = "::";
private final static String CSV_DELIMITER = ",";
private final static String CLABEL = "label";
// id suffixes
private final static String PROJECTS_ID_SUFFIX = ID_SEPARATOR + "projects";
private final static String CONTENTPROVIDERS_ID_SUFFIX = ID_SEPARATOR + "contentproviders";
private final static String ZENODOCOMMUNITY_ID_SUFFIX = ID_SEPARATOR + "zenodocommunities";
private final static String ORGANIZATION_ID_SUFFIX = ID_SEPARATOR + "organizations";
// community summary
private final static String CSUMMARY_DESCRIPTION = "description";
private final static String CSUMMARY_LOGOURL = "logourl";
private final static String CSUMMARY_STATUS = "status";
private final static String CSUMMARY_NAME = "name";
private final static String CSUMMARY_MANAGER = "manager";
private final static String CSUMMARY_ZENODOC = "zenodoCommunity";
// community profile
private final static String CPROFILE_SUBJECT = "subject";
private final static String CPROFILE_CREATIONDATE = "creationdate";
private final static String CPROFILE_FOS = "fos";
private final static String CPROFILE_SDG = "sdg";
private final static String CPROFILE_ADVANCED_CONSTRAINT = "advancedConstraints";
private final static String CPROFILE_REMOVE_CONSTRAINT = "removeConstraints";
// community project
private final static String CPROJECT_FUNDER = "funder";
private final static String CPROJECT_NUMBER = "CD_PROJECT_NUMBER";
private final static String CPROJECT_FULLNAME = "projectfullname";
private final static String CPROJECT_ACRONYM = "acronym";
// community content provider
private final static String CCONTENTPROVIDER_NAME = "name";
private final static String CCONTENTPROVIDER_OFFICIALNAME = "officialname";
private final static String CCONTENTPROVIDER_ENABLED = "enabled";
private final static String CCONTENTPROVIDERENABLED_DEFAULT = "true";
private final static String CCONTENTPROVIDER_SELCRITERIA = "selcriteria";
// community zenodo community
private final static String CZENODOCOMMUNITY_ID = "zenodoid";
// community organization
private final static String CORGANIZATION_NAME = "name";
private final static String CORGANIZATION_LOGOURL = "logourl";
private final static String CORGANIZATION_WEBSITEURL = "websiteurl";
@Autowired
private DbOrganizationRepository dbOrganizationRepository;
@Autowired
private CommunityService service;
private static final Log log = LogFactory.getLog(CommunityImporterService.class);
@Transactional
public List<DbOrganization> importPropagationOrganizationsFromProfile(final String xml, final boolean simulation) throws Exception {
final String json = DocumentHelper.parseText(xml)
.selectSingleNode("//NODE[@name='setPropagationOrganizationCommunityMap']//PARAM[@name='parameterValue']")
.getText();
final List<DbOrganization> list = new ObjectMapper()
.readValue(json, new TypeReference<Map<String, List<String>>>() {})
.entrySet()
.stream()
.flatMap(e -> e.getValue().stream().map(community -> new DbOrganization(community, e.getKey())))
.collect(Collectors.toList());
if (!simulation) {
dbOrganizationRepository.saveAll(list);
}
return list;
}
@Transactional
public void importCommunity(final Context context) {
try {
final CommunityDetails community = asCommunityDetails(context);
final List<CommunityContentprovider> datasources =
getCommunityInfo(context, CONTENTPROVIDERS_ID_SUFFIX, c -> asCommunityDataprovider(context.getId(), c));
final List<CommunityProject> projects =
getCommunityInfo(context, PROJECTS_ID_SUFFIX, c -> asCommunityProject(context.getId(), c));
final List<CommunityOrganization> orgs =
getCommunityInfo(context, ORGANIZATION_ID_SUFFIX, c -> asCommunityOrganization(context.getId(), c));
final List<String> otherZenodoCimmunities =
getCommunityInfo(context, ZENODOCOMMUNITY_ID_SUFFIX, c -> asZenodoCommunity(c));
community.setOtherZenodoCommunities(otherZenodoCimmunities);
final List<SubCommunity> subs = context.getCategories()
.entrySet()
.stream()
.filter(e -> !e.getKey().equals(context.getId() + CONTENTPROVIDERS_ID_SUFFIX))
.filter(e -> !e.getKey().equals(context.getId() + PROJECTS_ID_SUFFIX))
.filter(e -> !e.getKey().equals(context.getId() + ORGANIZATION_ID_SUFFIX))
.filter(e -> !e.getKey().equals(context.getId() + ZENODOCOMMUNITY_ID_SUFFIX))
.map(e -> e.getValue())
.map(cat -> asSubCommunities(context.getId(), null, cat.getLabel(), cat.getConcepts()))
.flatMap(List::stream)
.collect(Collectors.toList());
service.saveCommunity(community);
service.addCommunityProjectList(context.getId(), projects);
service.addCommunityContentProvidersList(context.getId(), datasources);
service.addCommunityOrganizationList(context.getId(), orgs);
service.addSubCommunityList(subs);
} catch (final Exception e) {
throw new RuntimeException("Error importing community: " + context.getId(), e);
}
}
private <R> List<R> getCommunityInfo(final Context context, final String idSuffix, final Function<Concept, R> mapping)
throws CommunityException {
if (context != null) {
final Map<String, Category> categories = context.getCategories();
final Category category = categories.get(context.getId() + idSuffix);
if (category != null) { return category.getConcepts()
.stream()
.map(mapping)
.collect(Collectors.toList()); }
}
return Lists.newArrayList();
}
private static CommunityDetails asCommunityDetails(final Context c) {
final CommunityDetails details = new CommunityDetails();
details.setId(c.getId());
details.setShortName(c.getLabel());
details.setLastUpdateDate(convertToLocalDateTime(c.getLastUpdateDate()));
details.setCreationDate(convertToLocalDateTime(c.getCreationDate()));
details.setQueryId(c.getId() + PIPE_SEPARATOR + c.getLabel());
details.setType(CommunityType.valueOf(c.getType()));
details.setMembership(CommunityMembershipType.open);
details.setClaim(CommunityClaimType.all);
final Map<String, List<Param>> params = c.getParams();
if (params.containsKey(CSUMMARY_DESCRIPTION)) {
details.setDescription(asCsv(params.get(CSUMMARY_DESCRIPTION)));
}
if (params.containsKey(CSUMMARY_LOGOURL)) {
details.setLogoUrl(asCsv(params.get(CSUMMARY_LOGOURL)));
}
if (params.containsKey(CSUMMARY_STATUS)) {
details.setStatus(CommunityStatus.valueOf(firstValue(params, CSUMMARY_STATUS)));
}
if (params.containsKey(CSUMMARY_NAME)) {
details.setName(asCsv(params.get(CSUMMARY_NAME)));
} else {
details.setName(c.getLabel());
}
if (params.containsKey(CSUMMARY_ZENODOC)) {
details.setZenodoCommunity(asCsv(params.get(CSUMMARY_ZENODOC)));
}
if (params.containsKey(CPROFILE_SUBJECT)) {
details.setSubjects(splitValues(asValues(params.get(CPROFILE_SUBJECT)), CSV_DELIMITER));
}
if (params.containsKey(CPROFILE_FOS)) {
details.setFos(splitValues(asValues(params.get(CPROFILE_FOS)), CSV_DELIMITER));
}
if (params.containsKey(CPROFILE_SDG)) {
details.setSdg(splitValues(asValues(params.get(CPROFILE_SDG)), CSV_DELIMITER));
}
if (params.containsKey(CPROFILE_ADVANCED_CONSTRAINT)) {
// In the map the string is the serialization of the json representing the selection criteria so it is a valid json
details.setAdvancedConstraints(SelectionCriteria.fromJson(asCsv(params.get(CPROFILE_ADVANCED_CONSTRAINT))));
}
if (params.containsKey(CPROFILE_REMOVE_CONSTRAINT)) {
// In the map the string is the serialization of the json representing the selection criteria so it is a valid json
details.setRemoveConstraints(SelectionCriteria.fromJson(asCsv(params.get(CPROFILE_REMOVE_CONSTRAINT))));
}
if (params.containsKey(CPROFILE_CREATIONDATE)) {
try {
final Date d = org.apache.commons.lang3.time.DateUtils.parseDate(asCsv(params.get(CPROFILE_CREATIONDATE)), pattern);
details.setCreationDate(convertToLocalDateTime(d));
} catch (final Exception e) {
log.debug("Exception on date format: " + e.getMessage());
}
}
return details;
}
private static CommunityProject asCommunityProject(final String communityId, final Concept c) {
final Map<String, List<Param>> p = c.getParams();
final CommunityProject project = new CommunityProject();
project.setCommunityId(communityId);
project.setOpenaireId(firstValue(p, OPENAIRE_ID));
project.setFunder(firstValue(p, CPROJECT_FUNDER));
project.setGrantId(firstValue(p, CPROJECT_NUMBER));
project.setName(firstValue(p, CPROJECT_FULLNAME));
project.setAcronym(firstValue(p, CPROJECT_ACRONYM));
return project;
}
private static CommunityContentprovider asCommunityDataprovider(final String communityId, final Concept c) {
final Map<String, List<Param>> p = c.getParams();
final CommunityContentprovider d = new CommunityContentprovider();
d.setCommunityId(communityId);
d.setOpenaireId(firstValue(p, OPENAIRE_ID));
d.setName(firstValue(p, CCONTENTPROVIDER_NAME));
d.setOfficialname(firstValue(p, CCONTENTPROVIDER_OFFICIALNAME));
d.setSelectioncriteria(SelectionCriteria.fromJson(firstValue(p, CCONTENTPROVIDER_SELCRITERIA)));
return d;
}
private static CommunityOrganization asCommunityOrganization(final String id, final Concept c) {
final Map<String, List<Param>> p = c.getParams();
final CommunityOrganization o = new CommunityOrganization();
o.setCommunityId(id);
// o.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
o.setName(firstValue(p, CORGANIZATION_NAME));
o.setLogo_url(getDecodedUrl(firstValue(p, CORGANIZATION_LOGOURL)));
o.setWebsite_url(getDecodedUrl(firstValue(p, CORGANIZATION_WEBSITEURL)));
return o;
}
private static String asZenodoCommunity(final Concept c) {
return firstValue(c.getParams(), CZENODOCOMMUNITY_ID);
}
private static List<SubCommunity> asSubCommunities(final String communityId, final String parent, final String category, final List<Concept> concepts) {
final List<SubCommunity> list = new ArrayList<>();
for (final Concept c : concepts) {
final SubCommunity sc = new SubCommunity();
sc.setSubCommunityId(c.getId());
sc.setCommunityId(communityId);
sc.setParent(parent);
sc.setCategory(category);
sc.setLabel(c.getLabel());
sc.setParams(c.getParams());
sc.setClaim(c.isClaim());
list.add(sc);
list.addAll(asSubCommunities(communityId, c.getId(), category, c.getConcepts()));
}
return list;
}
private static String getDecodedUrl(final String encoded_url) {
if (encoded_url == null) { return encoded_url; }
return new String(Base64.getDecoder().decode(encoded_url));
}
private static List<String> splitValues(final Stream<String> stream, final String separator) {
return stream.map(s -> s.split(separator))
.map(Arrays::asList)
.flatMap(List::stream)
.filter(StringUtils::isNotBlank)
.map(StringUtils::trim)
.collect(Collectors.toList());
}
private static String firstValue(final Map<String, List<Param>> p, final String paramName) {
return asValues(p.get(paramName)).findFirst().orElse(null);
}
private static String asCsv(final List<Param> params) {
return asValues(params)
.collect(Collectors.joining(CSV_DELIMITER));
}
private static Stream<String> asValues(final List<Param> params) {
return params == null ? Stream.empty()
: params.stream()
.map(Param::getValue)
.map(StringUtils::trim)
.distinct();
}
private static LocalDateTime convertToLocalDateTime(final Date date) {
return date.toInstant()
.atZone(ZoneId.systemDefault())
.toLocalDateTime();
}
}

9
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/db/CommunityService.java
View File

@ -48,12 +48,9 @@ import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionC
public class CommunityService {
// TODO
// 1) Import tramite profili:
// 1.1) Completare CommunityImporterController
// 2) Subcommunities: visualizzazione tramite le context api
// 3) Importare le orgs dal wf di provision, ad esempio:
// https://services.openaire.eu/is/mvc/ui/isManager.do#/profile/4801c33c-66ca-4ab6-af64-aa812194ec61_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl
// 4) Verificare Tickets: #8835, #8854, #6483, #3259, #3494
// 1) Subcommunities: visualizzazione tramite le context api
// 2) Test di import (context e wf)
// 3) Verificare Tickets: #8835, #8854, #6483, #3259, #3494
@Autowired
private DbCommunityRepository dbCommunityRepository;

14
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/db/model/DbSubCommunity.java
View File

@ -1,6 +1,7 @@
package eu.dnetlib.openaire.community.db.model;
import java.io.Serializable;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@ -45,11 +46,14 @@ public class DbSubCommunity implements Serializable {
@Type(type = "jsonb")
@Column(name = "params")
private Map<String, List<Param>> params;
private Map<String, List<Param>> params = new LinkedHashMap<>();
@Column(name = "parent")
private String parent;
@Column(name = "claim")
private boolean claim = false;
public String getId() {
return id;
}
@ -97,4 +101,12 @@ public class DbSubCommunity implements Serializable {
public void setParent(final String parent) {
this.parent = parent;
}
public boolean isClaim() {
return claim;
}
public void setClaim(final boolean claim) {
this.claim = claim;
}
}

2
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/db/utils/CommunityMappingUtils.java
View File

@ -199,6 +199,7 @@ public class CommunityMappingUtils {
dbsc.setLabel(sub.getLabel());
dbsc.setParams(sub.getParams());
dbsc.setParent(sub.getParent());
dbsc.setClaim(sub.isClaim());
return dbsc;
}
@ -210,6 +211,7 @@ public class CommunityMappingUtils {
sc.setLabel(sub.getLabel());
sc.setParams(sub.getParams());
sc.setParent(sub.getParent());
sc.setClaim(sub.isClaim());
return sc;
}

2
apps/dnet-exporter-api/src/main/resources/sql/community-schema.sql
View File

@ -1,3 +1,4 @@
DROP TABLE IF EXISTS community_subs;
DROP TABLE IF EXISTS community_projects;
DROP TABLE IF EXISTS community_datasources;
DROP TABLE IF EXISTS community_support_orgs;
@ -64,6 +65,7 @@ CREATE TABLE community_subs (
community text NOT NULL REFERENCES communities(id),
label text NOT NULL,
category text NOT NULL,
claim boolean NOT NULL DEFAULT false,
params jsonb,
parent text REFERENCES community_subs(sub_id) -- NULL for the first level
);

11
libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/community/SubCommunity.java
View File

@ -30,6 +30,9 @@ public class SubCommunity {
@Schema(description = "the parameters of the subCommunity", required = true)
private Map<String, List<Param>> params = new LinkedHashMap<>();
@Schema(description = "it supports the claims", required = true)
private boolean claim = false;
public String getSubCommunityId() {
return subCommunityId;
}
@ -78,4 +81,12 @@ public class SubCommunity {
this.params = map;
}
public boolean isClaim() {
return claim;
}
public void setClaim(final boolean claim) {
this.claim = claim;
}
}