D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

new_model_for_communities #15

Merged
michele.artini merged 81 commits from new_model_for_communities into master 2023-10-24 08:12:26 +02:00
Conversation 0 Commits 81 Files Changed 70 +391 -298
9 changed files with 391 additions and 298 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 8ac49d1dab - Show all commits

3
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClient.java
View File

@ -4,6 +4,7 @@ import java.io.IOException;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo; import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo;
import eu.dnetlib.openaire.exporter.model.context.Context; import eu.dnetlib.openaire.exporter.model.context.Context;
@ -54,4 +55,6 @@ public interface ISClient {
@Deprecated @Deprecated
void updateConceptParamNoEscape(String id, String name, String value); void updateConceptParamNoEscape(String id, String name, String value);
String getProfile(String profileId) throws ISLookUpException;
} }

5
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java
View File

@ -374,4 +374,9 @@ public class ISClientImpl implements ISClient {
log.debug("dropped dsManager IS cache"); log.debug("dropped dsManager IS cache");
} }
@Override
public String getProfile(final String profileId) throws ISLookUpException {
return isLookUpService.getResourceProfile(profileId);
}
} }

300
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityImporterController.java
View File

@ -1,112 +1,36 @@
package eu.dnetlib.openaire.community; package eu.dnetlib.openaire.community;
import java.io.IOException; import java.io.IOException;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.Date;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets; import com.google.common.collect.Sets;
import eu.dnetlib.openaire.common.ISClient; import eu.dnetlib.openaire.common.ISClient;
import eu.dnetlib.openaire.community.db.CommunityService; import eu.dnetlib.openaire.community.db.CommunityImporterService;
import eu.dnetlib.openaire.community.db.model.DbOrganization;
import eu.dnetlib.openaire.exporter.exceptions.CommunityException; import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider;
import eu.dnetlib.openaire.exporter.model.community.CommunityDetails;
import eu.dnetlib.openaire.exporter.model.community.CommunityMembershipType;
import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization;
import eu.dnetlib.openaire.exporter.model.community.CommunityProject;
import eu.dnetlib.openaire.exporter.model.community.CommunityStatus;
import eu.dnetlib.openaire.exporter.model.community.CommunityType;
import eu.dnetlib.openaire.exporter.model.community.SubCommunity;
import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionCriteria;
import eu.dnetlib.openaire.exporter.model.context.Category;
import eu.dnetlib.openaire.exporter.model.context.Concept;
import eu.dnetlib.openaire.exporter.model.context.Context; import eu.dnetlib.openaire.exporter.model.context.Context;
import eu.dnetlib.openaire.exporter.model.context.Param;
@RestController @RestController
public class CommunityImporterController { public class CommunityImporterController {
private final static String pattern = "yyyy-MM-dd'T'hh:mm:ss";
// common
private final static String OPENAIRE_ID = "openaireId";
private final static String PIPE_SEPARATOR = "||";
private final static String ID_SEPARATOR = "::";
private final static String CSV_DELIMITER = ",";
private final static String CLABEL = "label";
// id suffixes
private final static String PROJECTS_ID_SUFFIX = ID_SEPARATOR + "projects";
private final static String CONTENTPROVIDERS_ID_SUFFIX = ID_SEPARATOR + "contentproviders";
private final static String ZENODOCOMMUNITY_ID_SUFFIX = ID_SEPARATOR + "zenodocommunities";
private final static String ORGANIZATION_ID_SUFFIX = ID_SEPARATOR + "organizations";
// community summary
private final static String CSUMMARY_DESCRIPTION = "description";
private final static String CSUMMARY_LOGOURL = "logourl";
private final static String CSUMMARY_STATUS = "status";
private final static String CSUMMARY_NAME = "name";
private final static String CSUMMARY_MANAGER = "manager";
private final static String CSUMMARY_ZENODOC = "zenodoCommunity";
// community profile
private final static String CPROFILE_SUBJECT = "subject";
private final static String CPROFILE_CREATIONDATE = "creationdate";
private final static String CPROFILE_FOS = "fos";
private final static String CPROFILE_SDG = "sdg";
private final static String CPROFILE_ADVANCED_CONSTRAINT = "advancedConstraints";
private final static String CPROFILE_REMOVE_CONSTRAINT = "removeConstraints";
// community project
private final static String CPROJECT_FUNDER = "funder";
private final static String CPROJECT_NUMBER = "CD_PROJECT_NUMBER";
private final static String CPROJECT_FULLNAME = "projectfullname";
private final static String CPROJECT_ACRONYM = "acronym";
// community content provider
private final static String CCONTENTPROVIDER_NAME = "name";
private final static String CCONTENTPROVIDER_OFFICIALNAME = "officialname";
private final static String CCONTENTPROVIDER_ENABLED = "enabled";
private final static String CCONTENTPROVIDERENABLED_DEFAULT = "true";
private final static String CCONTENTPROVIDER_SELCRITERIA = "selcriteria";
// community zenodo community
private final static String CZENODOCOMMUNITY_ID = "zenodoid";
// community organization
private final static String CORGANIZATION_NAME = "name";
private final static String CORGANIZATION_LOGOURL = "logourl";
private final static String CORGANIZATION_WEBSITEURL = "websiteurl";
private static final Log log = LogFactory.getLog(CommunityImporterController.class);
public final static Set<String> communityBlackList = Sets.newHashSet("fet-fp7", "fet-h2020"); public final static Set<String> communityBlackList = Sets.newHashSet("fet-fp7", "fet-h2020");
@Autowired @Autowired
private CommunityService service; private CommunityImporterService importer;
@Autowired @Autowired
private ISClient isClient; private ISClient isClient;
@GetMapping("/community_importer/import") @GetMapping("/community_importer/communities")
public List<String> importProfiles() throws CommunityException { public List<String> importProfiles() throws CommunityException {
final Map<String, Context> contextMap = getContextMap(); final Map<String, Context> contextMap = getContextMap();
@ -116,12 +40,19 @@ public class CommunityImporterController {
.collect(Collectors.toList()); .collect(Collectors.toList());
list.forEach(id -> { list.forEach(id -> {
importCommunity(contextMap.get(id)); importer.importCommunity(contextMap.get(id));
}); });
return list; return list;
} }
@GetMapping("/community_importer/propagationOrgs")
public List<DbOrganization> importPropagationOrgs(@RequestParam final String profileId,
@RequestParam(required = false, defaultValue = "false") final boolean simulation) throws Exception {
final String xml = isClient.getProfile(profileId);
return importer.importPropagationOrganizationsFromProfile(xml, simulation);
}
private Map<String, Context> getContextMap() throws CommunityException { private Map<String, Context> getContextMap() throws CommunityException {
try { try {
return isClient.getCommunityContextMap(); return isClient.getCommunityContextMap();
@ -130,209 +61,4 @@ public class CommunityImporterController {
} }
} }
public <R> List<R> getCommunityInfo(final Context context, final String idSuffix, final Function<Concept, R> mapping)
throws CommunityException {
if (context != null) {
final Map<String, Category> categories = context.getCategories();
final Category category = categories.get(context.getId() + idSuffix);
if (category != null) { return category.getConcepts()
.stream()
.map(mapping)
.collect(Collectors.toList()); }
}
return Lists.newArrayList();
}
private void importCommunity(final Context context) {
try {
final CommunityDetails community = asCommunityDetails(context);
final List<CommunityContentprovider> datasources =
getCommunityInfo(context, CONTENTPROVIDERS_ID_SUFFIX, c -> asCommunityDataprovider(context.getId(), c));
final List<CommunityProject> projects =
getCommunityInfo(context, PROJECTS_ID_SUFFIX, c -> asCommunityProject(context.getId(), c));
final List<CommunityOrganization> orgs =
getCommunityInfo(context, ORGANIZATION_ID_SUFFIX, c -> asCommunityOrganization(context.getId(), c));
final List<String> otherZenodoCimmunities =
getCommunityInfo(context, ZENODOCOMMUNITY_ID_SUFFIX, c -> asZenodoCommunity(c));
community.setOtherZenodoCommunities(otherZenodoCimmunities);
final List<SubCommunity> subs = context.getCategories()
.entrySet()
.stream()
.filter(e -> !e.getKey().equals(context.getId() + CONTENTPROVIDERS_ID_SUFFIX))
.filter(e -> !e.getKey().equals(context.getId() + PROJECTS_ID_SUFFIX))
.filter(e -> !e.getKey().equals(context.getId() + ORGANIZATION_ID_SUFFIX))
.filter(e -> !e.getKey().equals(context.getId() + ZENODOCOMMUNITY_ID_SUFFIX))
.map(e -> e.getValue())
.map(cat -> asSubCommunities(context.getId(), null, cat.getLabel(), cat.getConcepts()))
.flatMap(List::stream)
.collect(Collectors.toList());
service.saveCommunity(community);
service.addCommunityProjectList(context.getId(), projects);
service.addCommunityContentProvidersList(context.getId(), datasources);
service.addCommunityOrganizationList(context.getId(), orgs);
service.addSubCommunityList(subs);
} catch (final Exception e) {
throw new RuntimeException("Error importing community: " + context.getId(), e);
}
}
private static CommunityDetails asCommunityDetails(final Context c) {
final CommunityDetails details = new CommunityDetails();
details.setId(c.getId());
details.setShortName(c.getLabel());
details.setLastUpdateDate(convertToLocalDateTime(c.getLastUpdateDate()));
details.setCreationDate(convertToLocalDateTime(c.getCreationDate()));
details.setQueryId(c.getId() + PIPE_SEPARATOR + c.getLabel());
details.setType(CommunityType.valueOf(c.getType()));
details.setMembership(CommunityMembershipType.open);
// TODO: Considerare anche i campi claim
final Map<String, List<Param>> params = c.getParams();
if (params.containsKey(CSUMMARY_DESCRIPTION)) {
details.setDescription(asCsv(params.get(CSUMMARY_DESCRIPTION)));
}
if (params.containsKey(CSUMMARY_LOGOURL)) {
details.setLogoUrl(asCsv(params.get(CSUMMARY_LOGOURL)));
}
if (params.containsKey(CSUMMARY_STATUS)) {
details.setStatus(CommunityStatus.valueOf(firstValue(params, CSUMMARY_STATUS)));
}
if (params.containsKey(CSUMMARY_NAME)) {
details.setName(asCsv(params.get(CSUMMARY_NAME)));
} else {
details.setName(c.getLabel());
}
if (params.containsKey(CSUMMARY_ZENODOC)) {
details.setZenodoCommunity(asCsv(params.get(CSUMMARY_ZENODOC)));
}
if (params.containsKey(CPROFILE_SUBJECT)) {
details.setSubjects(splitValues(asValues(params.get(CPROFILE_SUBJECT)), CSV_DELIMITER));
}
if (params.containsKey(CPROFILE_FOS)) {
details.setFos(splitValues(asValues(params.get(CPROFILE_FOS)), CSV_DELIMITER));
}
if (params.containsKey(CPROFILE_SDG)) {
details.setSdg(splitValues(asValues(params.get(CPROFILE_SDG)), CSV_DELIMITER));
}
if (params.containsKey(CPROFILE_ADVANCED_CONSTRAINT)) {
// In the map the string is the serialization of the json representing the selection criteria so it is a valid json
details.setAdvancedConstraints(SelectionCriteria.fromJson(asCsv(params.get(CPROFILE_ADVANCED_CONSTRAINT))));
}
if (params.containsKey(CPROFILE_REMOVE_CONSTRAINT)) {
// In the map the string is the serialization of the json representing the selection criteria so it is a valid json
details.setRemoveConstraints(SelectionCriteria.fromJson(asCsv(params.get(CPROFILE_REMOVE_CONSTRAINT))));
}
if (params.containsKey(CPROFILE_CREATIONDATE)) {
try {
final Date d = org.apache.commons.lang3.time.DateUtils.parseDate(asCsv(params.get(CPROFILE_CREATIONDATE)), pattern);
details.setCreationDate(convertToLocalDateTime(d));
} catch (final Exception e) {
log.debug("Exception on date format: " + e.getMessage());
}
}
return details;
}
private static CommunityProject asCommunityProject(final String communityId, final Concept c) {
final Map<String, List<Param>> p = c.getParams();
final CommunityProject project = new CommunityProject();
project.setCommunityId(communityId);
project.setOpenaireId(firstValue(p, OPENAIRE_ID));
project.setFunder(firstValue(p, CPROJECT_FUNDER));
project.setGrantId(firstValue(p, CPROJECT_NUMBER));
project.setName(firstValue(p, CPROJECT_FULLNAME));
project.setAcronym(firstValue(p, CPROJECT_ACRONYM));
return project;
}
private static CommunityContentprovider asCommunityDataprovider(final String communityId, final Concept c) {
final Map<String, List<Param>> p = c.getParams();
final CommunityContentprovider d = new CommunityContentprovider();
d.setCommunityId(communityId);
d.setOpenaireId(firstValue(p, OPENAIRE_ID));
d.setName(firstValue(p, CCONTENTPROVIDER_NAME));
d.setOfficialname(firstValue(p, CCONTENTPROVIDER_OFFICIALNAME));
d.setSelectioncriteria(SelectionCriteria.fromJson(firstValue(p, CCONTENTPROVIDER_SELCRITERIA)));
return d;
}
private static CommunityOrganization asCommunityOrganization(final String id, final Concept c) {
final Map<String, List<Param>> p = c.getParams();
final CommunityOrganization o = new CommunityOrganization();
o.setCommunityId(id);
// o.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
o.setName(firstValue(p, CORGANIZATION_NAME));
o.setLogo_url(getDecodedUrl(firstValue(p, CORGANIZATION_LOGOURL)));
o.setWebsite_url(getDecodedUrl(firstValue(p, CORGANIZATION_WEBSITEURL)));
return o;
}
private static String asZenodoCommunity(final Concept c) {
return firstValue(c.getParams(), CZENODOCOMMUNITY_ID);
}
private static List<SubCommunity> asSubCommunities(final String communityId, final String parent, final String category, final List<Concept> concepts) {
final List<SubCommunity> list = new ArrayList<>();
for (final Concept c : concepts) {
final SubCommunity sc = new SubCommunity();
sc.setSubCommunityId(c.getId());
sc.setCommunityId(communityId);
sc.setParent(parent);
sc.setCategory(category);
sc.setLabel(c.getLabel());
sc.setParams(c.getParams());
list.add(sc);
list.addAll(asSubCommunities(communityId, c.getId(), category, c.getConcepts()));
}
return list;
}
private static String getDecodedUrl(final String encoded_url) {
if (encoded_url == null) { return encoded_url; }
return new String(Base64.getDecoder().decode(encoded_url));
}
private static List<String> splitValues(final Stream<String> stream, final String separator) {
return stream.map(s -> s.split(separator))
.map(Arrays::asList)
.flatMap(List::stream)
.filter(StringUtils::isNotBlank)
.map(StringUtils::trim)
.collect(Collectors.toList());
}
private static String firstValue(final Map<String, List<Param>> p, final String paramName) {
return asValues(p.get(paramName)).findFirst().orElse(null);
}
private static String asCsv(final List<Param> params) {
return asValues(params)
.collect(Collectors.joining(CSV_DELIMITER));
}
private static Stream<String> asValues(final List<Param> params) {
return params == null ? Stream.empty()
: params.stream()
.map(Param::getValue)
.map(StringUtils::trim)
.distinct();
}
private static LocalDateTime convertToLocalDateTime(final Date date) {
return date.toInstant()
.atZone(ZoneId.systemDefault())
.toLocalDateTime();
}
} }

335
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/db/CommunityImporterService.java Normal file
View File

@ -0,0 +1,335 @@
package eu.dnetlib.openaire.community.db;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.transaction.Transactional;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.dom4j.DocumentHelper;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
import eu.dnetlib.openaire.community.db.model.DbOrganization;
import eu.dnetlib.openaire.community.db.repository.DbOrganizationRepository;
import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
import eu.dnetlib.openaire.exporter.model.community.CommunityClaimType;
import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider;
import eu.dnetlib.openaire.exporter.model.community.CommunityDetails;
import eu.dnetlib.openaire.exporter.model.community.CommunityMembershipType;
import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization;
import eu.dnetlib.openaire.exporter.model.community.CommunityProject;
import eu.dnetlib.openaire.exporter.model.community.CommunityStatus;
import eu.dnetlib.openaire.exporter.model.community.CommunityType;
import eu.dnetlib.openaire.exporter.model.community.SubCommunity;
import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionCriteria;
import eu.dnetlib.openaire.exporter.model.context.Category;
import eu.dnetlib.openaire.exporter.model.context.Concept;
import eu.dnetlib.openaire.exporter.model.context.Context;
import eu.dnetlib.openaire.exporter.model.context.Param;
@Service
public class CommunityImporterService {
private final static String pattern = "yyyy-MM-dd'T'hh:mm:ss";
// common
private final static String OPENAIRE_ID = "openaireId";
private final static String PIPE_SEPARATOR = "||";
private final static String ID_SEPARATOR = "::";
private final static String CSV_DELIMITER = ",";
private final static String CLABEL = "label";
// id suffixes
private final static String PROJECTS_ID_SUFFIX = ID_SEPARATOR + "projects";
private final static String CONTENTPROVIDERS_ID_SUFFIX = ID_SEPARATOR + "contentproviders";
private final static String ZENODOCOMMUNITY_ID_SUFFIX = ID_SEPARATOR + "zenodocommunities";
private final static String ORGANIZATION_ID_SUFFIX = ID_SEPARATOR + "organizations";
// community summary
private final static String CSUMMARY_DESCRIPTION = "description";
private final static String CSUMMARY_LOGOURL = "logourl";
private final static String CSUMMARY_STATUS = "status";
private final static String CSUMMARY_NAME = "name";
private final static String CSUMMARY_MANAGER = "manager";
private final static String CSUMMARY_ZENODOC = "zenodoCommunity";
// community profile
private final static String CPROFILE_SUBJECT = "subject";
private final static String CPROFILE_CREATIONDATE = "creationdate";
private final static String CPROFILE_FOS = "fos";
private final static String CPROFILE_SDG = "sdg";
private final static String CPROFILE_ADVANCED_CONSTRAINT = "advancedConstraints";
private final static String CPROFILE_REMOVE_CONSTRAINT = "removeConstraints";
// community project
private final static String CPROJECT_FUNDER = "funder";
private final static String CPROJECT_NUMBER = "CD_PROJECT_NUMBER";
private final static String CPROJECT_FULLNAME = "projectfullname";
private final static String CPROJECT_ACRONYM = "acronym";
// community content provider
private final static String CCONTENTPROVIDER_NAME = "name";
private final static String CCONTENTPROVIDER_OFFICIALNAME = "officialname";
private final static String CCONTENTPROVIDER_ENABLED = "enabled";
private final static String CCONTENTPROVIDERENABLED_DEFAULT = "true";
private final static String CCONTENTPROVIDER_SELCRITERIA = "selcriteria";
// community zenodo community
private final static String CZENODOCOMMUNITY_ID = "zenodoid";
// community organization
private final static String CORGANIZATION_NAME = "name";
private final static String CORGANIZATION_LOGOURL = "logourl";
private final static String CORGANIZATION_WEBSITEURL = "websiteurl";
@Autowired
private DbOrganizationRepository dbOrganizationRepository;
@Autowired
private CommunityService service;
private static final Log log = LogFactory.getLog(CommunityImporterService.class);
@Transactional
public List<DbOrganization> importPropagationOrganizationsFromProfile(final String xml, final boolean simulation) throws Exception {
final String json = DocumentHelper.parseText(xml)
.selectSingleNode("//NODE[@name='setPropagationOrganizationCommunityMap']//PARAM[@name='parameterValue']")
.getText();
final List<DbOrganization> list = new ObjectMapper()
.readValue(json, new TypeReference<Map<String, List<String>>>() {})
.entrySet()
.stream()
.flatMap(e -> e.getValue().stream().map(community -> new DbOrganization(community, e.getKey())))
.collect(Collectors.toList());
if (!simulation) {
dbOrganizationRepository.saveAll(list);
}
return list;
}
@Transactional
public void importCommunity(final Context context) {
try {
final CommunityDetails community = asCommunityDetails(context);
final List<CommunityContentprovider> datasources =
getCommunityInfo(context, CONTENTPROVIDERS_ID_SUFFIX, c -> asCommunityDataprovider(context.getId(), c));
final List<CommunityProject> projects =
getCommunityInfo(context, PROJECTS_ID_SUFFIX, c -> asCommunityProject(context.getId(), c));
final List<CommunityOrganization> orgs =
getCommunityInfo(context, ORGANIZATION_ID_SUFFIX, c -> asCommunityOrganization(context.getId(), c));
final List<String> otherZenodoCimmunities =
getCommunityInfo(context, ZENODOCOMMUNITY_ID_SUFFIX, c -> asZenodoCommunity(c));
community.setOtherZenodoCommunities(otherZenodoCimmunities);
final List<SubCommunity> subs = context.getCategories()
.entrySet()
.stream()
.filter(e -> !e.getKey().equals(context.getId() + CONTENTPROVIDERS_ID_SUFFIX))
.filter(e -> !e.getKey().equals(context.getId() + PROJECTS_ID_SUFFIX))
.filter(e -> !e.getKey().equals(context.getId() + ORGANIZATION_ID_SUFFIX))
.filter(e -> !e.getKey().equals(context.getId() + ZENODOCOMMUNITY_ID_SUFFIX))
.map(e -> e.getValue())
.map(cat -> asSubCommunities(context.getId(), null, cat.getLabel(), cat.getConcepts()))
.flatMap(List::stream)
.collect(Collectors.toList());
service.saveCommunity(community);
service.addCommunityProjectList(context.getId(), projects);
service.addCommunityContentProvidersList(context.getId(), datasources);
service.addCommunityOrganizationList(context.getId(), orgs);
service.addSubCommunityList(subs);
} catch (final Exception e) {
throw new RuntimeException("Error importing community: " + context.getId(), e);
}
}
private <R> List<R> getCommunityInfo(final Context context, final String idSuffix, final Function<Concept, R> mapping)
throws CommunityException {
if (context != null) {
final Map<String, Category> categories = context.getCategories();
final Category category = categories.get(context.getId() + idSuffix);
if (category != null) { return category.getConcepts()
.stream()
.map(mapping)
.collect(Collectors.toList()); }
}
return Lists.newArrayList();
}
private static CommunityDetails asCommunityDetails(final Context c) {
final CommunityDetails details = new CommunityDetails();
details.setId(c.getId());
details.setShortName(c.getLabel());
details.setLastUpdateDate(convertToLocalDateTime(c.getLastUpdateDate()));
details.setCreationDate(convertToLocalDateTime(c.getCreationDate()));
details.setQueryId(c.getId() + PIPE_SEPARATOR + c.getLabel());
details.setType(CommunityType.valueOf(c.getType()));
details.setMembership(CommunityMembershipType.open);
details.setClaim(CommunityClaimType.all);
final Map<String, List<Param>> params = c.getParams();
if (params.containsKey(CSUMMARY_DESCRIPTION)) {
details.setDescription(asCsv(params.get(CSUMMARY_DESCRIPTION)));
}
if (params.containsKey(CSUMMARY_LOGOURL)) {
details.setLogoUrl(asCsv(params.get(CSUMMARY_LOGOURL)));
}
if (params.containsKey(CSUMMARY_STATUS)) {
details.setStatus(CommunityStatus.valueOf(firstValue(params, CSUMMARY_STATUS)));
}
if (params.containsKey(CSUMMARY_NAME)) {
details.setName(asCsv(params.get(CSUMMARY_NAME)));
} else {
details.setName(c.getLabel());
}
if (params.containsKey(CSUMMARY_ZENODOC)) {
details.setZenodoCommunity(asCsv(params.get(CSUMMARY_ZENODOC)));
}
if (params.containsKey(CPROFILE_SUBJECT)) {
details.setSubjects(splitValues(asValues(params.get(CPROFILE_SUBJECT)), CSV_DELIMITER));
}
if (params.containsKey(CPROFILE_FOS)) {
details.setFos(splitValues(asValues(params.get(CPROFILE_FOS)), CSV_DELIMITER));
}
if (params.containsKey(CPROFILE_SDG)) {
details.setSdg(splitValues(asValues(params.get(CPROFILE_SDG)), CSV_DELIMITER));
}
if (params.containsKey(CPROFILE_ADVANCED_CONSTRAINT)) {
// In the map the string is the serialization of the json representing the selection criteria so it is a valid json
details.setAdvancedConstraints(SelectionCriteria.fromJson(asCsv(params.get(CPROFILE_ADVANCED_CONSTRAINT))));
}
if (params.containsKey(CPROFILE_REMOVE_CONSTRAINT)) {
// In the map the string is the serialization of the json representing the selection criteria so it is a valid json
details.setRemoveConstraints(SelectionCriteria.fromJson(asCsv(params.get(CPROFILE_REMOVE_CONSTRAINT))));
}
if (params.containsKey(CPROFILE_CREATIONDATE)) {
try {
final Date d = org.apache.commons.lang3.time.DateUtils.parseDate(asCsv(params.get(CPROFILE_CREATIONDATE)), pattern);
details.setCreationDate(convertToLocalDateTime(d));
} catch (final Exception e) {
log.debug("Exception on date format: " + e.getMessage());
}
}
return details;
}
private static CommunityProject asCommunityProject(final String communityId, final Concept c) {
final Map<String, List<Param>> p = c.getParams();
final CommunityProject project = new CommunityProject();
project.setCommunityId(communityId);
project.setOpenaireId(firstValue(p, OPENAIRE_ID));
project.setFunder(firstValue(p, CPROJECT_FUNDER));
project.setGrantId(firstValue(p, CPROJECT_NUMBER));
project.setName(firstValue(p, CPROJECT_FULLNAME));
project.setAcronym(firstValue(p, CPROJECT_ACRONYM));
return project;
}
private static CommunityContentprovider asCommunityDataprovider(final String communityId, final Concept c) {
final Map<String, List<Param>> p = c.getParams();
final CommunityContentprovider d = new CommunityContentprovider();
d.setCommunityId(communityId);
d.setOpenaireId(firstValue(p, OPENAIRE_ID));
d.setName(firstValue(p, CCONTENTPROVIDER_NAME));
d.setOfficialname(firstValue(p, CCONTENTPROVIDER_OFFICIALNAME));
d.setSelectioncriteria(SelectionCriteria.fromJson(firstValue(p, CCONTENTPROVIDER_SELCRITERIA)));
return d;
}
private static CommunityOrganization asCommunityOrganization(final String id, final Concept c) {
final Map<String, List<Param>> p = c.getParams();
final CommunityOrganization o = new CommunityOrganization();
o.setCommunityId(id);
// o.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
o.setName(firstValue(p, CORGANIZATION_NAME));
o.setLogo_url(getDecodedUrl(firstValue(p, CORGANIZATION_LOGOURL)));
o.setWebsite_url(getDecodedUrl(firstValue(p, CORGANIZATION_WEBSITEURL)));
return o;
}
private static String asZenodoCommunity(final Concept c) {
return firstValue(c.getParams(), CZENODOCOMMUNITY_ID);
}
private static List<SubCommunity> asSubCommunities(final String communityId, final String parent, final String category, final List<Concept> concepts) {
final List<SubCommunity> list = new ArrayList<>();
for (final Concept c : concepts) {
final SubCommunity sc = new SubCommunity();
sc.setSubCommunityId(c.getId());
sc.setCommunityId(communityId);
sc.setParent(parent);
sc.setCategory(category);
sc.setLabel(c.getLabel());
sc.setParams(c.getParams());
sc.setClaim(c.isClaim());
list.add(sc);
list.addAll(asSubCommunities(communityId, c.getId(), category, c.getConcepts()));
}
return list;
}
private static String getDecodedUrl(final String encoded_url) {
if (encoded_url == null) { return encoded_url; }
return new String(Base64.getDecoder().decode(encoded_url));
}
private static List<String> splitValues(final Stream<String> stream, final String separator) {
return stream.map(s -> s.split(separator))
.map(Arrays::asList)
.flatMap(List::stream)
.filter(StringUtils::isNotBlank)
.map(StringUtils::trim)
.collect(Collectors.toList());
}
private static String firstValue(final Map<String, List<Param>> p, final String paramName) {
return asValues(p.get(paramName)).findFirst().orElse(null);
}
private static String asCsv(final List<Param> params) {
return asValues(params)
.collect(Collectors.joining(CSV_DELIMITER));
}
private static Stream<String> asValues(final List<Param> params) {
return params == null ? Stream.empty()
: params.stream()
.map(Param::getValue)
.map(StringUtils::trim)
.distinct();
}
private static LocalDateTime convertToLocalDateTime(final Date date) {
return date.toInstant()
.atZone(ZoneId.systemDefault())
.toLocalDateTime();
}
}

9
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/db/CommunityService.java
View File

@ -48,12 +48,9 @@ import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionC
public class CommunityService { public class CommunityService {
// TODO // TODO
// 1) Import tramite profili: // 1) Subcommunities: visualizzazione tramite le context api
// 1.1) Completare CommunityImporterController // 2) Test di import (context e wf)
// 2) Subcommunities: visualizzazione tramite le context api // 3) Verificare Tickets: #8835, #8854, #6483, #3259, #3494
// 3) Importare le orgs dal wf di provision, ad esempio:
// https://services.openaire.eu/is/mvc/ui/isManager.do#/profile/4801c33c-66ca-4ab6-af64-aa812194ec61_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl
// 4) Verificare Tickets: #8835, #8854, #6483, #3259, #3494
@Autowired @Autowired
private DbCommunityRepository dbCommunityRepository; private DbCommunityRepository dbCommunityRepository;

14
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/db/model/DbSubCommunity.java
View File

@ -1,6 +1,7 @@
package eu.dnetlib.openaire.community.db.model; package eu.dnetlib.openaire.community.db.model;
import java.io.Serializable; import java.io.Serializable;
import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -45,11 +46,14 @@ public class DbSubCommunity implements Serializable {
@Type(type = "jsonb") @Type(type = "jsonb")
@Column(name = "params") @Column(name = "params")
private Map<String, List<Param>> params; private Map<String, List<Param>> params = new LinkedHashMap<>();
@Column(name = "parent") @Column(name = "parent")
private String parent; private String parent;
@Column(name = "claim")
private boolean claim = false;
public String getId() { public String getId() {
return id; return id;
} }
@ -97,4 +101,12 @@ public class DbSubCommunity implements Serializable {
public void setParent(final String parent) { public void setParent(final String parent) {
this.parent = parent; this.parent = parent;
} }
public boolean isClaim() {
return claim;
}
public void setClaim(final boolean claim) {
this.claim = claim;
}
} }

2
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/db/utils/CommunityMappingUtils.java
View File

@ -199,6 +199,7 @@ public class CommunityMappingUtils {
dbsc.setLabel(sub.getLabel()); dbsc.setLabel(sub.getLabel());
dbsc.setParams(sub.getParams()); dbsc.setParams(sub.getParams());
dbsc.setParent(sub.getParent()); dbsc.setParent(sub.getParent());
dbsc.setClaim(sub.isClaim());
return dbsc; return dbsc;
} }
@ -210,6 +211,7 @@ public class CommunityMappingUtils {
sc.setLabel(sub.getLabel()); sc.setLabel(sub.getLabel());
sc.setParams(sub.getParams()); sc.setParams(sub.getParams());
sc.setParent(sub.getParent()); sc.setParent(sub.getParent());
sc.setClaim(sub.isClaim());
return sc; return sc;
} }

10
apps/dnet-exporter-api/src/main/resources/sql/community-schema.sql
View File

@ -1,3 +1,4 @@
DROP TABLE IF EXISTS community_subs;
DROP TABLE IF EXISTS community_projects; DROP TABLE IF EXISTS community_projects;
DROP TABLE IF EXISTS community_datasources; DROP TABLE IF EXISTS community_datasources;
DROP TABLE IF EXISTS community_support_orgs; DROP TABLE IF EXISTS community_support_orgs;
@ -60,10 +61,11 @@ CREATE TABLE community_orgs (
CREATE TABLE community_subs ( CREATE TABLE community_subs (
sub_id text NOT NULL PRIMARY KEY, sub_id text NOT NULL PRIMARY KEY,
community text NOT NULL REFERENCES communities(id), community text NOT NULL REFERENCES communities(id),
label text NOT NULL, label text NOT NULL,
category text NOT NULL, category text NOT NULL,
claim boolean NOT NULL DEFAULT false,
params jsonb, params jsonb,
parent text REFERENCES community_subs(sub_id) -- NULL for the first level parent text REFERENCES community_subs(sub_id) -- NULL for the first level
); );

11
libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/community/SubCommunity.java
View File

@ -30,6 +30,9 @@ public class SubCommunity {
@Schema(description = "the parameters of the subCommunity", required = true) @Schema(description = "the parameters of the subCommunity", required = true)
private Map<String, List<Param>> params = new LinkedHashMap<>(); private Map<String, List<Param>> params = new LinkedHashMap<>();
@Schema(description = "it supports the claims", required = true)
private boolean claim = false;
public String getSubCommunityId() { public String getSubCommunityId() {
return subCommunityId; return subCommunityId;
} }
@ -78,4 +81,12 @@ public class SubCommunity {
this.params = map; this.params = map;
} }
public boolean isClaim() {
return claim;
}
public void setClaim(final boolean claim) {
this.claim = claim;
}
} }