diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClient.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClient.java index c7cf3345..9a4ffdd6 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClient.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClient.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException; import eu.dnetlib.openaire.dsm.dao.utils.IndexDsInfo; import eu.dnetlib.openaire.exporter.model.context.Context; @@ -54,4 +55,6 @@ public interface ISClient { @Deprecated void updateConceptParamNoEscape(String id, String name, String value); + String getProfile(String profileId) throws ISLookUpException; + } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java index 0ebcf66b..42bb80a8 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/ISClientImpl.java @@ -374,4 +374,9 @@ public class ISClientImpl implements ISClient { log.debug("dropped dsManager IS cache"); } + @Override + public String getProfile(final String profileId) throws ISLookUpException { + return isLookUpService.getResourceProfile(profileId); + } + } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityImporterController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityImporterController.java index af903e65..afb981db 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityImporterController.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityImporterController.java @@ -1,112 +1,36 @@ package eu.dnetlib.openaire.community; import java.io.IOException; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Base64; -import java.util.Date; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.function.Function; import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import com.google.common.collect.Lists; import com.google.common.collect.Sets; import eu.dnetlib.openaire.common.ISClient; -import eu.dnetlib.openaire.community.db.CommunityService; +import eu.dnetlib.openaire.community.db.CommunityImporterService; +import eu.dnetlib.openaire.community.db.model.DbOrganization; import eu.dnetlib.openaire.exporter.exceptions.CommunityException; -import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider; -import eu.dnetlib.openaire.exporter.model.community.CommunityDetails; -import eu.dnetlib.openaire.exporter.model.community.CommunityMembershipType; -import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization; -import eu.dnetlib.openaire.exporter.model.community.CommunityProject; -import eu.dnetlib.openaire.exporter.model.community.CommunityStatus; -import eu.dnetlib.openaire.exporter.model.community.CommunityType; -import eu.dnetlib.openaire.exporter.model.community.SubCommunity; -import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionCriteria; -import eu.dnetlib.openaire.exporter.model.context.Category; -import eu.dnetlib.openaire.exporter.model.context.Concept; import eu.dnetlib.openaire.exporter.model.context.Context; -import eu.dnetlib.openaire.exporter.model.context.Param; @RestController public class CommunityImporterController { - private final static String pattern = "yyyy-MM-dd'T'hh:mm:ss"; - - // common - private final static String OPENAIRE_ID = "openaireId"; - private final static String PIPE_SEPARATOR = "||"; - private final static String ID_SEPARATOR = "::"; - private final static String CSV_DELIMITER = ","; - private final static String CLABEL = "label"; - - // id suffixes - private final static String PROJECTS_ID_SUFFIX = ID_SEPARATOR + "projects"; - private final static String CONTENTPROVIDERS_ID_SUFFIX = ID_SEPARATOR + "contentproviders"; - private final static String ZENODOCOMMUNITY_ID_SUFFIX = ID_SEPARATOR + "zenodocommunities"; - private final static String ORGANIZATION_ID_SUFFIX = ID_SEPARATOR + "organizations"; - - // community summary - private final static String CSUMMARY_DESCRIPTION = "description"; - private final static String CSUMMARY_LOGOURL = "logourl"; - private final static String CSUMMARY_STATUS = "status"; - private final static String CSUMMARY_NAME = "name"; - private final static String CSUMMARY_MANAGER = "manager"; - private final static String CSUMMARY_ZENODOC = "zenodoCommunity"; - - // community profile - private final static String CPROFILE_SUBJECT = "subject"; - private final static String CPROFILE_CREATIONDATE = "creationdate"; - private final static String CPROFILE_FOS = "fos"; - private final static String CPROFILE_SDG = "sdg"; - private final static String CPROFILE_ADVANCED_CONSTRAINT = "advancedConstraints"; - private final static String CPROFILE_REMOVE_CONSTRAINT = "removeConstraints"; - - // community project - private final static String CPROJECT_FUNDER = "funder"; - private final static String CPROJECT_NUMBER = "CD_PROJECT_NUMBER"; - private final static String CPROJECT_FULLNAME = "projectfullname"; - private final static String CPROJECT_ACRONYM = "acronym"; - - // community content provider - private final static String CCONTENTPROVIDER_NAME = "name"; - private final static String CCONTENTPROVIDER_OFFICIALNAME = "officialname"; - private final static String CCONTENTPROVIDER_ENABLED = "enabled"; - private final static String CCONTENTPROVIDERENABLED_DEFAULT = "true"; - private final static String CCONTENTPROVIDER_SELCRITERIA = "selcriteria"; - - // community zenodo community - private final static String CZENODOCOMMUNITY_ID = "zenodoid"; - - // community organization - private final static String CORGANIZATION_NAME = "name"; - private final static String CORGANIZATION_LOGOURL = "logourl"; - private final static String CORGANIZATION_WEBSITEURL = "websiteurl"; - - private static final Log log = LogFactory.getLog(CommunityImporterController.class); - public final static Set communityBlackList = Sets.newHashSet("fet-fp7", "fet-h2020"); @Autowired - private CommunityService service; + private CommunityImporterService importer; @Autowired private ISClient isClient; - @GetMapping("/community_importer/import") + @GetMapping("/community_importer/communities") public List importProfiles() throws CommunityException { final Map contextMap = getContextMap(); @@ -116,12 +40,19 @@ public class CommunityImporterController { .collect(Collectors.toList()); list.forEach(id -> { - importCommunity(contextMap.get(id)); + importer.importCommunity(contextMap.get(id)); }); return list; } + @GetMapping("/community_importer/propagationOrgs") + public List importPropagationOrgs(@RequestParam final String profileId, + @RequestParam(required = false, defaultValue = "false") final boolean simulation) throws Exception { + final String xml = isClient.getProfile(profileId); + return importer.importPropagationOrganizationsFromProfile(xml, simulation); + } + private Map getContextMap() throws CommunityException { try { return isClient.getCommunityContextMap(); @@ -130,209 +61,4 @@ public class CommunityImporterController { } } - public List getCommunityInfo(final Context context, final String idSuffix, final Function mapping) - throws CommunityException { - if (context != null) { - final Map categories = context.getCategories(); - final Category category = categories.get(context.getId() + idSuffix); - if (category != null) { return category.getConcepts() - .stream() - .map(mapping) - .collect(Collectors.toList()); } - } - return Lists.newArrayList(); - } - - private void importCommunity(final Context context) { - try { - - final CommunityDetails community = asCommunityDetails(context); - - final List datasources = - getCommunityInfo(context, CONTENTPROVIDERS_ID_SUFFIX, c -> asCommunityDataprovider(context.getId(), c)); - - final List projects = - getCommunityInfo(context, PROJECTS_ID_SUFFIX, c -> asCommunityProject(context.getId(), c)); - - final List orgs = - getCommunityInfo(context, ORGANIZATION_ID_SUFFIX, c -> asCommunityOrganization(context.getId(), c)); - - final List otherZenodoCimmunities = - getCommunityInfo(context, ZENODOCOMMUNITY_ID_SUFFIX, c -> asZenodoCommunity(c)); - - community.setOtherZenodoCommunities(otherZenodoCimmunities); - - final List subs = context.getCategories() - .entrySet() - .stream() - .filter(e -> !e.getKey().equals(context.getId() + CONTENTPROVIDERS_ID_SUFFIX)) - .filter(e -> !e.getKey().equals(context.getId() + PROJECTS_ID_SUFFIX)) - .filter(e -> !e.getKey().equals(context.getId() + ORGANIZATION_ID_SUFFIX)) - .filter(e -> !e.getKey().equals(context.getId() + ZENODOCOMMUNITY_ID_SUFFIX)) - .map(e -> e.getValue()) - .map(cat -> asSubCommunities(context.getId(), null, cat.getLabel(), cat.getConcepts())) - .flatMap(List::stream) - .collect(Collectors.toList()); - - service.saveCommunity(community); - service.addCommunityProjectList(context.getId(), projects); - service.addCommunityContentProvidersList(context.getId(), datasources); - service.addCommunityOrganizationList(context.getId(), orgs); - service.addSubCommunityList(subs); - } catch (final Exception e) { - throw new RuntimeException("Error importing community: " + context.getId(), e); - } - } - - private static CommunityDetails asCommunityDetails(final Context c) { - - final CommunityDetails details = new CommunityDetails(); - - details.setId(c.getId()); - details.setShortName(c.getLabel()); - details.setLastUpdateDate(convertToLocalDateTime(c.getLastUpdateDate())); - details.setCreationDate(convertToLocalDateTime(c.getCreationDate())); - details.setQueryId(c.getId() + PIPE_SEPARATOR + c.getLabel()); - details.setType(CommunityType.valueOf(c.getType())); - details.setMembership(CommunityMembershipType.open); - // TODO: Considerare anche i campi claim - - final Map> params = c.getParams(); - if (params.containsKey(CSUMMARY_DESCRIPTION)) { - details.setDescription(asCsv(params.get(CSUMMARY_DESCRIPTION))); - } - if (params.containsKey(CSUMMARY_LOGOURL)) { - details.setLogoUrl(asCsv(params.get(CSUMMARY_LOGOURL))); - } - if (params.containsKey(CSUMMARY_STATUS)) { - details.setStatus(CommunityStatus.valueOf(firstValue(params, CSUMMARY_STATUS))); - } - if (params.containsKey(CSUMMARY_NAME)) { - details.setName(asCsv(params.get(CSUMMARY_NAME))); - } else { - details.setName(c.getLabel()); - } - if (params.containsKey(CSUMMARY_ZENODOC)) { - details.setZenodoCommunity(asCsv(params.get(CSUMMARY_ZENODOC))); - } - if (params.containsKey(CPROFILE_SUBJECT)) { - details.setSubjects(splitValues(asValues(params.get(CPROFILE_SUBJECT)), CSV_DELIMITER)); - } - if (params.containsKey(CPROFILE_FOS)) { - details.setFos(splitValues(asValues(params.get(CPROFILE_FOS)), CSV_DELIMITER)); - } - if (params.containsKey(CPROFILE_SDG)) { - details.setSdg(splitValues(asValues(params.get(CPROFILE_SDG)), CSV_DELIMITER)); - } - if (params.containsKey(CPROFILE_ADVANCED_CONSTRAINT)) { - // In the map the string is the serialization of the json representing the selection criteria so it is a valid json - details.setAdvancedConstraints(SelectionCriteria.fromJson(asCsv(params.get(CPROFILE_ADVANCED_CONSTRAINT)))); - } - if (params.containsKey(CPROFILE_REMOVE_CONSTRAINT)) { - // In the map the string is the serialization of the json representing the selection criteria so it is a valid json - details.setRemoveConstraints(SelectionCriteria.fromJson(asCsv(params.get(CPROFILE_REMOVE_CONSTRAINT)))); - } - if (params.containsKey(CPROFILE_CREATIONDATE)) { - try { - final Date d = org.apache.commons.lang3.time.DateUtils.parseDate(asCsv(params.get(CPROFILE_CREATIONDATE)), pattern); - details.setCreationDate(convertToLocalDateTime(d)); - } catch (final Exception e) { - log.debug("Exception on date format: " + e.getMessage()); - } - } - - return details; - } - - private static CommunityProject asCommunityProject(final String communityId, final Concept c) { - final Map> p = c.getParams(); - final CommunityProject project = new CommunityProject(); - project.setCommunityId(communityId); - project.setOpenaireId(firstValue(p, OPENAIRE_ID)); - project.setFunder(firstValue(p, CPROJECT_FUNDER)); - project.setGrantId(firstValue(p, CPROJECT_NUMBER)); - project.setName(firstValue(p, CPROJECT_FULLNAME)); - project.setAcronym(firstValue(p, CPROJECT_ACRONYM)); - return project; - } - - private static CommunityContentprovider asCommunityDataprovider(final String communityId, final Concept c) { - final Map> p = c.getParams(); - final CommunityContentprovider d = new CommunityContentprovider(); - d.setCommunityId(communityId); - d.setOpenaireId(firstValue(p, OPENAIRE_ID)); - d.setName(firstValue(p, CCONTENTPROVIDER_NAME)); - d.setOfficialname(firstValue(p, CCONTENTPROVIDER_OFFICIALNAME)); - d.setSelectioncriteria(SelectionCriteria.fromJson(firstValue(p, CCONTENTPROVIDER_SELCRITERIA))); - return d; - } - - private static CommunityOrganization asCommunityOrganization(final String id, final Concept c) { - final Map> p = c.getParams(); - final CommunityOrganization o = new CommunityOrganization(); - o.setCommunityId(id); - // o.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR)); - o.setName(firstValue(p, CORGANIZATION_NAME)); - o.setLogo_url(getDecodedUrl(firstValue(p, CORGANIZATION_LOGOURL))); - o.setWebsite_url(getDecodedUrl(firstValue(p, CORGANIZATION_WEBSITEURL))); - return o; - } - - private static String asZenodoCommunity(final Concept c) { - return firstValue(c.getParams(), CZENODOCOMMUNITY_ID); - } - - private static List asSubCommunities(final String communityId, final String parent, final String category, final List concepts) { - final List list = new ArrayList<>(); - for (final Concept c : concepts) { - final SubCommunity sc = new SubCommunity(); - sc.setSubCommunityId(c.getId()); - sc.setCommunityId(communityId); - sc.setParent(parent); - sc.setCategory(category); - sc.setLabel(c.getLabel()); - sc.setParams(c.getParams()); - list.add(sc); - list.addAll(asSubCommunities(communityId, c.getId(), category, c.getConcepts())); - } - return list; - } - - private static String getDecodedUrl(final String encoded_url) { - if (encoded_url == null) { return encoded_url; } - return new String(Base64.getDecoder().decode(encoded_url)); - } - - private static List splitValues(final Stream stream, final String separator) { - return stream.map(s -> s.split(separator)) - .map(Arrays::asList) - .flatMap(List::stream) - .filter(StringUtils::isNotBlank) - .map(StringUtils::trim) - .collect(Collectors.toList()); - } - - private static String firstValue(final Map> p, final String paramName) { - return asValues(p.get(paramName)).findFirst().orElse(null); - } - - private static String asCsv(final List params) { - return asValues(params) - .collect(Collectors.joining(CSV_DELIMITER)); - } - - private static Stream asValues(final List params) { - return params == null ? Stream.empty() - : params.stream() - .map(Param::getValue) - .map(StringUtils::trim) - .distinct(); - } - - private static LocalDateTime convertToLocalDateTime(final Date date) { - return date.toInstant() - .atZone(ZoneId.systemDefault()) - .toLocalDateTime(); - } - } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/db/CommunityImporterService.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/db/CommunityImporterService.java new file mode 100644 index 00000000..8b35122e --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/db/CommunityImporterService.java @@ -0,0 +1,335 @@ +package eu.dnetlib.openaire.community.db; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import javax.transaction.Transactional; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.dom4j.DocumentHelper; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Lists; + +import eu.dnetlib.openaire.community.db.model.DbOrganization; +import eu.dnetlib.openaire.community.db.repository.DbOrganizationRepository; +import eu.dnetlib.openaire.exporter.exceptions.CommunityException; +import eu.dnetlib.openaire.exporter.model.community.CommunityClaimType; +import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider; +import eu.dnetlib.openaire.exporter.model.community.CommunityDetails; +import eu.dnetlib.openaire.exporter.model.community.CommunityMembershipType; +import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization; +import eu.dnetlib.openaire.exporter.model.community.CommunityProject; +import eu.dnetlib.openaire.exporter.model.community.CommunityStatus; +import eu.dnetlib.openaire.exporter.model.community.CommunityType; +import eu.dnetlib.openaire.exporter.model.community.SubCommunity; +import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionCriteria; +import eu.dnetlib.openaire.exporter.model.context.Category; +import eu.dnetlib.openaire.exporter.model.context.Concept; +import eu.dnetlib.openaire.exporter.model.context.Context; +import eu.dnetlib.openaire.exporter.model.context.Param; + +@Service +public class CommunityImporterService { + + private final static String pattern = "yyyy-MM-dd'T'hh:mm:ss"; + + // common + private final static String OPENAIRE_ID = "openaireId"; + private final static String PIPE_SEPARATOR = "||"; + private final static String ID_SEPARATOR = "::"; + private final static String CSV_DELIMITER = ","; + private final static String CLABEL = "label"; + + // id suffixes + private final static String PROJECTS_ID_SUFFIX = ID_SEPARATOR + "projects"; + private final static String CONTENTPROVIDERS_ID_SUFFIX = ID_SEPARATOR + "contentproviders"; + private final static String ZENODOCOMMUNITY_ID_SUFFIX = ID_SEPARATOR + "zenodocommunities"; + private final static String ORGANIZATION_ID_SUFFIX = ID_SEPARATOR + "organizations"; + + // community summary + private final static String CSUMMARY_DESCRIPTION = "description"; + private final static String CSUMMARY_LOGOURL = "logourl"; + private final static String CSUMMARY_STATUS = "status"; + private final static String CSUMMARY_NAME = "name"; + private final static String CSUMMARY_MANAGER = "manager"; + private final static String CSUMMARY_ZENODOC = "zenodoCommunity"; + + // community profile + private final static String CPROFILE_SUBJECT = "subject"; + private final static String CPROFILE_CREATIONDATE = "creationdate"; + private final static String CPROFILE_FOS = "fos"; + private final static String CPROFILE_SDG = "sdg"; + private final static String CPROFILE_ADVANCED_CONSTRAINT = "advancedConstraints"; + private final static String CPROFILE_REMOVE_CONSTRAINT = "removeConstraints"; + + // community project + private final static String CPROJECT_FUNDER = "funder"; + private final static String CPROJECT_NUMBER = "CD_PROJECT_NUMBER"; + private final static String CPROJECT_FULLNAME = "projectfullname"; + private final static String CPROJECT_ACRONYM = "acronym"; + + // community content provider + private final static String CCONTENTPROVIDER_NAME = "name"; + private final static String CCONTENTPROVIDER_OFFICIALNAME = "officialname"; + private final static String CCONTENTPROVIDER_ENABLED = "enabled"; + private final static String CCONTENTPROVIDERENABLED_DEFAULT = "true"; + private final static String CCONTENTPROVIDER_SELCRITERIA = "selcriteria"; + + // community zenodo community + private final static String CZENODOCOMMUNITY_ID = "zenodoid"; + + // community organization + private final static String CORGANIZATION_NAME = "name"; + private final static String CORGANIZATION_LOGOURL = "logourl"; + private final static String CORGANIZATION_WEBSITEURL = "websiteurl"; + + @Autowired + private DbOrganizationRepository dbOrganizationRepository; + + @Autowired + private CommunityService service; + + private static final Log log = LogFactory.getLog(CommunityImporterService.class); + + @Transactional + public List importPropagationOrganizationsFromProfile(final String xml, final boolean simulation) throws Exception { + final String json = DocumentHelper.parseText(xml) + .selectSingleNode("//NODE[@name='setPropagationOrganizationCommunityMap']//PARAM[@name='parameterValue']") + .getText(); + + final List list = new ObjectMapper() + .readValue(json, new TypeReference>>() {}) + .entrySet() + .stream() + .flatMap(e -> e.getValue().stream().map(community -> new DbOrganization(community, e.getKey()))) + .collect(Collectors.toList()); + + if (!simulation) { + dbOrganizationRepository.saveAll(list); + } + + return list; + } + + @Transactional + public void importCommunity(final Context context) { + try { + + final CommunityDetails community = asCommunityDetails(context); + + final List datasources = + getCommunityInfo(context, CONTENTPROVIDERS_ID_SUFFIX, c -> asCommunityDataprovider(context.getId(), c)); + + final List projects = + getCommunityInfo(context, PROJECTS_ID_SUFFIX, c -> asCommunityProject(context.getId(), c)); + + final List orgs = + getCommunityInfo(context, ORGANIZATION_ID_SUFFIX, c -> asCommunityOrganization(context.getId(), c)); + + final List otherZenodoCimmunities = + getCommunityInfo(context, ZENODOCOMMUNITY_ID_SUFFIX, c -> asZenodoCommunity(c)); + + community.setOtherZenodoCommunities(otherZenodoCimmunities); + + final List subs = context.getCategories() + .entrySet() + .stream() + .filter(e -> !e.getKey().equals(context.getId() + CONTENTPROVIDERS_ID_SUFFIX)) + .filter(e -> !e.getKey().equals(context.getId() + PROJECTS_ID_SUFFIX)) + .filter(e -> !e.getKey().equals(context.getId() + ORGANIZATION_ID_SUFFIX)) + .filter(e -> !e.getKey().equals(context.getId() + ZENODOCOMMUNITY_ID_SUFFIX)) + .map(e -> e.getValue()) + .map(cat -> asSubCommunities(context.getId(), null, cat.getLabel(), cat.getConcepts())) + .flatMap(List::stream) + .collect(Collectors.toList()); + + service.saveCommunity(community); + service.addCommunityProjectList(context.getId(), projects); + service.addCommunityContentProvidersList(context.getId(), datasources); + service.addCommunityOrganizationList(context.getId(), orgs); + service.addSubCommunityList(subs); + } catch (final Exception e) { + throw new RuntimeException("Error importing community: " + context.getId(), e); + } + } + + private List getCommunityInfo(final Context context, final String idSuffix, final Function mapping) + throws CommunityException { + if (context != null) { + final Map categories = context.getCategories(); + final Category category = categories.get(context.getId() + idSuffix); + if (category != null) { return category.getConcepts() + .stream() + .map(mapping) + .collect(Collectors.toList()); } + } + return Lists.newArrayList(); + } + + private static CommunityDetails asCommunityDetails(final Context c) { + + final CommunityDetails details = new CommunityDetails(); + + details.setId(c.getId()); + details.setShortName(c.getLabel()); + details.setLastUpdateDate(convertToLocalDateTime(c.getLastUpdateDate())); + details.setCreationDate(convertToLocalDateTime(c.getCreationDate())); + details.setQueryId(c.getId() + PIPE_SEPARATOR + c.getLabel()); + details.setType(CommunityType.valueOf(c.getType())); + details.setMembership(CommunityMembershipType.open); + details.setClaim(CommunityClaimType.all); + + final Map> params = c.getParams(); + if (params.containsKey(CSUMMARY_DESCRIPTION)) { + details.setDescription(asCsv(params.get(CSUMMARY_DESCRIPTION))); + } + if (params.containsKey(CSUMMARY_LOGOURL)) { + details.setLogoUrl(asCsv(params.get(CSUMMARY_LOGOURL))); + } + if (params.containsKey(CSUMMARY_STATUS)) { + details.setStatus(CommunityStatus.valueOf(firstValue(params, CSUMMARY_STATUS))); + } + if (params.containsKey(CSUMMARY_NAME)) { + details.setName(asCsv(params.get(CSUMMARY_NAME))); + } else { + details.setName(c.getLabel()); + } + if (params.containsKey(CSUMMARY_ZENODOC)) { + details.setZenodoCommunity(asCsv(params.get(CSUMMARY_ZENODOC))); + } + if (params.containsKey(CPROFILE_SUBJECT)) { + details.setSubjects(splitValues(asValues(params.get(CPROFILE_SUBJECT)), CSV_DELIMITER)); + } + if (params.containsKey(CPROFILE_FOS)) { + details.setFos(splitValues(asValues(params.get(CPROFILE_FOS)), CSV_DELIMITER)); + } + if (params.containsKey(CPROFILE_SDG)) { + details.setSdg(splitValues(asValues(params.get(CPROFILE_SDG)), CSV_DELIMITER)); + } + if (params.containsKey(CPROFILE_ADVANCED_CONSTRAINT)) { + // In the map the string is the serialization of the json representing the selection criteria so it is a valid json + details.setAdvancedConstraints(SelectionCriteria.fromJson(asCsv(params.get(CPROFILE_ADVANCED_CONSTRAINT)))); + } + if (params.containsKey(CPROFILE_REMOVE_CONSTRAINT)) { + // In the map the string is the serialization of the json representing the selection criteria so it is a valid json + details.setRemoveConstraints(SelectionCriteria.fromJson(asCsv(params.get(CPROFILE_REMOVE_CONSTRAINT)))); + } + if (params.containsKey(CPROFILE_CREATIONDATE)) { + try { + final Date d = org.apache.commons.lang3.time.DateUtils.parseDate(asCsv(params.get(CPROFILE_CREATIONDATE)), pattern); + details.setCreationDate(convertToLocalDateTime(d)); + } catch (final Exception e) { + log.debug("Exception on date format: " + e.getMessage()); + } + } + + return details; + } + + private static CommunityProject asCommunityProject(final String communityId, final Concept c) { + final Map> p = c.getParams(); + final CommunityProject project = new CommunityProject(); + project.setCommunityId(communityId); + project.setOpenaireId(firstValue(p, OPENAIRE_ID)); + project.setFunder(firstValue(p, CPROJECT_FUNDER)); + project.setGrantId(firstValue(p, CPROJECT_NUMBER)); + project.setName(firstValue(p, CPROJECT_FULLNAME)); + project.setAcronym(firstValue(p, CPROJECT_ACRONYM)); + return project; + } + + private static CommunityContentprovider asCommunityDataprovider(final String communityId, final Concept c) { + final Map> p = c.getParams(); + final CommunityContentprovider d = new CommunityContentprovider(); + d.setCommunityId(communityId); + d.setOpenaireId(firstValue(p, OPENAIRE_ID)); + d.setName(firstValue(p, CCONTENTPROVIDER_NAME)); + d.setOfficialname(firstValue(p, CCONTENTPROVIDER_OFFICIALNAME)); + d.setSelectioncriteria(SelectionCriteria.fromJson(firstValue(p, CCONTENTPROVIDER_SELCRITERIA))); + return d; + } + + private static CommunityOrganization asCommunityOrganization(final String id, final Concept c) { + final Map> p = c.getParams(); + final CommunityOrganization o = new CommunityOrganization(); + o.setCommunityId(id); + // o.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR)); + o.setName(firstValue(p, CORGANIZATION_NAME)); + o.setLogo_url(getDecodedUrl(firstValue(p, CORGANIZATION_LOGOURL))); + o.setWebsite_url(getDecodedUrl(firstValue(p, CORGANIZATION_WEBSITEURL))); + return o; + } + + private static String asZenodoCommunity(final Concept c) { + return firstValue(c.getParams(), CZENODOCOMMUNITY_ID); + } + + private static List asSubCommunities(final String communityId, final String parent, final String category, final List concepts) { + final List list = new ArrayList<>(); + for (final Concept c : concepts) { + final SubCommunity sc = new SubCommunity(); + sc.setSubCommunityId(c.getId()); + sc.setCommunityId(communityId); + sc.setParent(parent); + sc.setCategory(category); + sc.setLabel(c.getLabel()); + sc.setParams(c.getParams()); + sc.setClaim(c.isClaim()); + list.add(sc); + list.addAll(asSubCommunities(communityId, c.getId(), category, c.getConcepts())); + } + return list; + } + + private static String getDecodedUrl(final String encoded_url) { + if (encoded_url == null) { return encoded_url; } + return new String(Base64.getDecoder().decode(encoded_url)); + } + + private static List splitValues(final Stream stream, final String separator) { + return stream.map(s -> s.split(separator)) + .map(Arrays::asList) + .flatMap(List::stream) + .filter(StringUtils::isNotBlank) + .map(StringUtils::trim) + .collect(Collectors.toList()); + } + + private static String firstValue(final Map> p, final String paramName) { + return asValues(p.get(paramName)).findFirst().orElse(null); + } + + private static String asCsv(final List params) { + return asValues(params) + .collect(Collectors.joining(CSV_DELIMITER)); + } + + private static Stream asValues(final List params) { + return params == null ? Stream.empty() + : params.stream() + .map(Param::getValue) + .map(StringUtils::trim) + .distinct(); + } + + private static LocalDateTime convertToLocalDateTime(final Date date) { + return date.toInstant() + .atZone(ZoneId.systemDefault()) + .toLocalDateTime(); + } +} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/db/CommunityService.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/db/CommunityService.java index fb310f0f..3af9e9cb 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/db/CommunityService.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/db/CommunityService.java @@ -48,12 +48,9 @@ import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionC public class CommunityService { // TODO - // 1) Import tramite profili: - // 1.1) Completare CommunityImporterController - // 2) Subcommunities: visualizzazione tramite le context api - // 3) Importare le orgs dal wf di provision, ad esempio: - // https://services.openaire.eu/is/mvc/ui/isManager.do#/profile/4801c33c-66ca-4ab6-af64-aa812194ec61_V29ya2Zsb3dEU1Jlc291cmNlcy9Xb3JrZmxvd0RTUmVzb3VyY2VUeXBl - // 4) Verificare Tickets: #8835, #8854, #6483, #3259, #3494 + // 1) Subcommunities: visualizzazione tramite le context api + // 2) Test di import (context e wf) + // 3) Verificare Tickets: #8835, #8854, #6483, #3259, #3494 @Autowired private DbCommunityRepository dbCommunityRepository; diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/db/model/DbSubCommunity.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/db/model/DbSubCommunity.java index 41b8cefc..26924ff1 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/db/model/DbSubCommunity.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/db/model/DbSubCommunity.java @@ -1,6 +1,7 @@ package eu.dnetlib.openaire.community.db.model; import java.io.Serializable; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -45,11 +46,14 @@ public class DbSubCommunity implements Serializable { @Type(type = "jsonb") @Column(name = "params") - private Map> params; + private Map> params = new LinkedHashMap<>(); @Column(name = "parent") private String parent; + @Column(name = "claim") + private boolean claim = false; + public String getId() { return id; } @@ -97,4 +101,12 @@ public class DbSubCommunity implements Serializable { public void setParent(final String parent) { this.parent = parent; } + + public boolean isClaim() { + return claim; + } + + public void setClaim(final boolean claim) { + this.claim = claim; + } } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/db/utils/CommunityMappingUtils.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/db/utils/CommunityMappingUtils.java index 26d255fd..383e582a 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/db/utils/CommunityMappingUtils.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/db/utils/CommunityMappingUtils.java @@ -199,6 +199,7 @@ public class CommunityMappingUtils { dbsc.setLabel(sub.getLabel()); dbsc.setParams(sub.getParams()); dbsc.setParent(sub.getParent()); + dbsc.setClaim(sub.isClaim()); return dbsc; } @@ -210,6 +211,7 @@ public class CommunityMappingUtils { sc.setLabel(sub.getLabel()); sc.setParams(sub.getParams()); sc.setParent(sub.getParent()); + sc.setClaim(sub.isClaim()); return sc; } diff --git a/apps/dnet-exporter-api/src/main/resources/sql/community-schema.sql b/apps/dnet-exporter-api/src/main/resources/sql/community-schema.sql index 982af224..b72b2453 100644 --- a/apps/dnet-exporter-api/src/main/resources/sql/community-schema.sql +++ b/apps/dnet-exporter-api/src/main/resources/sql/community-schema.sql @@ -1,3 +1,4 @@ +DROP TABLE IF EXISTS community_subs; DROP TABLE IF EXISTS community_projects; DROP TABLE IF EXISTS community_datasources; DROP TABLE IF EXISTS community_support_orgs; @@ -60,10 +61,11 @@ CREATE TABLE community_orgs ( CREATE TABLE community_subs ( - sub_id text NOT NULL PRIMARY KEY, - community text NOT NULL REFERENCES communities(id), - label text NOT NULL, - category text NOT NULL, + sub_id text NOT NULL PRIMARY KEY, + community text NOT NULL REFERENCES communities(id), + label text NOT NULL, + category text NOT NULL, + claim boolean NOT NULL DEFAULT false, params jsonb, parent text REFERENCES community_subs(sub_id) -- NULL for the first level ); diff --git a/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/community/SubCommunity.java b/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/community/SubCommunity.java index 03818ed5..c6eedc12 100644 --- a/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/community/SubCommunity.java +++ b/libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/community/SubCommunity.java @@ -30,6 +30,9 @@ public class SubCommunity { @Schema(description = "the parameters of the subCommunity", required = true) private Map> params = new LinkedHashMap<>(); + @Schema(description = "it supports the claims", required = true) + private boolean claim = false; + public String getSubCommunityId() { return subCommunityId; } @@ -78,4 +81,12 @@ public class SubCommunity { this.params = map; } + public boolean isClaim() { + return claim; + } + + public void setClaim(final boolean claim) { + this.claim = claim; + } + }