From 174a0ea9f24eace004da788607368fef78f9f9f4 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 26 Jun 2023 14:56:40 +0200 Subject: [PATCH] cleaning --- .../openaire/community/CommunityCommon.java | 293 ------------------ .../CommunityImporterController.java | 86 +++-- .../community/CommunityMappingUtils.java | 75 +++-- 3 files changed, 113 insertions(+), 341 deletions(-) delete mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityCommon.java diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityCommon.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityCommon.java deleted file mode 100644 index 789f7d48..00000000 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityCommon.java +++ /dev/null @@ -1,293 +0,0 @@ -package eu.dnetlib.openaire.community; - -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.function.Function; -import java.util.stream.Collectors; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import com.google.common.base.Joiner; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import com.google.gson.Gson; - -import eu.dnetlib.openaire.common.ISClient; -import eu.dnetlib.openaire.exporter.exceptions.CommunityException; -import eu.dnetlib.openaire.exporter.exceptions.ResourceNotFoundException; -import eu.dnetlib.openaire.exporter.model.community.CommunityDetails; -import eu.dnetlib.openaire.exporter.model.community.CommunitySummary; -import eu.dnetlib.openaire.exporter.model.community.CommunityWritableProperties; -import eu.dnetlib.openaire.exporter.model.community.CommunityZenodoCommunity; -import eu.dnetlib.openaire.exporter.model.context.Category; -import eu.dnetlib.openaire.exporter.model.context.Concept; -import eu.dnetlib.openaire.exporter.model.context.Context; -import eu.dnetlib.openaire.exporter.model.context.Param; - -@Component -@Deprecated -public class CommunityCommon { - - public final static Set communityBlackList = Sets.newHashSet("fet-fp7", "fet-h2020"); - - // common - public final static String OPENAIRE_ID = "openaireId"; - public final static String PIPE_SEPARATOR = "||"; - public final static String ID_SEPARATOR = "::"; - public final static String CSV_DELIMITER = ","; - public final static String CLABEL = "label"; - - // id suffixes - public final static String PROJECTS_ID_SUFFIX = ID_SEPARATOR + "projects"; - public final static String CONTENTPROVIDERS_ID_SUFFIX = ID_SEPARATOR + "contentproviders"; - public final static String ZENODOCOMMUNITY_ID_SUFFIX = ID_SEPARATOR + "zenodocommunities"; - public final static String ORGANIZATION_ID_SUFFIX = ID_SEPARATOR + "organizations"; - - // community summary - public final static String CSUMMARY_DESCRIPTION = "description"; - public final static String CSUMMARY_LOGOURL = "logourl"; - public final static String CSUMMARY_STATUS = "status"; - public final static String CSUMMARY_NAME = "name"; - public final static String CSUMMARY_MANAGER = "manager"; - public final static String CSUMMARY_ZENODOC = "zenodoCommunity"; - - // community profile - public final static String CPROFILE_SUBJECT = "subject"; - public final static String CPROFILE_CREATIONDATE = "creationdate"; - public final static String CPROFILE_FOS = "fos"; - public final static String CPROFILE_SDG = "sdg"; - public final static String CPROFILE_ADVANCED_CONSTRAINT = "advancedConstraints"; - - // community project - public final static String CPROJECT_FUNDER = "funder"; - public final static String CPROJECT_NUMBER = "CD_PROJECT_NUMBER"; - public final static String CPROJECT_FULLNAME = "projectfullname"; - public final static String CPROJECT_ACRONYM = "acronym"; - - // community content provider - public final static String CCONTENTPROVIDER_NAME = "name"; - public final static String CCONTENTPROVIDER_OFFICIALNAME = "officialname"; - public final static String CCONTENTPROVIDER_ENABLED = "enabled"; - public final static String CCONTENTPROVIDERENABLED_DEFAULT = "true"; - public final static String CCONTENTPROVIDER_SELCRITERIA = "selcriteria"; - - // community zenodo community - public final static String CZENODOCOMMUNITY_ID = "zenodoid"; - - // community organization - public final static String CORGANIZATION_NAME = "name"; - public final static String CORGANIZATION_LOGOURL = "logourl"; - public final static String CORGANIZATION_WEBSITEURL = "websiteurl"; - - @Autowired - private ISClient isClient; - - public Map getContextMap() throws CommunityException { - try { - return isClient.getCommunityContextMap(); - } catch (final IOException e) { - throw new CommunityException(e); - } - } - - public List listCommunities() throws CommunityException { - return getContextMap().values() - .stream() - .filter(context -> !communityBlackList.contains(context.getId())) - .map(CommunityMappingUtils::asCommunitySummary) - .collect(Collectors.toList()); - } - - public List getCommunityInfo(final String id, final String idSuffix, final Function mapping) throws CommunityException { - final Map contextMap = getContextMap(); - final Context context = contextMap.get(id); - if (context != null) { - final Map categories = context.getCategories(); - final Category category = categories.get(id + idSuffix); - if (category != null) { return category.getConcepts() - .stream() - .map(mapping) - .collect(Collectors.toList()); } - } - return Lists.newArrayList(); - } - - public CommunityDetails getCommunity(final String id) throws CommunityException, ResourceNotFoundException { - final Context context = getContextMap().get(id); - if (context == null || communityBlackList.contains(id)) { - // ResponseStatusException(NOT_FOUND, "Unable to find resource"); - throw new ResourceNotFoundException(); - } - return CommunityMappingUtils.asCommunityProfile(context); - } - - public List getCommunityZenodoCommunities(final String id) throws CommunityException, ResourceNotFoundException { - getCommunity(id); // ensure the community exists. - return getCommunityInfo(id, ZENODOCOMMUNITY_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityZenodoCommunity(id, c)); - } - - public void removeAdvancedConstraint(final String id) throws CommunityException { - final Context context = getContextMap().get(id); - context.getParams() - .replace(CPROFILE_ADVANCED_CONSTRAINT, Arrays.asList(new Param() - .setName(CPROFILE_ADVANCED_CONSTRAINT) - .setValue(null))); - } - - public void updateCommunity(final String id, final CommunityWritableProperties community) throws CommunityException { - final Context context = getContextMap().get(id); - - if (community.getShortName() != null) { - context.setLabel(community.getShortName()); - } - - if (community.getName() != null) { - context.getParams() - .replace(CSUMMARY_NAME, Arrays.asList(new Param() - .setValue(community.getName()) - .setName(CSUMMARY_NAME))); - } - if (community.getDescription() != null) { - context.getParams() - .replace(CSUMMARY_DESCRIPTION, Arrays.asList(new Param() - .setName(CSUMMARY_DESCRIPTION) - .setValue(community.getDescription()))); - } - if (community.getLogoUrl() != null) { - context.getParams() - .replace(CSUMMARY_LOGOURL, Arrays.asList(new Param() - .setName(CSUMMARY_LOGOURL) - .setValue(community.getLogoUrl()))); - - } - if (community.getStatus() != null) { - context.getParams() - .replace(CSUMMARY_STATUS, Arrays.asList(new Param() - .setName(CSUMMARY_STATUS) - .setValue(community.getStatus().name()))); - } - if (community.getSubjects() != null) { - context.getParams() - .replace(CPROFILE_SUBJECT, Arrays.asList(new Param().setName(CPROFILE_SUBJECT) - .setValue(Joiner.on(CSV_DELIMITER) - .join(community.getSubjects())))); - } - if (community.getFos() != null) { - if (context.getParams().containsKey(CPROFILE_FOS)) { - context.getParams() - .replace(CPROFILE_FOS, Arrays.asList(new Param().setName(CPROFILE_FOS) - .setValue(Joiner.on(CSV_DELIMITER) - .join(community.getFos())))); - } else { - context.getParams() - .put(CPROFILE_FOS, Arrays.asList(new Param().setName(CPROFILE_FOS) - .setValue(Joiner.on(CSV_DELIMITER) - .join(community.getFos())))); - } - } - if (community.getSdg() != null) { - if (context.getParams().containsKey(CPROFILE_SDG)) { - context.getParams() - .replace(CPROFILE_SDG, Arrays.asList(new Param().setName(CPROFILE_SDG) - .setValue(Joiner.on(CSV_DELIMITER) - .join(community.getSdg())))); - } else { - context.getParams() - .put(CPROFILE_SDG, Arrays.asList(new Param().setName(CPROFILE_SDG) - .setValue(Joiner.on(CSV_DELIMITER) - .join(community.getSdg())))); - } - } - if (community.getAdvancedConstraints() != null) { - if (context.getParams().containsKey(CPROFILE_ADVANCED_CONSTRAINT)) { - context.getParams() - .replace(CPROFILE_ADVANCED_CONSTRAINT, Arrays.asList(new Param() - .setName(CPROFILE_ADVANCED_CONSTRAINT) - .setValue(new Gson().toJson(community.getAdvancedConstraints())))); - } else { - context.getParams() - .put(CPROFILE_ADVANCED_CONSTRAINT, Arrays.asList(new Param() - .setName(CPROFILE_ADVANCED_CONSTRAINT) - .setValue(new Gson().toJson(community.getAdvancedConstraints())))); - } - } - if (community.getMainZenodoCommunity() != null) { - context.getParams() - .replace(CSUMMARY_ZENODOC, Arrays.asList(new Param() - .setName(CSUMMARY_ZENODOC) - .setValue(community.getMainZenodoCommunity()))); - - } - - } - - public void removeFromCategory(final String communityId, final String category, final String conceptId) throws CommunityException { - final Map cmap = getContextMap(); - final Context context = cmap.get(communityId); - final Map cat = context.getCategories(); - - final List concepts = cat.get(communityId + category) - .getConcepts() - .stream() - .filter(c -> !c.getId().equals(communityId + category + ID_SEPARATOR + conceptId)) - .collect(Collectors.toList()); - - cat.get(communityId + category).setConcepts(concepts); - } - - public void updateZenodoCommunity(final String communityId, final CommunityZenodoCommunity zc) throws CommunityException { - final Context context = getContextMap().get(communityId); - final Category zcs = context.getCategories().get(communityId + ZENODOCOMMUNITY_ID_SUFFIX); - if (zcs.getConcepts() - .stream() - .map(c -> c.getId()) - .collect(Collectors.toList()) - .contains(communityId + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zc.getId())) { - zcs.getConcepts().forEach(concept -> { - if (concept.getId().equals(communityId + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zc.getId())) { - - if (zc.getZenodoid() != null) { - if (concept.getParams().keySet().contains(CZENODOCOMMUNITY_ID)) { - concept.getParams() - .replace(CZENODOCOMMUNITY_ID, Arrays.asList(new Param() - .setName(CZENODOCOMMUNITY_ID) - .setValue(zc.getZenodoid()))); - } else { - concept.getParams() - .put(CZENODOCOMMUNITY_ID, Arrays.asList(new Param() - .setName(CZENODOCOMMUNITY_ID) - .setValue(zc.getZenodoid()))); - } - - } - - } - - }); - } else { - - final Concept concept = new Concept(); - concept.setId(communityId + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zc.getId()); - concept.setClaim(false); - - final Map> params = new TreeMap<>(); - - if (zc.getZenodoid() != null) { - params.put(CZENODOCOMMUNITY_ID, Arrays.asList(new Param().setValue(zc.getZenodoid()).setName(CZENODOCOMMUNITY_ID))); - concept.setLabel(zc.getZenodoid()); - } else { - concept.setLabel(""); - } - concept.setParams(params); - zcs.getConcepts().add(concept); - - } - - } - -} diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityImporterController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityImporterController.java index 4d1c691b..a8ead9f9 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityImporterController.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityImporterController.java @@ -1,60 +1,102 @@ package eu.dnetlib.openaire.community; -import static eu.dnetlib.openaire.community.CommunityCommon.CONTENTPROVIDERS_ID_SUFFIX; -import static eu.dnetlib.openaire.community.CommunityCommon.ORGANIZATION_ID_SUFFIX; -import static eu.dnetlib.openaire.community.CommunityCommon.PROJECTS_ID_SUFFIX; +import static eu.dnetlib.openaire.community.CommunityMappingUtils.CONTENTPROVIDERS_ID_SUFFIX; +import static eu.dnetlib.openaire.community.CommunityMappingUtils.ORGANIZATION_ID_SUFFIX; +import static eu.dnetlib.openaire.community.CommunityMappingUtils.PROJECTS_ID_SUFFIX; +import java.io.IOException; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RestController; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; + +import eu.dnetlib.openaire.common.ISClient; import eu.dnetlib.openaire.community.db.CommunityService; import eu.dnetlib.openaire.exporter.exceptions.CommunityException; import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider; import eu.dnetlib.openaire.exporter.model.community.CommunityDetails; import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization; import eu.dnetlib.openaire.exporter.model.community.CommunityProject; +import eu.dnetlib.openaire.exporter.model.context.Category; +import eu.dnetlib.openaire.exporter.model.context.Concept; +import eu.dnetlib.openaire.exporter.model.context.Context; @RestController public class CommunityImporterController { + public final static Set communityBlackList = Sets.newHashSet("fet-fp7", "fet-h2020"); + @Autowired private CommunityService service; @Autowired - private CommunityCommon cc; + private ISClient isClient; @GetMapping("/community_importer/import") public List importProfiles() throws CommunityException { - final List list = cc.listCommunities() + final Map contextMap = getContextMap(); + + final List list = contextMap.keySet() .stream() - .map(c -> c.getId()) + .filter(id -> !communityBlackList.contains(id)) .collect(Collectors.toList()); - list.forEach(this::importCommunity); + list.forEach(id -> { + importCommunity(contextMap.get(id)); + }); return list; } - private void importCommunity(final String id) { + private Map getContextMap() throws CommunityException { try { - - final CommunityDetails community = cc.getCommunity(id); - final List datasources = - cc.getCommunityInfo(id, CONTENTPROVIDERS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityDataprovider(id, c)); - final List projects = cc.getCommunityInfo(id, PROJECTS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityProject(id, c)); - final List orgs = cc.getCommunityInfo(id, ORGANIZATION_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityOrganization(id, c)); - - service.saveCommunity(community); - service.addCommunityProjectList(id, projects); - service.addCommunityContentProvidersList(id, datasources); - service.addCommunityOrganizationList(id, orgs); - // TODO MANAGE new fields and tables - } catch (final Exception e) { - throw new RuntimeException("Error importing community: " + id, e); + return isClient.getCommunityContextMap(); + } catch (final IOException e) { + throw new CommunityException(e); } } + + public List getCommunityInfo(final Context context, final String idSuffix, final Function mapping) + throws CommunityException { + if (context != null) { + final Map categories = context.getCategories(); + final Category category = categories.get(context.getId() + idSuffix); + if (category != null) { return category.getConcepts() + .stream() + .map(mapping) + .collect(Collectors.toList()); } + } + return Lists.newArrayList(); + } + + private void importCommunity(final Context context) { + try { + + final CommunityDetails community = CommunityMappingUtils.asCommunityProfile(context); + + final List datasources = + getCommunityInfo(context, CONTENTPROVIDERS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityDataprovider(context.getId(), c)); + final List projects = + getCommunityInfo(context, PROJECTS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityProject(context.getId(), c)); + final List orgs = + getCommunityInfo(context, ORGANIZATION_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityOrganization(context.getId(), c)); + + service.saveCommunity(community); + service.addCommunityProjectList(context.getId(), projects); + service.addCommunityContentProvidersList(context.getId(), datasources); + service.addCommunityOrganizationList(context.getId(), orgs); + // TODO MANAGE new fields and tables + } catch (final Exception e) { + throw new RuntimeException("Error importing community: " + context.getId(), e); + } + } + } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityMappingUtils.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityMappingUtils.java index a1ac2966..201ff5c3 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityMappingUtils.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityMappingUtils.java @@ -1,32 +1,6 @@ package eu.dnetlib.openaire.community; import static eu.dnetlib.openaire.common.Utils.escape; -import static eu.dnetlib.openaire.community.CommunityCommon.CCONTENTPROVIDER_NAME; -import static eu.dnetlib.openaire.community.CommunityCommon.CCONTENTPROVIDER_OFFICIALNAME; -import static eu.dnetlib.openaire.community.CommunityCommon.CCONTENTPROVIDER_SELCRITERIA; -import static eu.dnetlib.openaire.community.CommunityCommon.CORGANIZATION_LOGOURL; -import static eu.dnetlib.openaire.community.CommunityCommon.CORGANIZATION_NAME; -import static eu.dnetlib.openaire.community.CommunityCommon.CORGANIZATION_WEBSITEURL; -import static eu.dnetlib.openaire.community.CommunityCommon.CPROFILE_ADVANCED_CONSTRAINT; -import static eu.dnetlib.openaire.community.CommunityCommon.CPROFILE_CREATIONDATE; -import static eu.dnetlib.openaire.community.CommunityCommon.CPROFILE_FOS; -import static eu.dnetlib.openaire.community.CommunityCommon.CPROFILE_SDG; -import static eu.dnetlib.openaire.community.CommunityCommon.CPROFILE_SUBJECT; -import static eu.dnetlib.openaire.community.CommunityCommon.CPROJECT_ACRONYM; -import static eu.dnetlib.openaire.community.CommunityCommon.CPROJECT_FULLNAME; -import static eu.dnetlib.openaire.community.CommunityCommon.CPROJECT_FUNDER; -import static eu.dnetlib.openaire.community.CommunityCommon.CPROJECT_NUMBER; -import static eu.dnetlib.openaire.community.CommunityCommon.CSUMMARY_DESCRIPTION; -import static eu.dnetlib.openaire.community.CommunityCommon.CSUMMARY_LOGOURL; -import static eu.dnetlib.openaire.community.CommunityCommon.CSUMMARY_NAME; -import static eu.dnetlib.openaire.community.CommunityCommon.CSUMMARY_STATUS; -import static eu.dnetlib.openaire.community.CommunityCommon.CSUMMARY_ZENODOC; -import static eu.dnetlib.openaire.community.CommunityCommon.CSV_DELIMITER; -import static eu.dnetlib.openaire.community.CommunityCommon.CZENODOCOMMUNITY_ID; -import static eu.dnetlib.openaire.community.CommunityCommon.ID_SEPARATOR; -import static eu.dnetlib.openaire.community.CommunityCommon.OPENAIRE_ID; -import static eu.dnetlib.openaire.community.CommunityCommon.PIPE_SEPARATOR; -import static eu.dnetlib.openaire.community.CommunityCommon.ZENODOCOMMUNITY_ID_SUFFIX; import java.time.LocalDateTime; import java.time.ZoneId; @@ -63,6 +37,55 @@ public class CommunityMappingUtils { private final static String pattern = "yyyy-MM-dd'T'hh:mm:ss"; + // common + public final static String OPENAIRE_ID = "openaireId"; + public final static String PIPE_SEPARATOR = "||"; + public final static String ID_SEPARATOR = "::"; + public final static String CSV_DELIMITER = ","; + public final static String CLABEL = "label"; + + // id suffixes + public final static String PROJECTS_ID_SUFFIX = ID_SEPARATOR + "projects"; + public final static String CONTENTPROVIDERS_ID_SUFFIX = ID_SEPARATOR + "contentproviders"; + public final static String ZENODOCOMMUNITY_ID_SUFFIX = ID_SEPARATOR + "zenodocommunities"; + public final static String ORGANIZATION_ID_SUFFIX = ID_SEPARATOR + "organizations"; + + // community summary + public final static String CSUMMARY_DESCRIPTION = "description"; + public final static String CSUMMARY_LOGOURL = "logourl"; + public final static String CSUMMARY_STATUS = "status"; + public final static String CSUMMARY_NAME = "name"; + public final static String CSUMMARY_MANAGER = "manager"; + public final static String CSUMMARY_ZENODOC = "zenodoCommunity"; + + // community profile + public final static String CPROFILE_SUBJECT = "subject"; + public final static String CPROFILE_CREATIONDATE = "creationdate"; + public final static String CPROFILE_FOS = "fos"; + public final static String CPROFILE_SDG = "sdg"; + public final static String CPROFILE_ADVANCED_CONSTRAINT = "advancedConstraints"; + + // community project + public final static String CPROJECT_FUNDER = "funder"; + public final static String CPROJECT_NUMBER = "CD_PROJECT_NUMBER"; + public final static String CPROJECT_FULLNAME = "projectfullname"; + public final static String CPROJECT_ACRONYM = "acronym"; + + // community content provider + public final static String CCONTENTPROVIDER_NAME = "name"; + public final static String CCONTENTPROVIDER_OFFICIALNAME = "officialname"; + public final static String CCONTENTPROVIDER_ENABLED = "enabled"; + public final static String CCONTENTPROVIDERENABLED_DEFAULT = "true"; + public final static String CCONTENTPROVIDER_SELCRITERIA = "selcriteria"; + + // community zenodo community + public final static String CZENODOCOMMUNITY_ID = "zenodoid"; + + // community organization + public final static String CORGANIZATION_NAME = "name"; + public final static String CORGANIZATION_LOGOURL = "logourl"; + public final static String CORGANIZATION_WEBSITEURL = "websiteurl"; + private static final Log log = LogFactory.getLog(CommunityMappingUtils.class); public static CommunitySummary asCommunitySummary(final Context c) {