diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityImporterController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityImporterController.java index a8ead9f9..fe4b913b 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityImporterController.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityImporterController.java @@ -1,16 +1,21 @@ package eu.dnetlib.openaire.community; -import static eu.dnetlib.openaire.community.CommunityMappingUtils.CONTENTPROVIDERS_ID_SUFFIX; -import static eu.dnetlib.openaire.community.CommunityMappingUtils.ORGANIZATION_ID_SUFFIX; -import static eu.dnetlib.openaire.community.CommunityMappingUtils.PROJECTS_ID_SUFFIX; - import java.io.IOException; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Arrays; +import java.util.Base64; +import java.util.Date; import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RestController; @@ -25,13 +30,72 @@ import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider; import eu.dnetlib.openaire.exporter.model.community.CommunityDetails; import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization; import eu.dnetlib.openaire.exporter.model.community.CommunityProject; +import eu.dnetlib.openaire.exporter.model.community.CommunityStatus; +import eu.dnetlib.openaire.exporter.model.community.CommunitySummary; +import eu.dnetlib.openaire.exporter.model.community.CommunityType; +import eu.dnetlib.openaire.exporter.model.community.CommunityZenodoCommunity; +import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionCriteria; import eu.dnetlib.openaire.exporter.model.context.Category; import eu.dnetlib.openaire.exporter.model.context.Concept; import eu.dnetlib.openaire.exporter.model.context.Context; +import eu.dnetlib.openaire.exporter.model.context.Param; @RestController public class CommunityImporterController { + private final static String pattern = "yyyy-MM-dd'T'hh:mm:ss"; + + // common + private final static String OPENAIRE_ID = "openaireId"; + private final static String PIPE_SEPARATOR = "||"; + private final static String ID_SEPARATOR = "::"; + private final static String CSV_DELIMITER = ","; + private final static String CLABEL = "label"; + + // id suffixes + private final static String PROJECTS_ID_SUFFIX = ID_SEPARATOR + "projects"; + private final static String CONTENTPROVIDERS_ID_SUFFIX = ID_SEPARATOR + "contentproviders"; + private final static String ZENODOCOMMUNITY_ID_SUFFIX = ID_SEPARATOR + "zenodocommunities"; + private final static String ORGANIZATION_ID_SUFFIX = ID_SEPARATOR + "organizations"; + + // community summary + private final static String CSUMMARY_DESCRIPTION = "description"; + private final static String CSUMMARY_LOGOURL = "logourl"; + private final static String CSUMMARY_STATUS = "status"; + private final static String CSUMMARY_NAME = "name"; + private final static String CSUMMARY_MANAGER = "manager"; + private final static String CSUMMARY_ZENODOC = "zenodoCommunity"; + + // community profile + private final static String CPROFILE_SUBJECT = "subject"; + private final static String CPROFILE_CREATIONDATE = "creationdate"; + private final static String CPROFILE_FOS = "fos"; + private final static String CPROFILE_SDG = "sdg"; + private final static String CPROFILE_ADVANCED_CONSTRAINT = "advancedConstraints"; + + // community project + private final static String CPROJECT_FUNDER = "funder"; + private final static String CPROJECT_NUMBER = "CD_PROJECT_NUMBER"; + private final static String CPROJECT_FULLNAME = "projectfullname"; + private final static String CPROJECT_ACRONYM = "acronym"; + + // community content provider + private final static String CCONTENTPROVIDER_NAME = "name"; + private final static String CCONTENTPROVIDER_OFFICIALNAME = "officialname"; + private final static String CCONTENTPROVIDER_ENABLED = "enabled"; + private final static String CCONTENTPROVIDERENABLED_DEFAULT = "true"; + private final static String CCONTENTPROVIDER_SELCRITERIA = "selcriteria"; + + // community zenodo community + private final static String CZENODOCOMMUNITY_ID = "zenodoid"; + + // community organization + private final static String CORGANIZATION_NAME = "name"; + private final static String CORGANIZATION_LOGOURL = "logourl"; + private final static String CORGANIZATION_WEBSITEURL = "websiteurl"; + + private static final Log log = LogFactory.getLog(CommunityImporterController.class); + public final static Set communityBlackList = Sets.newHashSet("fet-fp7", "fet-h2020"); @Autowired @@ -80,14 +144,14 @@ public class CommunityImporterController { private void importCommunity(final Context context) { try { - final CommunityDetails community = CommunityMappingUtils.asCommunityProfile(context); + final CommunityDetails community = asCommunityProfile(context); final List datasources = - getCommunityInfo(context, CONTENTPROVIDERS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityDataprovider(context.getId(), c)); + getCommunityInfo(context, CONTENTPROVIDERS_ID_SUFFIX, c -> asCommunityDataprovider(context.getId(), c)); final List projects = - getCommunityInfo(context, PROJECTS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityProject(context.getId(), c)); + getCommunityInfo(context, PROJECTS_ID_SUFFIX, c -> asCommunityProject(context.getId(), c)); final List orgs = - getCommunityInfo(context, ORGANIZATION_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityOrganization(context.getId(), c)); + getCommunityInfo(context, ORGANIZATION_ID_SUFFIX, c -> asCommunityOrganization(context.getId(), c)); service.saveCommunity(community); service.addCommunityProjectList(context.getId(), projects); @@ -99,4 +163,152 @@ public class CommunityImporterController { } } + private static CommunitySummary asCommunitySummary(final Context c) { + final CommunitySummary summary = new CommunitySummary(); + + summary.setId(c.getId()); + summary.setShortName(c.getLabel()); + summary.setLastUpdateDate(convertToLocalDateTime(c.getLastUpdateDate())); + summary.setCreationDate(convertToLocalDateTime(c.getCreationDate())); + summary.setQueryId(c.getId() + PIPE_SEPARATOR + c.getLabel()); + summary.setType(CommunityType.valueOf(c.getType())); + + final Map> params = c.getParams(); + if (params.containsKey(CSUMMARY_DESCRIPTION)) { + summary.setDescription(asCsv(params.get(CSUMMARY_DESCRIPTION))); + } + if (params.containsKey(CSUMMARY_LOGOURL)) { + summary.setLogoUrl(asCsv(params.get(CSUMMARY_LOGOURL))); + } + if (params.containsKey(CSUMMARY_STATUS)) { + summary.setStatus(CommunityStatus.valueOf(firstValue(params, CSUMMARY_STATUS))); + } + if (params.containsKey(CSUMMARY_NAME)) { + summary.setName(asCsv(params.get(CSUMMARY_NAME))); + } + if (params.containsKey(CSUMMARY_ZENODOC)) { + summary.setZenodoCommunity(asCsv(params.get(CSUMMARY_ZENODOC))); + } + + return summary; + } + + private static CommunityDetails asCommunityProfile(final Context c) { + + final CommunityDetails p = new CommunityDetails(asCommunitySummary(c)); + p.setLastUpdateDate(convertToLocalDateTime(c.getLastUpdateDate())); + final Map> params = c.getParams(); + if (params.containsKey(CPROFILE_SUBJECT)) { + p.setSubjects(splitValues(asValues(params.get(CPROFILE_SUBJECT)), CSV_DELIMITER)); + } + if (params.containsKey(CPROFILE_FOS)) { + p.setFos(splitValues(asValues(params.get(CPROFILE_FOS)), CSV_DELIMITER)); + } + if (params.containsKey(CPROFILE_SDG)) { + p.setSdg(splitValues(asValues(params.get(CPROFILE_SDG)), CSV_DELIMITER)); + } + if (params.containsKey(CPROFILE_ADVANCED_CONSTRAINT)) { + // In the map the string is the serialization of the json representing the selection criteria so it is a valid json + p.setAdvancedConstraints(SelectionCriteria.fromJson(asCsv(params.get(CPROFILE_ADVANCED_CONSTRAINT)))); + + } + if (params.containsKey(CPROFILE_CREATIONDATE)) { + try { + final Date d = org.apache.commons.lang3.time.DateUtils.parseDate(asCsv(params.get(CPROFILE_CREATIONDATE)), pattern); + p.setCreationDate(convertToLocalDateTime(d)); + } catch (final Exception e) { + log.debug("Exception on date format: " + e.getMessage()); + } + } + + return p; + } + + private static CommunityProject asCommunityProject(final String communityId, final Concept c) { + + final Map> p = c.getParams(); + final CommunityProject project = new CommunityProject(); + project.setCommunityId(communityId); + // project.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR)); + project.setOpenaireId(firstValue(p, OPENAIRE_ID)); + project.setFunder(firstValue(p, CPROJECT_FUNDER)); + project.setGrantId(firstValue(p, CPROJECT_NUMBER)); + project.setName(firstValue(p, CPROJECT_FULLNAME)); + project.setAcronym(firstValue(p, CPROJECT_ACRONYM)); + + return project; + } + + private static CommunityContentprovider asCommunityDataprovider(final String communityId, final Concept c) { + + final Map> p = c.getParams(); + final CommunityContentprovider d = new CommunityContentprovider(); + d.setCommunityId(communityId); + // d.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR)); + d.setOpenaireId(firstValue(p, OPENAIRE_ID)); + d.setName(firstValue(p, CCONTENTPROVIDER_NAME)); + d.setOfficialname(firstValue(p, CCONTENTPROVIDER_OFFICIALNAME)); + d.setSelectioncriteria(SelectionCriteria.fromJson(firstValue(p, CCONTENTPROVIDER_SELCRITERIA))); + return d; + } + + private static CommunityZenodoCommunity asCommunityZenodoCommunity(final String communityId, final Concept c) { + final CommunityZenodoCommunity z = new CommunityZenodoCommunity(); + final Map> p = c.getParams(); + z.setCommunityId(communityId); + z.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR)); + z.setZenodoid(firstValue(p, CZENODOCOMMUNITY_ID)); + // z.setName(c.getLabel()); + return z; + } + + private static CommunityOrganization asCommunityOrganization(final String id, final Concept c) { + final Map> p = c.getParams(); + final CommunityOrganization o = new CommunityOrganization(); + o.setCommunityId(id); + // o.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR)); + o.setName(firstValue(p, CORGANIZATION_NAME)); + o.setLogo_url(getDecodedUrl(firstValue(p, CORGANIZATION_LOGOURL))); + o.setWebsite_url(getDecodedUrl(firstValue(p, CORGANIZATION_WEBSITEURL))); + + return o; + } + + private static String getDecodedUrl(final String encoded_url) { + if (encoded_url == null) { return encoded_url; } + return new String(Base64.getDecoder().decode(encoded_url)); + } + + private static List splitValues(final Stream stream, final String separator) { + return stream.map(s -> s.split(separator)) + .map(Arrays::asList) + .flatMap(List::stream) + .filter(StringUtils::isNotBlank) + .map(StringUtils::trim) + .collect(Collectors.toList()); + } + + private static String firstValue(final Map> p, final String paramName) { + return asValues(p.get(paramName)).findFirst().orElse(null); + } + + private static String asCsv(final List params) { + return asValues(params) + .collect(Collectors.joining(CSV_DELIMITER)); + } + + private static Stream asValues(final List params) { + return params == null ? Stream.empty() + : params.stream() + .map(Param::getValue) + .map(StringUtils::trim) + .distinct(); + } + + private static LocalDateTime convertToLocalDateTime(final Date date) { + return date.toInstant() + .atZone(ZoneId.systemDefault()) + .toLocalDateTime(); + } + } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityMappingUtils.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityMappingUtils.java deleted file mode 100644 index 201ff5c3..00000000 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityMappingUtils.java +++ /dev/null @@ -1,259 +0,0 @@ -package eu.dnetlib.openaire.community; - -import static eu.dnetlib.openaire.common.Utils.escape; - -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.util.Arrays; -import java.util.Base64; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import com.google.common.escape.Escaper; -import com.google.common.xml.XmlEscapers; - -import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider; -import eu.dnetlib.openaire.exporter.model.community.CommunityDetails; -import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization; -import eu.dnetlib.openaire.exporter.model.community.CommunityProject; -import eu.dnetlib.openaire.exporter.model.community.CommunityStatus; -import eu.dnetlib.openaire.exporter.model.community.CommunitySummary; -import eu.dnetlib.openaire.exporter.model.community.CommunityType; -import eu.dnetlib.openaire.exporter.model.community.CommunityZenodoCommunity; -import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionCriteria; -import eu.dnetlib.openaire.exporter.model.context.Concept; -import eu.dnetlib.openaire.exporter.model.context.Context; -import eu.dnetlib.openaire.exporter.model.context.Param; - -@Deprecated -public class CommunityMappingUtils { - - private final static String pattern = "yyyy-MM-dd'T'hh:mm:ss"; - - // common - public final static String OPENAIRE_ID = "openaireId"; - public final static String PIPE_SEPARATOR = "||"; - public final static String ID_SEPARATOR = "::"; - public final static String CSV_DELIMITER = ","; - public final static String CLABEL = "label"; - - // id suffixes - public final static String PROJECTS_ID_SUFFIX = ID_SEPARATOR + "projects"; - public final static String CONTENTPROVIDERS_ID_SUFFIX = ID_SEPARATOR + "contentproviders"; - public final static String ZENODOCOMMUNITY_ID_SUFFIX = ID_SEPARATOR + "zenodocommunities"; - public final static String ORGANIZATION_ID_SUFFIX = ID_SEPARATOR + "organizations"; - - // community summary - public final static String CSUMMARY_DESCRIPTION = "description"; - public final static String CSUMMARY_LOGOURL = "logourl"; - public final static String CSUMMARY_STATUS = "status"; - public final static String CSUMMARY_NAME = "name"; - public final static String CSUMMARY_MANAGER = "manager"; - public final static String CSUMMARY_ZENODOC = "zenodoCommunity"; - - // community profile - public final static String CPROFILE_SUBJECT = "subject"; - public final static String CPROFILE_CREATIONDATE = "creationdate"; - public final static String CPROFILE_FOS = "fos"; - public final static String CPROFILE_SDG = "sdg"; - public final static String CPROFILE_ADVANCED_CONSTRAINT = "advancedConstraints"; - - // community project - public final static String CPROJECT_FUNDER = "funder"; - public final static String CPROJECT_NUMBER = "CD_PROJECT_NUMBER"; - public final static String CPROJECT_FULLNAME = "projectfullname"; - public final static String CPROJECT_ACRONYM = "acronym"; - - // community content provider - public final static String CCONTENTPROVIDER_NAME = "name"; - public final static String CCONTENTPROVIDER_OFFICIALNAME = "officialname"; - public final static String CCONTENTPROVIDER_ENABLED = "enabled"; - public final static String CCONTENTPROVIDERENABLED_DEFAULT = "true"; - public final static String CCONTENTPROVIDER_SELCRITERIA = "selcriteria"; - - // community zenodo community - public final static String CZENODOCOMMUNITY_ID = "zenodoid"; - - // community organization - public final static String CORGANIZATION_NAME = "name"; - public final static String CORGANIZATION_LOGOURL = "logourl"; - public final static String CORGANIZATION_WEBSITEURL = "websiteurl"; - - private static final Log log = LogFactory.getLog(CommunityMappingUtils.class); - - public static CommunitySummary asCommunitySummary(final Context c) { - final CommunitySummary summary = new CommunitySummary(); - - summary.setId(c.getId()); - summary.setShortName(c.getLabel()); - summary.setLastUpdateDate(convertToLocalDateTime(c.getLastUpdateDate())); - summary.setCreationDate(convertToLocalDateTime(c.getCreationDate())); - summary.setQueryId(c.getId() + PIPE_SEPARATOR + c.getLabel()); - summary.setType(CommunityType.valueOf(c.getType())); - - final Map> params = c.getParams(); - if (params.containsKey(CSUMMARY_DESCRIPTION)) { - summary.setDescription(asCsv(params.get(CSUMMARY_DESCRIPTION))); - } - if (params.containsKey(CSUMMARY_LOGOURL)) { - summary.setLogoUrl(asCsv(params.get(CSUMMARY_LOGOURL))); - } - if (params.containsKey(CSUMMARY_STATUS)) { - summary.setStatus(CommunityStatus.valueOf(firstValue(params, CSUMMARY_STATUS))); - } - if (params.containsKey(CSUMMARY_NAME)) { - summary.setName(asCsv(params.get(CSUMMARY_NAME))); - } - if (params.containsKey(CSUMMARY_ZENODOC)) { - summary.setZenodoCommunity(asCsv(params.get(CSUMMARY_ZENODOC))); - } - - return summary; - } - - public static CommunityDetails asCommunityProfile(final Context c) { - - final CommunityDetails p = new CommunityDetails(asCommunitySummary(c)); - p.setLastUpdateDate(convertToLocalDateTime(c.getLastUpdateDate())); - final Map> params = c.getParams(); - if (params.containsKey(CPROFILE_SUBJECT)) { - p.setSubjects(splitValues(asValues(params.get(CPROFILE_SUBJECT)), CSV_DELIMITER)); - } - if (params.containsKey(CPROFILE_FOS)) { - p.setFos(splitValues(asValues(params.get(CPROFILE_FOS)), CSV_DELIMITER)); - } - if (params.containsKey(CPROFILE_SDG)) { - p.setSdg(splitValues(asValues(params.get(CPROFILE_SDG)), CSV_DELIMITER)); - } - if (params.containsKey(CPROFILE_ADVANCED_CONSTRAINT)) { - // In the map the string is the serialization of the json representing the selection criteria so it is a valid json - p.setAdvancedConstraints(SelectionCriteria.fromJson(asCsv(params.get(CPROFILE_ADVANCED_CONSTRAINT)))); - - } - if (params.containsKey(CPROFILE_CREATIONDATE)) { - try { - final Date d = org.apache.commons.lang3.time.DateUtils.parseDate(asCsv(params.get(CPROFILE_CREATIONDATE)), pattern); - p.setCreationDate(convertToLocalDateTime(d)); - } catch (final Exception e) { - log.debug("Exception on date format: " + e.getMessage()); - } - } - - return p; - } - - public static CommunityProject asCommunityProject(final String communityId, final Concept c) { - - final Map> p = c.getParams(); - final CommunityProject project = new CommunityProject(); - project.setCommunityId(communityId); - // project.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR)); - project.setOpenaireId(firstValue(p, OPENAIRE_ID)); - project.setFunder(firstValue(p, CPROJECT_FUNDER)); - project.setGrantId(firstValue(p, CPROJECT_NUMBER)); - project.setName(firstValue(p, CPROJECT_FULLNAME)); - project.setAcronym(firstValue(p, CPROJECT_ACRONYM)); - - return project; - } - - public static CommunityContentprovider asCommunityDataprovider(final String communityId, final Concept c) { - - final Map> p = c.getParams(); - final CommunityContentprovider d = new CommunityContentprovider(); - d.setCommunityId(communityId); - // d.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR)); - d.setOpenaireId(firstValue(p, OPENAIRE_ID)); - d.setName(firstValue(p, CCONTENTPROVIDER_NAME)); - d.setOfficialname(firstValue(p, CCONTENTPROVIDER_OFFICIALNAME)); - d.setSelectioncriteria(SelectionCriteria.fromJson(firstValue(p, CCONTENTPROVIDER_SELCRITERIA))); - return d; - } - - public static CommunityZenodoCommunity asCommunityZenodoCommunity(final String communityId, final Concept c) { - final CommunityZenodoCommunity z = new CommunityZenodoCommunity(); - final Map> p = c.getParams(); - z.setCommunityId(communityId); - z.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR)); - z.setZenodoid(firstValue(p, CZENODOCOMMUNITY_ID)); - // z.setName(c.getLabel()); - return z; - } - - public static CommunityOrganization asCommunityOrganization(final String id, final Concept c) { - final Map> p = c.getParams(); - final CommunityOrganization o = new CommunityOrganization(); - o.setCommunityId(id); - // o.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR)); - o.setName(firstValue(p, CORGANIZATION_NAME)); - o.setLogo_url(getDecodedUrl(firstValue(p, CORGANIZATION_LOGOURL))); - o.setWebsite_url(getDecodedUrl(firstValue(p, CORGANIZATION_WEBSITEURL))); - - return o; - } - - private static String getDecodedUrl(final String encoded_url) { - if (encoded_url == null) { return encoded_url; } - return new String(Base64.getDecoder().decode(encoded_url)); - } - - private static List splitValues(final Stream stream, final String separator) { - return stream.map(s -> s.split(separator)) - .map(Arrays::asList) - .flatMap(List::stream) - .filter(StringUtils::isNotBlank) - .map(StringUtils::trim) - .collect(Collectors.toList()); - } - - private static String firstValue(final Map> p, final String paramName) { - return asValues(p.get(paramName)).findFirst().orElse(null); - } - - private static String asCsv(final List params) { - return asValues(params) - .collect(Collectors.joining(CSV_DELIMITER)); - } - - private static Stream asValues(final List params) { - return params == null ? Stream.empty() - : params.stream() - .map(Param::getValue) - .map(StringUtils::trim) - .distinct(); - } - - public static String asZenodoCommunityXML(final String contextId, final CommunityZenodoCommunity zc) { - final Escaper esc = XmlEscapers.xmlAttributeEscaper(); - final StringBuilder sb = new StringBuilder(); - sb.append(String - .format("\n", escape(esc, contextId), ZENODOCOMMUNITY_ID_SUFFIX, ID_SEPARATOR, escape(esc, String - .valueOf(zc.getId())), escape(esc, zc.getZenodoid()))); - - sb.append(paramXML(CZENODOCOMMUNITY_ID, zc.getZenodoid())); - sb.append("\n"); - return sb.toString(); - } - - private static String paramXML(final String paramName, final String value) { - return String.format("%s\n", paramName, escape(XmlEscapers.xmlContentEscaper(), value)); - } - - private static String paramXMLNoEscape(final String paramName, final String value) { - return String.format("%s\n", paramName, value); - } - - private static LocalDateTime convertToLocalDateTime(final Date date) { - return date.toInstant() - .atZone(ZoneId.systemDefault()) - .toLocalDateTime(); - } - -}