package eu.dnetlib.openaire.community; import java.io.IOException; import java.time.LocalDateTime; import java.time.ZoneId; import java.util.Arrays; import java.util.Base64; import java.util.Date; import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RestController; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import eu.dnetlib.openaire.common.ISClient; import eu.dnetlib.openaire.community.db.CommunityService; import eu.dnetlib.openaire.exporter.exceptions.CommunityException; import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider; import eu.dnetlib.openaire.exporter.model.community.CommunityDetails; import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization; import eu.dnetlib.openaire.exporter.model.community.CommunityProject; import eu.dnetlib.openaire.exporter.model.community.CommunityStatus; import eu.dnetlib.openaire.exporter.model.community.CommunitySummary; import eu.dnetlib.openaire.exporter.model.community.CommunityType; import eu.dnetlib.openaire.exporter.model.community.CommunityZenodoCommunity; import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionCriteria; import eu.dnetlib.openaire.exporter.model.context.Category; import eu.dnetlib.openaire.exporter.model.context.Concept; import eu.dnetlib.openaire.exporter.model.context.Context; import eu.dnetlib.openaire.exporter.model.context.Param; @RestController public class CommunityImporterController { private final static String pattern = "yyyy-MM-dd'T'hh:mm:ss"; // common private final static String OPENAIRE_ID = "openaireId"; private final static String PIPE_SEPARATOR = "||"; private final static String ID_SEPARATOR = "::"; private final static String CSV_DELIMITER = ","; private final static String CLABEL = "label"; // id suffixes private final static String PROJECTS_ID_SUFFIX = ID_SEPARATOR + "projects"; private final static String CONTENTPROVIDERS_ID_SUFFIX = ID_SEPARATOR + "contentproviders"; private final static String ZENODOCOMMUNITY_ID_SUFFIX = ID_SEPARATOR + "zenodocommunities"; private final static String ORGANIZATION_ID_SUFFIX = ID_SEPARATOR + "organizations"; // community summary private final static String CSUMMARY_DESCRIPTION = "description"; private final static String CSUMMARY_LOGOURL = "logourl"; private final static String CSUMMARY_STATUS = "status"; private final static String CSUMMARY_NAME = "name"; private final static String CSUMMARY_MANAGER = "manager"; private final static String CSUMMARY_ZENODOC = "zenodoCommunity"; // community profile private final static String CPROFILE_SUBJECT = "subject"; private final static String CPROFILE_CREATIONDATE = "creationdate"; private final static String CPROFILE_FOS = "fos"; private final static String CPROFILE_SDG = "sdg"; private final static String CPROFILE_ADVANCED_CONSTRAINT = "advancedConstraints"; // community project private final static String CPROJECT_FUNDER = "funder"; private final static String CPROJECT_NUMBER = "CD_PROJECT_NUMBER"; private final static String CPROJECT_FULLNAME = "projectfullname"; private final static String CPROJECT_ACRONYM = "acronym"; // community content provider private final static String CCONTENTPROVIDER_NAME = "name"; private final static String CCONTENTPROVIDER_OFFICIALNAME = "officialname"; private final static String CCONTENTPROVIDER_ENABLED = "enabled"; private final static String CCONTENTPROVIDERENABLED_DEFAULT = "true"; private final static String CCONTENTPROVIDER_SELCRITERIA = "selcriteria"; // community zenodo community private final static String CZENODOCOMMUNITY_ID = "zenodoid"; // community organization private final static String CORGANIZATION_NAME = "name"; private final static String CORGANIZATION_LOGOURL = "logourl"; private final static String CORGANIZATION_WEBSITEURL = "websiteurl"; private static final Log log = LogFactory.getLog(CommunityImporterController.class); public final static Set communityBlackList = Sets.newHashSet("fet-fp7", "fet-h2020"); @Autowired private CommunityService service; @Autowired private ISClient isClient; @GetMapping("/community_importer/import") public List importProfiles() throws CommunityException { final Map contextMap = getContextMap(); final List list = contextMap.keySet() .stream() .filter(id -> !communityBlackList.contains(id)) .collect(Collectors.toList()); list.forEach(id -> { importCommunity(contextMap.get(id)); }); return list; } private Map getContextMap() throws CommunityException { try { return isClient.getCommunityContextMap(); } catch (final IOException e) { throw new CommunityException(e); } } public List getCommunityInfo(final Context context, final String idSuffix, final Function mapping) throws CommunityException { if (context != null) { final Map categories = context.getCategories(); final Category category = categories.get(context.getId() + idSuffix); if (category != null) { return category.getConcepts() .stream() .map(mapping) .collect(Collectors.toList()); } } return Lists.newArrayList(); } private void importCommunity(final Context context) { try { final CommunityDetails community = asCommunityProfile(context); final List datasources = getCommunityInfo(context, CONTENTPROVIDERS_ID_SUFFIX, c -> asCommunityDataprovider(context.getId(), c)); final List projects = getCommunityInfo(context, PROJECTS_ID_SUFFIX, c -> asCommunityProject(context.getId(), c)); final List orgs = getCommunityInfo(context, ORGANIZATION_ID_SUFFIX, c -> asCommunityOrganization(context.getId(), c)); service.saveCommunity(community); service.addCommunityProjectList(context.getId(), projects); service.addCommunityContentProvidersList(context.getId(), datasources); service.addCommunityOrganizationList(context.getId(), orgs); // TODO MANAGE new fields and tables } catch (final Exception e) { throw new RuntimeException("Error importing community: " + context.getId(), e); } } private static CommunitySummary asCommunitySummary(final Context c) { final CommunitySummary summary = new CommunitySummary(); summary.setId(c.getId()); summary.setShortName(c.getLabel()); summary.setLastUpdateDate(convertToLocalDateTime(c.getLastUpdateDate())); summary.setCreationDate(convertToLocalDateTime(c.getCreationDate())); summary.setQueryId(c.getId() + PIPE_SEPARATOR + c.getLabel()); summary.setType(CommunityType.valueOf(c.getType())); final Map> params = c.getParams(); if (params.containsKey(CSUMMARY_DESCRIPTION)) { summary.setDescription(asCsv(params.get(CSUMMARY_DESCRIPTION))); } if (params.containsKey(CSUMMARY_LOGOURL)) { summary.setLogoUrl(asCsv(params.get(CSUMMARY_LOGOURL))); } if (params.containsKey(CSUMMARY_STATUS)) { summary.setStatus(CommunityStatus.valueOf(firstValue(params, CSUMMARY_STATUS))); } if (params.containsKey(CSUMMARY_NAME)) { summary.setName(asCsv(params.get(CSUMMARY_NAME))); } if (params.containsKey(CSUMMARY_ZENODOC)) { summary.setZenodoCommunity(asCsv(params.get(CSUMMARY_ZENODOC))); } return summary; } private static CommunityDetails asCommunityProfile(final Context c) { final CommunityDetails p = new CommunityDetails(asCommunitySummary(c)); p.setLastUpdateDate(convertToLocalDateTime(c.getLastUpdateDate())); final Map> params = c.getParams(); if (params.containsKey(CPROFILE_SUBJECT)) { p.setSubjects(splitValues(asValues(params.get(CPROFILE_SUBJECT)), CSV_DELIMITER)); } if (params.containsKey(CPROFILE_FOS)) { p.setFos(splitValues(asValues(params.get(CPROFILE_FOS)), CSV_DELIMITER)); } if (params.containsKey(CPROFILE_SDG)) { p.setSdg(splitValues(asValues(params.get(CPROFILE_SDG)), CSV_DELIMITER)); } if (params.containsKey(CPROFILE_ADVANCED_CONSTRAINT)) { // In the map the string is the serialization of the json representing the selection criteria so it is a valid json p.setAdvancedConstraints(SelectionCriteria.fromJson(asCsv(params.get(CPROFILE_ADVANCED_CONSTRAINT)))); } if (params.containsKey(CPROFILE_CREATIONDATE)) { try { final Date d = org.apache.commons.lang3.time.DateUtils.parseDate(asCsv(params.get(CPROFILE_CREATIONDATE)), pattern); p.setCreationDate(convertToLocalDateTime(d)); } catch (final Exception e) { log.debug("Exception on date format: " + e.getMessage()); } } return p; } private static CommunityProject asCommunityProject(final String communityId, final Concept c) { final Map> p = c.getParams(); final CommunityProject project = new CommunityProject(); project.setCommunityId(communityId); // project.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR)); project.setOpenaireId(firstValue(p, OPENAIRE_ID)); project.setFunder(firstValue(p, CPROJECT_FUNDER)); project.setGrantId(firstValue(p, CPROJECT_NUMBER)); project.setName(firstValue(p, CPROJECT_FULLNAME)); project.setAcronym(firstValue(p, CPROJECT_ACRONYM)); return project; } private static CommunityContentprovider asCommunityDataprovider(final String communityId, final Concept c) { final Map> p = c.getParams(); final CommunityContentprovider d = new CommunityContentprovider(); d.setCommunityId(communityId); // d.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR)); d.setOpenaireId(firstValue(p, OPENAIRE_ID)); d.setName(firstValue(p, CCONTENTPROVIDER_NAME)); d.setOfficialname(firstValue(p, CCONTENTPROVIDER_OFFICIALNAME)); d.setSelectioncriteria(SelectionCriteria.fromJson(firstValue(p, CCONTENTPROVIDER_SELCRITERIA))); return d; } private static CommunityZenodoCommunity asCommunityZenodoCommunity(final String communityId, final Concept c) { final CommunityZenodoCommunity z = new CommunityZenodoCommunity(); final Map> p = c.getParams(); z.setCommunityId(communityId); z.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR)); z.setZenodoid(firstValue(p, CZENODOCOMMUNITY_ID)); // z.setName(c.getLabel()); return z; } private static CommunityOrganization asCommunityOrganization(final String id, final Concept c) { final Map> p = c.getParams(); final CommunityOrganization o = new CommunityOrganization(); o.setCommunityId(id); // o.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR)); o.setName(firstValue(p, CORGANIZATION_NAME)); o.setLogo_url(getDecodedUrl(firstValue(p, CORGANIZATION_LOGOURL))); o.setWebsite_url(getDecodedUrl(firstValue(p, CORGANIZATION_WEBSITEURL))); return o; } private static String getDecodedUrl(final String encoded_url) { if (encoded_url == null) { return encoded_url; } return new String(Base64.getDecoder().decode(encoded_url)); } private static List splitValues(final Stream stream, final String separator) { return stream.map(s -> s.split(separator)) .map(Arrays::asList) .flatMap(List::stream) .filter(StringUtils::isNotBlank) .map(StringUtils::trim) .collect(Collectors.toList()); } private static String firstValue(final Map> p, final String paramName) { return asValues(p.get(paramName)).findFirst().orElse(null); } private static String asCsv(final List params) { return asValues(params) .collect(Collectors.joining(CSV_DELIMITER)); } private static Stream asValues(final List params) { return params == null ? Stream.empty() : params.stream() .map(Param::getValue) .map(StringUtils::trim) .distinct(); } private static LocalDateTime convertToLocalDateTime(final Date date) { return date.toInstant() .atZone(ZoneId.systemDefault()) .toLocalDateTime(); } }