D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

cleaning

This commit is contained in:
Michele Artini 2023-06-26 15:02:07 +02:00
parent 174a0ea9f2
commit 1d199f008a
2 changed files with 220 additions and 267 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

228
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityImporterController.java
View File

@ -1,16 +1,21 @@
package eu.dnetlib.openaire.community;
import static eu.dnetlib.openaire.community.CommunityMappingUtils.CONTENTPROVIDERS_ID_SUFFIX;
import static eu.dnetlib.openaire.community.CommunityMappingUtils.ORGANIZATION_ID_SUFFIX;
import static eu.dnetlib.openaire.community.CommunityMappingUtils.PROJECTS_ID_SUFFIX;
import java.io.IOException;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.Arrays;
import java.util.Base64;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;
@ -25,13 +30,72 @@ import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider;
import eu.dnetlib.openaire.exporter.model.community.CommunityDetails;
import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization;
import eu.dnetlib.openaire.exporter.model.community.CommunityProject;
import eu.dnetlib.openaire.exporter.model.community.CommunityStatus;
import eu.dnetlib.openaire.exporter.model.community.CommunitySummary;
import eu.dnetlib.openaire.exporter.model.community.CommunityType;
import eu.dnetlib.openaire.exporter.model.community.CommunityZenodoCommunity;
import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionCriteria;
import eu.dnetlib.openaire.exporter.model.context.Category;
import eu.dnetlib.openaire.exporter.model.context.Concept;
import eu.dnetlib.openaire.exporter.model.context.Context;
import eu.dnetlib.openaire.exporter.model.context.Param;
@RestController
public class CommunityImporterController {
private final static String pattern = "yyyy-MM-dd'T'hh:mm:ss";
// common
private final static String OPENAIRE_ID = "openaireId";
private final static String PIPE_SEPARATOR = "||";
private final static String ID_SEPARATOR = "::";
private final static String CSV_DELIMITER = ",";
private final static String CLABEL = "label";
// id suffixes
private final static String PROJECTS_ID_SUFFIX = ID_SEPARATOR + "projects";
private final static String CONTENTPROVIDERS_ID_SUFFIX = ID_SEPARATOR + "contentproviders";
private final static String ZENODOCOMMUNITY_ID_SUFFIX = ID_SEPARATOR + "zenodocommunities";
private final static String ORGANIZATION_ID_SUFFIX = ID_SEPARATOR + "organizations";
// community summary
private final static String CSUMMARY_DESCRIPTION = "description";
private final static String CSUMMARY_LOGOURL = "logourl";
private final static String CSUMMARY_STATUS = "status";
private final static String CSUMMARY_NAME = "name";
private final static String CSUMMARY_MANAGER = "manager";
private final static String CSUMMARY_ZENODOC = "zenodoCommunity";
// community profile
private final static String CPROFILE_SUBJECT = "subject";
private final static String CPROFILE_CREATIONDATE = "creationdate";
private final static String CPROFILE_FOS = "fos";
private final static String CPROFILE_SDG = "sdg";
private final static String CPROFILE_ADVANCED_CONSTRAINT = "advancedConstraints";
// community project
private final static String CPROJECT_FUNDER = "funder";
private final static String CPROJECT_NUMBER = "CD_PROJECT_NUMBER";
private final static String CPROJECT_FULLNAME = "projectfullname";
private final static String CPROJECT_ACRONYM = "acronym";
// community content provider
private final static String CCONTENTPROVIDER_NAME = "name";
private final static String CCONTENTPROVIDER_OFFICIALNAME = "officialname";
private final static String CCONTENTPROVIDER_ENABLED = "enabled";
private final static String CCONTENTPROVIDERENABLED_DEFAULT = "true";
private final static String CCONTENTPROVIDER_SELCRITERIA = "selcriteria";
// community zenodo community
private final static String CZENODOCOMMUNITY_ID = "zenodoid";
// community organization
private final static String CORGANIZATION_NAME = "name";
private final static String CORGANIZATION_LOGOURL = "logourl";
private final static String CORGANIZATION_WEBSITEURL = "websiteurl";
private static final Log log = LogFactory.getLog(CommunityImporterController.class);
public final static Set<String> communityBlackList = Sets.newHashSet("fet-fp7", "fet-h2020");
@Autowired
@ -80,14 +144,14 @@ public class CommunityImporterController {
private void importCommunity(final Context context) {
try {
final CommunityDetails community = CommunityMappingUtils.asCommunityProfile(context);
final CommunityDetails community = asCommunityProfile(context);
final List<CommunityContentprovider> datasources =
getCommunityInfo(context, CONTENTPROVIDERS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityDataprovider(context.getId(), c));
getCommunityInfo(context, CONTENTPROVIDERS_ID_SUFFIX, c -> asCommunityDataprovider(context.getId(), c));
final List<CommunityProject> projects =
getCommunityInfo(context, PROJECTS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityProject(context.getId(), c));
getCommunityInfo(context, PROJECTS_ID_SUFFIX, c -> asCommunityProject(context.getId(), c));
final List<CommunityOrganization> orgs =
getCommunityInfo(context, ORGANIZATION_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityOrganization(context.getId(), c));
getCommunityInfo(context, ORGANIZATION_ID_SUFFIX, c -> asCommunityOrganization(context.getId(), c));
service.saveCommunity(community);
service.addCommunityProjectList(context.getId(), projects);
@ -99,4 +163,152 @@ public class CommunityImporterController {
}
}
private static CommunitySummary asCommunitySummary(final Context c) {
final CommunitySummary summary = new CommunitySummary();
summary.setId(c.getId());
summary.setShortName(c.getLabel());
summary.setLastUpdateDate(convertToLocalDateTime(c.getLastUpdateDate()));
summary.setCreationDate(convertToLocalDateTime(c.getCreationDate()));
summary.setQueryId(c.getId() + PIPE_SEPARATOR + c.getLabel());
summary.setType(CommunityType.valueOf(c.getType()));
final Map<String, List<Param>> params = c.getParams();
if (params.containsKey(CSUMMARY_DESCRIPTION)) {
summary.setDescription(asCsv(params.get(CSUMMARY_DESCRIPTION)));
}
if (params.containsKey(CSUMMARY_LOGOURL)) {
summary.setLogoUrl(asCsv(params.get(CSUMMARY_LOGOURL)));
}
if (params.containsKey(CSUMMARY_STATUS)) {
summary.setStatus(CommunityStatus.valueOf(firstValue(params, CSUMMARY_STATUS)));
}
if (params.containsKey(CSUMMARY_NAME)) {
summary.setName(asCsv(params.get(CSUMMARY_NAME)));
}
if (params.containsKey(CSUMMARY_ZENODOC)) {
summary.setZenodoCommunity(asCsv(params.get(CSUMMARY_ZENODOC)));
}
return summary;
}
private static CommunityDetails asCommunityProfile(final Context c) {
final CommunityDetails p = new CommunityDetails(asCommunitySummary(c));
p.setLastUpdateDate(convertToLocalDateTime(c.getLastUpdateDate()));
final Map<String, List<Param>> params = c.getParams();
if (params.containsKey(CPROFILE_SUBJECT)) {
p.setSubjects(splitValues(asValues(params.get(CPROFILE_SUBJECT)), CSV_DELIMITER));
}
if (params.containsKey(CPROFILE_FOS)) {
p.setFos(splitValues(asValues(params.get(CPROFILE_FOS)), CSV_DELIMITER));
}
if (params.containsKey(CPROFILE_SDG)) {
p.setSdg(splitValues(asValues(params.get(CPROFILE_SDG)), CSV_DELIMITER));
}
if (params.containsKey(CPROFILE_ADVANCED_CONSTRAINT)) {
// In the map the string is the serialization of the json representing the selection criteria so it is a valid json
p.setAdvancedConstraints(SelectionCriteria.fromJson(asCsv(params.get(CPROFILE_ADVANCED_CONSTRAINT))));
}
if (params.containsKey(CPROFILE_CREATIONDATE)) {
try {
final Date d = org.apache.commons.lang3.time.DateUtils.parseDate(asCsv(params.get(CPROFILE_CREATIONDATE)), pattern);
p.setCreationDate(convertToLocalDateTime(d));
} catch (final Exception e) {
log.debug("Exception on date format: " + e.getMessage());
}
}
return p;
}
private static CommunityProject asCommunityProject(final String communityId, final Concept c) {
final Map<String, List<Param>> p = c.getParams();
final CommunityProject project = new CommunityProject();
project.setCommunityId(communityId);
// project.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
project.setOpenaireId(firstValue(p, OPENAIRE_ID));
project.setFunder(firstValue(p, CPROJECT_FUNDER));
project.setGrantId(firstValue(p, CPROJECT_NUMBER));
project.setName(firstValue(p, CPROJECT_FULLNAME));
project.setAcronym(firstValue(p, CPROJECT_ACRONYM));
return project;
}
private static CommunityContentprovider asCommunityDataprovider(final String communityId, final Concept c) {
final Map<String, List<Param>> p = c.getParams();
final CommunityContentprovider d = new CommunityContentprovider();
d.setCommunityId(communityId);
// d.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
d.setOpenaireId(firstValue(p, OPENAIRE_ID));
d.setName(firstValue(p, CCONTENTPROVIDER_NAME));
d.setOfficialname(firstValue(p, CCONTENTPROVIDER_OFFICIALNAME));
d.setSelectioncriteria(SelectionCriteria.fromJson(firstValue(p, CCONTENTPROVIDER_SELCRITERIA)));
return d;
}
private static CommunityZenodoCommunity asCommunityZenodoCommunity(final String communityId, final Concept c) {
final CommunityZenodoCommunity z = new CommunityZenodoCommunity();
final Map<String, List<Param>> p = c.getParams();
z.setCommunityId(communityId);
z.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
z.setZenodoid(firstValue(p, CZENODOCOMMUNITY_ID));
// z.setName(c.getLabel());
return z;
}
private static CommunityOrganization asCommunityOrganization(final String id, final Concept c) {
final Map<String, List<Param>> p = c.getParams();
final CommunityOrganization o = new CommunityOrganization();
o.setCommunityId(id);
// o.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
o.setName(firstValue(p, CORGANIZATION_NAME));
o.setLogo_url(getDecodedUrl(firstValue(p, CORGANIZATION_LOGOURL)));
o.setWebsite_url(getDecodedUrl(firstValue(p, CORGANIZATION_WEBSITEURL)));
return o;
}
private static String getDecodedUrl(final String encoded_url) {
if (encoded_url == null) { return encoded_url; }
return new String(Base64.getDecoder().decode(encoded_url));
}
private static List<String> splitValues(final Stream<String> stream, final String separator) {
return stream.map(s -> s.split(separator))
.map(Arrays::asList)
.flatMap(List::stream)
.filter(StringUtils::isNotBlank)
.map(StringUtils::trim)
.collect(Collectors.toList());
}
private static String firstValue(final Map<String, List<Param>> p, final String paramName) {
return asValues(p.get(paramName)).findFirst().orElse(null);
}
private static String asCsv(final List<Param> params) {
return asValues(params)
.collect(Collectors.joining(CSV_DELIMITER));
}
private static Stream<String> asValues(final List<Param> params) {
return params == null ? Stream.empty()
: params.stream()
.map(Param::getValue)
.map(StringUtils::trim)
.distinct();
}
private static LocalDateTime convertToLocalDateTime(final Date date) {
return date.toInstant()
.atZone(ZoneId.systemDefault())
.toLocalDateTime();
}
}

259
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityMappingUtils.java
View File

@ -1,259 +0,0 @@
package eu.dnetlib.openaire.community;
import static eu.dnetlib.openaire.common.Utils.escape;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.Arrays;
import java.util.Base64;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.google.common.escape.Escaper;
import com.google.common.xml.XmlEscapers;
import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider;
import eu.dnetlib.openaire.exporter.model.community.CommunityDetails;
import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization;
import eu.dnetlib.openaire.exporter.model.community.CommunityProject;
import eu.dnetlib.openaire.exporter.model.community.CommunityStatus;
import eu.dnetlib.openaire.exporter.model.community.CommunitySummary;
import eu.dnetlib.openaire.exporter.model.community.CommunityType;
import eu.dnetlib.openaire.exporter.model.community.CommunityZenodoCommunity;
import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionCriteria;
import eu.dnetlib.openaire.exporter.model.context.Concept;
import eu.dnetlib.openaire.exporter.model.context.Context;
import eu.dnetlib.openaire.exporter.model.context.Param;
@Deprecated
public class CommunityMappingUtils {
private final static String pattern = "yyyy-MM-dd'T'hh:mm:ss";
// common
public final static String OPENAIRE_ID = "openaireId";
public final static String PIPE_SEPARATOR = "||";
public final static String ID_SEPARATOR = "::";
public final static String CSV_DELIMITER = ",";
public final static String CLABEL = "label";
// id suffixes
public final static String PROJECTS_ID_SUFFIX = ID_SEPARATOR + "projects";
public final static String CONTENTPROVIDERS_ID_SUFFIX = ID_SEPARATOR + "contentproviders";
public final static String ZENODOCOMMUNITY_ID_SUFFIX = ID_SEPARATOR + "zenodocommunities";
public final static String ORGANIZATION_ID_SUFFIX = ID_SEPARATOR + "organizations";
// community summary
public final static String CSUMMARY_DESCRIPTION = "description";
public final static String CSUMMARY_LOGOURL = "logourl";
public final static String CSUMMARY_STATUS = "status";
public final static String CSUMMARY_NAME = "name";
public final static String CSUMMARY_MANAGER = "manager";
public final static String CSUMMARY_ZENODOC = "zenodoCommunity";
// community profile
public final static String CPROFILE_SUBJECT = "subject";
public final static String CPROFILE_CREATIONDATE = "creationdate";
public final static String CPROFILE_FOS = "fos";
public final static String CPROFILE_SDG = "sdg";
public final static String CPROFILE_ADVANCED_CONSTRAINT = "advancedConstraints";
// community project
public final static String CPROJECT_FUNDER = "funder";
public final static String CPROJECT_NUMBER = "CD_PROJECT_NUMBER";
public final static String CPROJECT_FULLNAME = "projectfullname";
public final static String CPROJECT_ACRONYM = "acronym";
// community content provider
public final static String CCONTENTPROVIDER_NAME = "name";
public final static String CCONTENTPROVIDER_OFFICIALNAME = "officialname";
public final static String CCONTENTPROVIDER_ENABLED = "enabled";
public final static String CCONTENTPROVIDERENABLED_DEFAULT = "true";
public final static String CCONTENTPROVIDER_SELCRITERIA = "selcriteria";
// community zenodo community
public final static String CZENODOCOMMUNITY_ID = "zenodoid";
// community organization
public final static String CORGANIZATION_NAME = "name";
public final static String CORGANIZATION_LOGOURL = "logourl";
public final static String CORGANIZATION_WEBSITEURL = "websiteurl";
private static final Log log = LogFactory.getLog(CommunityMappingUtils.class);
public static CommunitySummary asCommunitySummary(final Context c) {
final CommunitySummary summary = new CommunitySummary();
summary.setId(c.getId());
summary.setShortName(c.getLabel());
summary.setLastUpdateDate(convertToLocalDateTime(c.getLastUpdateDate()));
summary.setCreationDate(convertToLocalDateTime(c.getCreationDate()));
summary.setQueryId(c.getId() + PIPE_SEPARATOR + c.getLabel());
summary.setType(CommunityType.valueOf(c.getType()));
final Map<String, List<Param>> params = c.getParams();
if (params.containsKey(CSUMMARY_DESCRIPTION)) {
summary.setDescription(asCsv(params.get(CSUMMARY_DESCRIPTION)));
}
if (params.containsKey(CSUMMARY_LOGOURL)) {
summary.setLogoUrl(asCsv(params.get(CSUMMARY_LOGOURL)));
}
if (params.containsKey(CSUMMARY_STATUS)) {
summary.setStatus(CommunityStatus.valueOf(firstValue(params, CSUMMARY_STATUS)));
}
if (params.containsKey(CSUMMARY_NAME)) {
summary.setName(asCsv(params.get(CSUMMARY_NAME)));
}
if (params.containsKey(CSUMMARY_ZENODOC)) {
summary.setZenodoCommunity(asCsv(params.get(CSUMMARY_ZENODOC)));
}
return summary;
}
public static CommunityDetails asCommunityProfile(final Context c) {
final CommunityDetails p = new CommunityDetails(asCommunitySummary(c));
p.setLastUpdateDate(convertToLocalDateTime(c.getLastUpdateDate()));
final Map<String, List<Param>> params = c.getParams();
if (params.containsKey(CPROFILE_SUBJECT)) {
p.setSubjects(splitValues(asValues(params.get(CPROFILE_SUBJECT)), CSV_DELIMITER));
}
if (params.containsKey(CPROFILE_FOS)) {
p.setFos(splitValues(asValues(params.get(CPROFILE_FOS)), CSV_DELIMITER));
}
if (params.containsKey(CPROFILE_SDG)) {
p.setSdg(splitValues(asValues(params.get(CPROFILE_SDG)), CSV_DELIMITER));
}
if (params.containsKey(CPROFILE_ADVANCED_CONSTRAINT)) {
// In the map the string is the serialization of the json representing the selection criteria so it is a valid json
p.setAdvancedConstraints(SelectionCriteria.fromJson(asCsv(params.get(CPROFILE_ADVANCED_CONSTRAINT))));
}
if (params.containsKey(CPROFILE_CREATIONDATE)) {
try {
final Date d = org.apache.commons.lang3.time.DateUtils.parseDate(asCsv(params.get(CPROFILE_CREATIONDATE)), pattern);
p.setCreationDate(convertToLocalDateTime(d));
} catch (final Exception e) {
log.debug("Exception on date format: " + e.getMessage());
}
}
return p;
}
public static CommunityProject asCommunityProject(final String communityId, final Concept c) {
final Map<String, List<Param>> p = c.getParams();
final CommunityProject project = new CommunityProject();
project.setCommunityId(communityId);
// project.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
project.setOpenaireId(firstValue(p, OPENAIRE_ID));
project.setFunder(firstValue(p, CPROJECT_FUNDER));
project.setGrantId(firstValue(p, CPROJECT_NUMBER));
project.setName(firstValue(p, CPROJECT_FULLNAME));
project.setAcronym(firstValue(p, CPROJECT_ACRONYM));
return project;
}
public static CommunityContentprovider asCommunityDataprovider(final String communityId, final Concept c) {
final Map<String, List<Param>> p = c.getParams();
final CommunityContentprovider d = new CommunityContentprovider();
d.setCommunityId(communityId);
// d.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
d.setOpenaireId(firstValue(p, OPENAIRE_ID));
d.setName(firstValue(p, CCONTENTPROVIDER_NAME));
d.setOfficialname(firstValue(p, CCONTENTPROVIDER_OFFICIALNAME));
d.setSelectioncriteria(SelectionCriteria.fromJson(firstValue(p, CCONTENTPROVIDER_SELCRITERIA)));
return d;
}
public static CommunityZenodoCommunity asCommunityZenodoCommunity(final String communityId, final Concept c) {
final CommunityZenodoCommunity z = new CommunityZenodoCommunity();
final Map<String, List<Param>> p = c.getParams();
z.setCommunityId(communityId);
z.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
z.setZenodoid(firstValue(p, CZENODOCOMMUNITY_ID));
// z.setName(c.getLabel());
return z;
}
public static CommunityOrganization asCommunityOrganization(final String id, final Concept c) {
final Map<String, List<Param>> p = c.getParams();
final CommunityOrganization o = new CommunityOrganization();
o.setCommunityId(id);
// o.setId(StringUtils.substringAfterLast(c.getId(), ID_SEPARATOR));
o.setName(firstValue(p, CORGANIZATION_NAME));
o.setLogo_url(getDecodedUrl(firstValue(p, CORGANIZATION_LOGOURL)));
o.setWebsite_url(getDecodedUrl(firstValue(p, CORGANIZATION_WEBSITEURL)));
return o;
}
private static String getDecodedUrl(final String encoded_url) {
if (encoded_url == null) { return encoded_url; }
return new String(Base64.getDecoder().decode(encoded_url));
}
private static List<String> splitValues(final Stream<String> stream, final String separator) {
return stream.map(s -> s.split(separator))
.map(Arrays::asList)
.flatMap(List::stream)
.filter(StringUtils::isNotBlank)
.map(StringUtils::trim)
.collect(Collectors.toList());
}
private static String firstValue(final Map<String, List<Param>> p, final String paramName) {
return asValues(p.get(paramName)).findFirst().orElse(null);
}
private static String asCsv(final List<Param> params) {
return asValues(params)
.collect(Collectors.joining(CSV_DELIMITER));
}
private static Stream<String> asValues(final List<Param> params) {
return params == null ? Stream.empty()
: params.stream()
.map(Param::getValue)
.map(StringUtils::trim)
.distinct();
}
public static String asZenodoCommunityXML(final String contextId, final CommunityZenodoCommunity zc) {
final Escaper esc = XmlEscapers.xmlAttributeEscaper();
final StringBuilder sb = new StringBuilder();
sb.append(String
.format("<concept claim='false' id='%s%s%s%s' label='%s'>\n", escape(esc, contextId), ZENODOCOMMUNITY_ID_SUFFIX, ID_SEPARATOR, escape(esc, String
.valueOf(zc.getId())), escape(esc, zc.getZenodoid())));
sb.append(paramXML(CZENODOCOMMUNITY_ID, zc.getZenodoid()));
sb.append("</concept>\n");
return sb.toString();
}
private static String paramXML(final String paramName, final String value) {
return String.format("<param name='%s'>%s</param>\n", paramName, escape(XmlEscapers.xmlContentEscaper(), value));
}
private static String paramXMLNoEscape(final String paramName, final String value) {
return String.format("<param name='%s'>%s</param>\n", paramName, value);
}
private static LocalDateTime convertToLocalDateTime(final Date date) {
return date.toInstant()
.atZone(ZoneId.systemDefault())
.toLocalDateTime();
}
}