D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

cleaning

This commit is contained in:
Michele Artini 2023-06-26 14:56:40 +02:00
parent 71ba41a94c
commit 174a0ea9f2
3 changed files with 113 additions and 341 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

293
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityCommon.java
View File

@ -1,293 +0,0 @@
package eu.dnetlib.openaire.community;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.gson.Gson;
import eu.dnetlib.openaire.common.ISClient;
import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
import eu.dnetlib.openaire.exporter.exceptions.ResourceNotFoundException;
import eu.dnetlib.openaire.exporter.model.community.CommunityDetails;
import eu.dnetlib.openaire.exporter.model.community.CommunitySummary;
import eu.dnetlib.openaire.exporter.model.community.CommunityWritableProperties;
import eu.dnetlib.openaire.exporter.model.community.CommunityZenodoCommunity;
import eu.dnetlib.openaire.exporter.model.context.Category;
import eu.dnetlib.openaire.exporter.model.context.Concept;
import eu.dnetlib.openaire.exporter.model.context.Context;
import eu.dnetlib.openaire.exporter.model.context.Param;
@Component
@Deprecated
public class CommunityCommon {
public final static Set<String> communityBlackList = Sets.newHashSet("fet-fp7", "fet-h2020");
// common
public final static String OPENAIRE_ID = "openaireId";
public final static String PIPE_SEPARATOR = "||";
public final static String ID_SEPARATOR = "::";
public final static String CSV_DELIMITER = ",";
public final static String CLABEL = "label";
// id suffixes
public final static String PROJECTS_ID_SUFFIX = ID_SEPARATOR + "projects";
public final static String CONTENTPROVIDERS_ID_SUFFIX = ID_SEPARATOR + "contentproviders";
public final static String ZENODOCOMMUNITY_ID_SUFFIX = ID_SEPARATOR + "zenodocommunities";
public final static String ORGANIZATION_ID_SUFFIX = ID_SEPARATOR + "organizations";
// community summary
public final static String CSUMMARY_DESCRIPTION = "description";
public final static String CSUMMARY_LOGOURL = "logourl";
public final static String CSUMMARY_STATUS = "status";
public final static String CSUMMARY_NAME = "name";
public final static String CSUMMARY_MANAGER = "manager";
public final static String CSUMMARY_ZENODOC = "zenodoCommunity";
// community profile
public final static String CPROFILE_SUBJECT = "subject";
public final static String CPROFILE_CREATIONDATE = "creationdate";
public final static String CPROFILE_FOS = "fos";
public final static String CPROFILE_SDG = "sdg";
public final static String CPROFILE_ADVANCED_CONSTRAINT = "advancedConstraints";
// community project
public final static String CPROJECT_FUNDER = "funder";
public final static String CPROJECT_NUMBER = "CD_PROJECT_NUMBER";
public final static String CPROJECT_FULLNAME = "projectfullname";
public final static String CPROJECT_ACRONYM = "acronym";
// community content provider
public final static String CCONTENTPROVIDER_NAME = "name";
public final static String CCONTENTPROVIDER_OFFICIALNAME = "officialname";
public final static String CCONTENTPROVIDER_ENABLED = "enabled";
public final static String CCONTENTPROVIDERENABLED_DEFAULT = "true";
public final static String CCONTENTPROVIDER_SELCRITERIA = "selcriteria";
// community zenodo community
public final static String CZENODOCOMMUNITY_ID = "zenodoid";
// community organization
public final static String CORGANIZATION_NAME = "name";
public final static String CORGANIZATION_LOGOURL = "logourl";
public final static String CORGANIZATION_WEBSITEURL = "websiteurl";
@Autowired
private ISClient isClient;
public Map<String, Context> getContextMap() throws CommunityException {
try {
return isClient.getCommunityContextMap();
} catch (final IOException e) {
throw new CommunityException(e);
}
}
public List<CommunitySummary> listCommunities() throws CommunityException {
return getContextMap().values()
.stream()
.filter(context -> !communityBlackList.contains(context.getId()))
.map(CommunityMappingUtils::asCommunitySummary)
.collect(Collectors.toList());
}
public <R> List<R> getCommunityInfo(final String id, final String idSuffix, final Function<Concept, R> mapping) throws CommunityException {
final Map<String, Context> contextMap = getContextMap();
final Context context = contextMap.get(id);
if (context != null) {
final Map<String, Category> categories = context.getCategories();
final Category category = categories.get(id + idSuffix);
if (category != null) { return category.getConcepts()
.stream()
.map(mapping)
.collect(Collectors.toList()); }
}
return Lists.newArrayList();
}
public CommunityDetails getCommunity(final String id) throws CommunityException, ResourceNotFoundException {
final Context context = getContextMap().get(id);
if (context == null || communityBlackList.contains(id)) {
// ResponseStatusException(NOT_FOUND, "Unable to find resource");
throw new ResourceNotFoundException();
}
return CommunityMappingUtils.asCommunityProfile(context);
}
public List<CommunityZenodoCommunity> getCommunityZenodoCommunities(final String id) throws CommunityException, ResourceNotFoundException {
getCommunity(id); // ensure the community exists.
return getCommunityInfo(id, ZENODOCOMMUNITY_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityZenodoCommunity(id, c));
}
public void removeAdvancedConstraint(final String id) throws CommunityException {
final Context context = getContextMap().get(id);
context.getParams()
.replace(CPROFILE_ADVANCED_CONSTRAINT, Arrays.asList(new Param()
.setName(CPROFILE_ADVANCED_CONSTRAINT)
.setValue(null)));
}
public void updateCommunity(final String id, final CommunityWritableProperties community) throws CommunityException {
final Context context = getContextMap().get(id);
if (community.getShortName() != null) {
context.setLabel(community.getShortName());
}
if (community.getName() != null) {
context.getParams()
.replace(CSUMMARY_NAME, Arrays.asList(new Param()
.setValue(community.getName())
.setName(CSUMMARY_NAME)));
}
if (community.getDescription() != null) {
context.getParams()
.replace(CSUMMARY_DESCRIPTION, Arrays.asList(new Param()
.setName(CSUMMARY_DESCRIPTION)
.setValue(community.getDescription())));
}
if (community.getLogoUrl() != null) {
context.getParams()
.replace(CSUMMARY_LOGOURL, Arrays.asList(new Param()
.setName(CSUMMARY_LOGOURL)
.setValue(community.getLogoUrl())));
}
if (community.getStatus() != null) {
context.getParams()
.replace(CSUMMARY_STATUS, Arrays.asList(new Param()
.setName(CSUMMARY_STATUS)
.setValue(community.getStatus().name())));
}
if (community.getSubjects() != null) {
context.getParams()
.replace(CPROFILE_SUBJECT, Arrays.asList(new Param().setName(CPROFILE_SUBJECT)
.setValue(Joiner.on(CSV_DELIMITER)
.join(community.getSubjects()))));
}
if (community.getFos() != null) {
if (context.getParams().containsKey(CPROFILE_FOS)) {
context.getParams()
.replace(CPROFILE_FOS, Arrays.asList(new Param().setName(CPROFILE_FOS)
.setValue(Joiner.on(CSV_DELIMITER)
.join(community.getFos()))));
} else {
context.getParams()
.put(CPROFILE_FOS, Arrays.asList(new Param().setName(CPROFILE_FOS)
.setValue(Joiner.on(CSV_DELIMITER)
.join(community.getFos()))));
}
}
if (community.getSdg() != null) {
if (context.getParams().containsKey(CPROFILE_SDG)) {
context.getParams()
.replace(CPROFILE_SDG, Arrays.asList(new Param().setName(CPROFILE_SDG)
.setValue(Joiner.on(CSV_DELIMITER)
.join(community.getSdg()))));
} else {
context.getParams()
.put(CPROFILE_SDG, Arrays.asList(new Param().setName(CPROFILE_SDG)
.setValue(Joiner.on(CSV_DELIMITER)
.join(community.getSdg()))));
}
}
if (community.getAdvancedConstraints() != null) {
if (context.getParams().containsKey(CPROFILE_ADVANCED_CONSTRAINT)) {
context.getParams()
.replace(CPROFILE_ADVANCED_CONSTRAINT, Arrays.asList(new Param()
.setName(CPROFILE_ADVANCED_CONSTRAINT)
.setValue(new Gson().toJson(community.getAdvancedConstraints()))));
} else {
context.getParams()
.put(CPROFILE_ADVANCED_CONSTRAINT, Arrays.asList(new Param()
.setName(CPROFILE_ADVANCED_CONSTRAINT)
.setValue(new Gson().toJson(community.getAdvancedConstraints()))));
}
}
if (community.getMainZenodoCommunity() != null) {
context.getParams()
.replace(CSUMMARY_ZENODOC, Arrays.asList(new Param()
.setName(CSUMMARY_ZENODOC)
.setValue(community.getMainZenodoCommunity())));
}
}
public void removeFromCategory(final String communityId, final String category, final String conceptId) throws CommunityException {
final Map<String, Context> cmap = getContextMap();
final Context context = cmap.get(communityId);
final Map<String, Category> cat = context.getCategories();
final List<Concept> concepts = cat.get(communityId + category)
.getConcepts()
.stream()
.filter(c -> !c.getId().equals(communityId + category + ID_SEPARATOR + conceptId))
.collect(Collectors.toList());
cat.get(communityId + category).setConcepts(concepts);
}
public void updateZenodoCommunity(final String communityId, final CommunityZenodoCommunity zc) throws CommunityException {
final Context context = getContextMap().get(communityId);
final Category zcs = context.getCategories().get(communityId + ZENODOCOMMUNITY_ID_SUFFIX);
if (zcs.getConcepts()
.stream()
.map(c -> c.getId())
.collect(Collectors.toList())
.contains(communityId + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zc.getId())) {
zcs.getConcepts().forEach(concept -> {
if (concept.getId().equals(communityId + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zc.getId())) {
if (zc.getZenodoid() != null) {
if (concept.getParams().keySet().contains(CZENODOCOMMUNITY_ID)) {
concept.getParams()
.replace(CZENODOCOMMUNITY_ID, Arrays.asList(new Param()
.setName(CZENODOCOMMUNITY_ID)
.setValue(zc.getZenodoid())));
} else {
concept.getParams()
.put(CZENODOCOMMUNITY_ID, Arrays.asList(new Param()
.setName(CZENODOCOMMUNITY_ID)
.setValue(zc.getZenodoid())));
}
}
}
});
} else {
final Concept concept = new Concept();
concept.setId(communityId + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zc.getId());
concept.setClaim(false);
final Map<String, List<Param>> params = new TreeMap<>();
if (zc.getZenodoid() != null) {
params.put(CZENODOCOMMUNITY_ID, Arrays.asList(new Param().setValue(zc.getZenodoid()).setName(CZENODOCOMMUNITY_ID)));
concept.setLabel(zc.getZenodoid());
} else {
concept.setLabel("");
}
concept.setParams(params);
zcs.getConcepts().add(concept);
}
}
}

86
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityImporterController.java
View File

@ -1,60 +1,102 @@
package eu.dnetlib.openaire.community;
import static eu.dnetlib.openaire.community.CommunityCommon.CONTENTPROVIDERS_ID_SUFFIX;
import static eu.dnetlib.openaire.community.CommunityCommon.ORGANIZATION_ID_SUFFIX;
import static eu.dnetlib.openaire.community.CommunityCommon.PROJECTS_ID_SUFFIX;
import static eu.dnetlib.openaire.community.CommunityMappingUtils.CONTENTPROVIDERS_ID_SUFFIX;
import static eu.dnetlib.openaire.community.CommunityMappingUtils.ORGANIZATION_ID_SUFFIX;
import static eu.dnetlib.openaire.community.CommunityMappingUtils.PROJECTS_ID_SUFFIX;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import eu.dnetlib.openaire.common.ISClient;
import eu.dnetlib.openaire.community.db.CommunityService;
import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider;
import eu.dnetlib.openaire.exporter.model.community.CommunityDetails;
import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization;
import eu.dnetlib.openaire.exporter.model.community.CommunityProject;
import eu.dnetlib.openaire.exporter.model.context.Category;
import eu.dnetlib.openaire.exporter.model.context.Concept;
import eu.dnetlib.openaire.exporter.model.context.Context;
@RestController
public class CommunityImporterController {
public final static Set<String> communityBlackList = Sets.newHashSet("fet-fp7", "fet-h2020");
@Autowired
private CommunityService service;
@Autowired
private CommunityCommon cc;
private ISClient isClient;
@GetMapping("/community_importer/import")
public List<String> importProfiles() throws CommunityException {
final List<String> list = cc.listCommunities()
final Map<String, Context> contextMap = getContextMap();
final List<String> list = contextMap.keySet()
.stream()
.map(c -> c.getId())
.filter(id -> !communityBlackList.contains(id))
.collect(Collectors.toList());
list.forEach(this::importCommunity);
list.forEach(id -> {
importCommunity(contextMap.get(id));
});
return list;
}
private void importCommunity(final String id) {
private Map<String, Context> getContextMap() throws CommunityException {
try {
final CommunityDetails community = cc.getCommunity(id);
final List<CommunityContentprovider> datasources =
cc.getCommunityInfo(id, CONTENTPROVIDERS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityDataprovider(id, c));
final List<CommunityProject> projects = cc.getCommunityInfo(id, PROJECTS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityProject(id, c));
final List<CommunityOrganization> orgs = cc.getCommunityInfo(id, ORGANIZATION_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityOrganization(id, c));
service.saveCommunity(community);
service.addCommunityProjectList(id, projects);
service.addCommunityContentProvidersList(id, datasources);
service.addCommunityOrganizationList(id, orgs);
// TODO MANAGE new fields and tables
} catch (final Exception e) {
throw new RuntimeException("Error importing community: " + id, e);
return isClient.getCommunityContextMap();
} catch (final IOException e) {
throw new CommunityException(e);
}
}
public <R> List<R> getCommunityInfo(final Context context, final String idSuffix, final Function<Concept, R> mapping)
throws CommunityException {
if (context != null) {
final Map<String, Category> categories = context.getCategories();
final Category category = categories.get(context.getId() + idSuffix);
if (category != null) { return category.getConcepts()
.stream()
.map(mapping)
.collect(Collectors.toList()); }
}
return Lists.newArrayList();
}
private void importCommunity(final Context context) {
try {
final CommunityDetails community = CommunityMappingUtils.asCommunityProfile(context);
final List<CommunityContentprovider> datasources =
getCommunityInfo(context, CONTENTPROVIDERS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityDataprovider(context.getId(), c));
final List<CommunityProject> projects =
getCommunityInfo(context, PROJECTS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityProject(context.getId(), c));
final List<CommunityOrganization> orgs =
getCommunityInfo(context, ORGANIZATION_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityOrganization(context.getId(), c));
service.saveCommunity(community);
service.addCommunityProjectList(context.getId(), projects);
service.addCommunityContentProvidersList(context.getId(), datasources);
service.addCommunityOrganizationList(context.getId(), orgs);
// TODO MANAGE new fields and tables
} catch (final Exception e) {
throw new RuntimeException("Error importing community: " + context.getId(), e);
}
}
}

75
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/CommunityMappingUtils.java
View File

@ -1,32 +1,6 @@
package eu.dnetlib.openaire.community;
import static eu.dnetlib.openaire.common.Utils.escape;
import static eu.dnetlib.openaire.community.CommunityCommon.CCONTENTPROVIDER_NAME;
import static eu.dnetlib.openaire.community.CommunityCommon.CCONTENTPROVIDER_OFFICIALNAME;
import static eu.dnetlib.openaire.community.CommunityCommon.CCONTENTPROVIDER_SELCRITERIA;
import static eu.dnetlib.openaire.community.CommunityCommon.CORGANIZATION_LOGOURL;
import static eu.dnetlib.openaire.community.CommunityCommon.CORGANIZATION_NAME;
import static eu.dnetlib.openaire.community.CommunityCommon.CORGANIZATION_WEBSITEURL;
import static eu.dnetlib.openaire.community.CommunityCommon.CPROFILE_ADVANCED_CONSTRAINT;
import static eu.dnetlib.openaire.community.CommunityCommon.CPROFILE_CREATIONDATE;
import static eu.dnetlib.openaire.community.CommunityCommon.CPROFILE_FOS;
import static eu.dnetlib.openaire.community.CommunityCommon.CPROFILE_SDG;
import static eu.dnetlib.openaire.community.CommunityCommon.CPROFILE_SUBJECT;
import static eu.dnetlib.openaire.community.CommunityCommon.CPROJECT_ACRONYM;
import static eu.dnetlib.openaire.community.CommunityCommon.CPROJECT_FULLNAME;
import static eu.dnetlib.openaire.community.CommunityCommon.CPROJECT_FUNDER;
import static eu.dnetlib.openaire.community.CommunityCommon.CPROJECT_NUMBER;
import static eu.dnetlib.openaire.community.CommunityCommon.CSUMMARY_DESCRIPTION;
import static eu.dnetlib.openaire.community.CommunityCommon.CSUMMARY_LOGOURL;
import static eu.dnetlib.openaire.community.CommunityCommon.CSUMMARY_NAME;
import static eu.dnetlib.openaire.community.CommunityCommon.CSUMMARY_STATUS;
import static eu.dnetlib.openaire.community.CommunityCommon.CSUMMARY_ZENODOC;
import static eu.dnetlib.openaire.community.CommunityCommon.CSV_DELIMITER;
import static eu.dnetlib.openaire.community.CommunityCommon.CZENODOCOMMUNITY_ID;
import static eu.dnetlib.openaire.community.CommunityCommon.ID_SEPARATOR;
import static eu.dnetlib.openaire.community.CommunityCommon.OPENAIRE_ID;
import static eu.dnetlib.openaire.community.CommunityCommon.PIPE_SEPARATOR;
import static eu.dnetlib.openaire.community.CommunityCommon.ZENODOCOMMUNITY_ID_SUFFIX;
import java.time.LocalDateTime;
import java.time.ZoneId;
@ -63,6 +37,55 @@ public class CommunityMappingUtils {
private final static String pattern = "yyyy-MM-dd'T'hh:mm:ss";
// common
public final static String OPENAIRE_ID = "openaireId";
public final static String PIPE_SEPARATOR = "||";
public final static String ID_SEPARATOR = "::";
public final static String CSV_DELIMITER = ",";
public final static String CLABEL = "label";
// id suffixes
public final static String PROJECTS_ID_SUFFIX = ID_SEPARATOR + "projects";
public final static String CONTENTPROVIDERS_ID_SUFFIX = ID_SEPARATOR + "contentproviders";
public final static String ZENODOCOMMUNITY_ID_SUFFIX = ID_SEPARATOR + "zenodocommunities";
public final static String ORGANIZATION_ID_SUFFIX = ID_SEPARATOR + "organizations";
// community summary
public final static String CSUMMARY_DESCRIPTION = "description";
public final static String CSUMMARY_LOGOURL = "logourl";
public final static String CSUMMARY_STATUS = "status";
public final static String CSUMMARY_NAME = "name";
public final static String CSUMMARY_MANAGER = "manager";
public final static String CSUMMARY_ZENODOC = "zenodoCommunity";
// community profile
public final static String CPROFILE_SUBJECT = "subject";
public final static String CPROFILE_CREATIONDATE = "creationdate";
public final static String CPROFILE_FOS = "fos";
public final static String CPROFILE_SDG = "sdg";
public final static String CPROFILE_ADVANCED_CONSTRAINT = "advancedConstraints";
// community project
public final static String CPROJECT_FUNDER = "funder";
public final static String CPROJECT_NUMBER = "CD_PROJECT_NUMBER";
public final static String CPROJECT_FULLNAME = "projectfullname";
public final static String CPROJECT_ACRONYM = "acronym";
// community content provider
public final static String CCONTENTPROVIDER_NAME = "name";
public final static String CCONTENTPROVIDER_OFFICIALNAME = "officialname";
public final static String CCONTENTPROVIDER_ENABLED = "enabled";
public final static String CCONTENTPROVIDERENABLED_DEFAULT = "true";
public final static String CCONTENTPROVIDER_SELCRITERIA = "selcriteria";
// community zenodo community
public final static String CZENODOCOMMUNITY_ID = "zenodoid";
// community organization
public final static String CORGANIZATION_NAME = "name";
public final static String CORGANIZATION_LOGOURL = "logourl";
public final static String CORGANIZATION_WEBSITEURL = "websiteurl";
private static final Log log = LogFactory.getLog(CommunityMappingUtils.class);
public static CommunitySummary asCommunitySummary(final Context c) {