new_model_for_communities #15

Merged
michele.artini merged 81 commits from new_model_for_communities into master 2023-10-24 08:12:26 +02:00
7 changed files with 36 additions and 342 deletions
Showing only changes of commit 5a874b2379 - Show all commits

View File

@ -13,18 +13,25 @@ public interface ISClient {
String getObjectStoreId(String dsId) throws Exception;
@Deprecated
Map<String, Context> getFunderContextMap() throws IOException;
@Deprecated
Map<String, Context> getCommunityContextMap() throws IOException;
@Deprecated
Map<String, Context> getContextMap(final List<String> type) throws IOException;
@Deprecated
void updateContextParam(String id, String name, String value, boolean toEscape);
@Deprecated
void updateContextAttribute(String id, String name, String value);
@Deprecated
void addConcept(String id, String categoryId, String data);
@Deprecated
void removeConcept(String id, String categoryId, String conceptId);
void dropCache();
@ -38,10 +45,13 @@ public interface ISClient {
* @param value
* new value for the attribute
*/
@Deprecated
void updateConceptAttribute(String id, String name, String value);
@Deprecated
void updateConceptParam(String id, String name, String value);
@Deprecated
void updateConceptParamNoEscape(String id, String name, String value);
}

View File

@ -79,18 +79,21 @@ public class ISClientImpl implements ISClient {
@Override
@Cacheable("context-cache-funder")
@Deprecated
public Map<String, Context> getFunderContextMap() throws IOException {
return _processContext(_getQuery(config.getFindFunderContexts()));
}
@Override
@Cacheable("context-cache-community")
@Deprecated
public Map<String, Context> getCommunityContextMap() throws IOException {
return _processContext(_getQuery(config.getFindCommunityContexts()));
}
@Override
@Cacheable("context-cache")
@Deprecated
public Map<String, Context> getContextMap(final List<String> type) throws IOException {
if (Objects.isNull(type) || type.isEmpty()) {
return _processContext(_getQuery(config.getFindContextProfiles()));
@ -98,16 +101,15 @@ public class ISClientImpl implements ISClient {
try {
final String xqueryTemplate = _getQuery(config.getFindContextProfilesByType());
final String xquery = String.format(xqueryTemplate, type.stream()
.map(t -> String.format("./RESOURCE_PROFILE/BODY/CONFIGURATION/context/@type = '%s'", t))
.collect(Collectors.joining(" or ")));
.map(t -> String.format("./RESOURCE_PROFILE/BODY/CONFIGURATION/context/@type = '%s'", t))
.collect(Collectors.joining(" or ")));
return _processContext(xquery);
}catch(Exception e){
} catch (final Exception e) {
log.error(e.getMessage());
throw new RuntimeException(e);
}
}
}
@ -115,6 +117,7 @@ public class ISClientImpl implements ISClient {
@CacheEvict(value = {
"context-cache", "context-cache-funder"
}, allEntries = true)
@Deprecated
public void updateContextParam(final String id, final String name, final String value, final boolean toEscape) {
if (getSize(id, name) > 0) {
try {
@ -148,6 +151,7 @@ public class ISClientImpl implements ISClient {
@CacheEvict(value = {
"context-cache", "context-cache-funder"
}, allEntries = true)
@Deprecated
public void updateContextAttribute(final String id, final String name, final String value) {
final Escaper esc = XmlEscapers.xmlAttributeEscaper();
try {
@ -162,6 +166,7 @@ public class ISClientImpl implements ISClient {
@CacheEvict(value = {
"context-cache", "context-cache-funder"
}, allEntries = true)
@Deprecated
public void addConcept(final String id, final String categoryId, final String data) {
try {
_quickSeachProfile(String.format("update insert %s into collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" +
@ -175,6 +180,7 @@ public class ISClientImpl implements ISClient {
@CacheEvict(value = {
"context-cache", "context-cache-funder"
}, allEntries = true)
@Deprecated
public void removeConcept(final String id, final String categoryId, final String conceptId) {
try {
_quickSeachProfile(String.format("for $concept in collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')" +
@ -190,6 +196,7 @@ public class ISClientImpl implements ISClient {
@CacheEvict(value = {
"context-cache", "context-cache-community", "context-cache-funder"
}, allEntries = true)
@Deprecated
public void updateConceptAttribute(final String id, final String name, final String value) {
final Escaper esc = XmlEscapers.xmlAttributeEscaper();
try {
@ -205,6 +212,7 @@ public class ISClientImpl implements ISClient {
@CacheEvict(value = {
"context-cache", "context-cache-funder"
}, allEntries = true)
@Deprecated
public void updateConceptParam(final String id, final String name, final String value) {
try {
_quickSeachProfile(getConceptXQuery(id, name, value));
@ -217,6 +225,7 @@ public class ISClientImpl implements ISClient {
@CacheEvict(value = {
"context-cache", "context-cache-funder"
}, allEntries = true)
@Deprecated
public void updateConceptParamNoEscape(final String id, final String name, final String value) {
try {
_quickSeachProfile(getConceptXQueryNoEscape(id, name, value));
@ -227,6 +236,7 @@ public class ISClientImpl implements ISClient {
/// HELPERS
@Deprecated
private String getInsertXQuery(final String id, final String paramName, final String paramValue, final boolean toEscape) {
String value;
if (toEscape) {
@ -243,6 +253,7 @@ public class ISClientImpl implements ISClient {
}
}
@Deprecated
private String getXQuery(final String id, final String name, final String paramValue, final boolean toEscape) {
String value = paramValue;
if (toEscape) {
@ -269,6 +280,7 @@ public class ISClientImpl implements ISClient {
// }
// }
@Deprecated
private String getConceptXQuery(final String id, final String name, final String value) {
final Escaper esc = XmlEscapers.xmlContentEscaper();
if (StringUtils.isNotBlank(value)) {
@ -280,6 +292,7 @@ public class ISClientImpl implements ISClient {
}
}
@Deprecated
private String getConceptXQueryNoEscape(final String id, final String name, final String value) {
if (StringUtils.isNotBlank(value)) {
@ -291,10 +304,12 @@ public class ISClientImpl implements ISClient {
}
}
@Deprecated
private Map<String, Context> _processContext(final String xquery) throws IOException {
return _processContext(new LinkedBlockingQueue<>(), xquery);
}
@Deprecated
private Map<String, Context> _processContext(final Queue<Throwable> errors, final String xquery) throws IOException {
try {
return getContextProfiles(errors, xquery).stream()
@ -312,6 +327,7 @@ public class ISClientImpl implements ISClient {
}
}
@Deprecated
private List<String> getContextProfiles(final Queue<Throwable> errors, final String xquery) throws IOException {
log.warn("getContextProfiles(): not using cache");
try {
@ -335,13 +351,13 @@ public class ISClientImpl implements ISClient {
final List<String> res = Lists.newArrayList();
log.debug(String.format("running xquery:\n%s", xquery));
try{
try {
final List<String> list = isLookUpService.quickSearchProfile(xquery);
if (list != null) {
res.addAll(list);
}
log.debug(String.format("query result size: %s", res.size()));
}catch(Exception ex){
} catch (final Exception ex) {
log.error(ex.getMessage());
throw new ISLookUpException("");
}

View File

@ -1,55 +1,30 @@
package eu.dnetlib.openaire.community;
import static eu.dnetlib.openaire.community.CommunityConstants.CLABEL;
import static eu.dnetlib.openaire.community.CommunityConstants.CONTENTPROVIDERS_ID_SUFFIX;
import static eu.dnetlib.openaire.community.CommunityConstants.CPROFILE_ADVANCED_CONSTRAINT;
import static eu.dnetlib.openaire.community.CommunityConstants.CPROFILE_FOS;
import static eu.dnetlib.openaire.community.CommunityConstants.CPROFILE_SDG;
import static eu.dnetlib.openaire.community.CommunityConstants.CPROFILE_SUBJECT;
import static eu.dnetlib.openaire.community.CommunityConstants.CSUMMARY_DESCRIPTION;
import static eu.dnetlib.openaire.community.CommunityConstants.CSUMMARY_LOGOURL;
import static eu.dnetlib.openaire.community.CommunityConstants.CSUMMARY_NAME;
import static eu.dnetlib.openaire.community.CommunityConstants.CSUMMARY_STATUS;
import static eu.dnetlib.openaire.community.CommunityConstants.CSUMMARY_ZENODOC;
import static eu.dnetlib.openaire.community.CommunityConstants.CSV_DELIMITER;
import static eu.dnetlib.openaire.community.CommunityConstants.ID_SEPARATOR;
import static eu.dnetlib.openaire.community.CommunityConstants.ORGANIZATION_ID_SUFFIX;
import static eu.dnetlib.openaire.community.CommunityConstants.PROJECTS_ID_SUFFIX;
import static eu.dnetlib.openaire.community.CommunityConstants.ZENODOCOMMUNITY_ID_SUFFIX;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.stereotype.Component;
import com.google.common.base.Functions;
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.gson.Gson;
import eu.dnetlib.openaire.common.ISClient;
import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
import eu.dnetlib.openaire.exporter.exceptions.ResourceNotFoundException;
import eu.dnetlib.openaire.exporter.model.community.CommunityContentprovider;
import eu.dnetlib.openaire.exporter.model.community.CommunityDetails;
import eu.dnetlib.openaire.exporter.model.community.CommunityOpenAIRECommunities;
import eu.dnetlib.openaire.exporter.model.community.CommunityOrganization;
import eu.dnetlib.openaire.exporter.model.community.CommunityProject;
import eu.dnetlib.openaire.exporter.model.community.CommunitySummary;
import eu.dnetlib.openaire.exporter.model.community.CommunityWritableProperties;
import eu.dnetlib.openaire.exporter.model.community.CommunityZenodoCommunity;
import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionCriteria;
@Component
@ConditionalOnProperty(value = "openaire.exporter.enable.community", havingValue = "true")
@ -58,10 +33,7 @@ public class CommunityApiCore {// implements CommunityClient{
private static final Log log = LogFactory.getLog(CommunityApiCore.class);
@Autowired
private CommunityClient cci;
@Autowired
// @Autowired
private ISClient isClient;
@Autowired
@ -77,69 +49,11 @@ public class CommunityApiCore {// implements CommunityClient{
}
private void removeAdvancedConstraint(final String id) throws ResourceNotFoundException, CommunityException {
cc.getCommunity(id);
isClient.updateContextParam(id, CPROFILE_ADVANCED_CONSTRAINT, "", false);
cc.removeAdvancedConstraint(id);
}
public void setCommunity(final String id, final CommunityWritableProperties details) throws CommunityException, ResourceNotFoundException {
cc.getCommunity(id); // ensure the community exists.
if (details.getShortName() != null) {
isClient.updateContextAttribute(id, CLABEL, details.getShortName());
}
if (details.getName() != null) {
isClient.updateContextParam(id, CSUMMARY_NAME, details.getName(), true);
}
if (details.getDescription() != null) {
isClient.updateContextParam(id, CSUMMARY_DESCRIPTION, details.getDescription(), true);
}
if (details.getLogoUrl() != null) {
isClient.updateContextParam(id, CSUMMARY_LOGOURL, details.getLogoUrl(), true);
}
if (details.getStatus() != null) {
isClient.updateContextParam(id, CSUMMARY_STATUS, details.getStatus().name(), true);
}
if (details.getSubjects() != null) {
isClient.updateContextParam(id, CPROFILE_SUBJECT, Joiner.on(CSV_DELIMITER).join(details.getSubjects()), true);
}
if (details.getFos() != null) {
isClient.updateContextParam(id, CPROFILE_FOS, Joiner.on(CSV_DELIMITER).join(details.getFos()), true);
}
if (details.getSdg() != null) {
isClient.updateContextParam(id, CPROFILE_SDG, Joiner.on(CSV_DELIMITER).join(details.getSdg()), true);
}
if (details.getAdvancedConstraints() != null) {
isClient.updateContextParam(id, CPROFILE_ADVANCED_CONSTRAINT, "<![CDATA[" + new Gson().toJson(details.getAdvancedConstraints()) + "]]>", false);
}
if (details.getMainZenodoCommunity() != null) {
isClient.updateContextParam(id, CSUMMARY_ZENODOC, details.getMainZenodoCommunity(), true);
}
cc.updateCommunity(id, details);
}
public List<CommunityProject> getCommunityProjects(final String id) throws CommunityException, ResourceNotFoundException {
cc.getCommunity(id); // ensure the community exists.
return cc.getCommunityInfo(id, PROJECTS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityProject(id, c));
}
private String nextId(final Integer id) {
return String.valueOf(id + 1);
}
public List<CommunityContentprovider> getCommunityContentproviders(final String id) throws CommunityException, ResourceNotFoundException {
cc.getCommunity(id); // ensure the community exists.
return cc.getCommunityInfo(id, CONTENTPROVIDERS_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityDataprovider(id, c));
@ -155,162 +69,6 @@ public class CommunityApiCore {// implements CommunityClient{
return cc.getCommunityInfo(id, ORGANIZATION_ID_SUFFIX, c -> CommunityMappingUtils.asCommunityOrganization(id, c));
}
public CommunityDetails addCommunitySubjects(final String id, final List<String> subjects) throws CommunityException, ResourceNotFoundException {
final CommunityDetails cd = new CommunityDetails();
final Set<String> current = Sets.newHashSet(cc.getCommunity(id).getSubjects());
current.addAll(subjects);
cd.setSubjects(Lists.newArrayList(current));
setCommunity(id, CommunityWritableProperties.fromDetails(cd));
return cd;
}
public CommunityDetails removeCommunitySubjects(final String id, final List<String> subjects) throws CommunityException, ResourceNotFoundException {
final CommunityDetails cd = new CommunityDetails();
final Set<String> current = Sets.newHashSet(cc.getCommunity(id).getSubjects());
current.removeAll(subjects);
cd.setSubjects(Lists.newArrayList(current));
setCommunity(id, CommunityWritableProperties.fromDetails(cd));
return cd;
}
public CommunityDetails addCommunityFOS(final String id, final List<String> foss) throws CommunityException, ResourceNotFoundException {
final CommunityDetails cd = new CommunityDetails();
final Set<String> current = Sets.newHashSet();
if (Optional.ofNullable(cc.getCommunity(id).getFos()).isPresent()) {
current.addAll(cc.getCommunity(id).getFos());
}
current.addAll(foss);
cd.setFos(Lists.newArrayList(current));
setCommunity(id, CommunityWritableProperties.fromDetails(cd));
return cd;
}
public CommunityDetails removeCommunityFOS(final String id, final List<String> foss) throws CommunityException, ResourceNotFoundException {
final CommunityDetails cd = new CommunityDetails();
final Set<String> current = Sets.newHashSet(cc.getCommunity(id).getFos());
current.removeAll(foss);
cd.setFos(Lists.newArrayList(current));
setCommunity(id, CommunityWritableProperties.fromDetails(cd));
return cd;
}
public CommunityDetails addCommunitySDG(final String id, final List<String> sdgs) throws CommunityException, ResourceNotFoundException {
final CommunityDetails cd = new CommunityDetails();
final Set<String> current = Sets.newHashSet();
if (Optional.ofNullable(cc.getCommunity(id).getSdg()).isPresent()) {
current.addAll(cc.getCommunity(id).getSdg());
}
current.addAll(sdgs);
cd.setSdg(Lists.newArrayList(current));
setCommunity(id, CommunityWritableProperties.fromDetails(cd));
return cd;
}
public CommunityDetails removeCommunitySDG(final String id, final List<String> sdgs) throws CommunityException, ResourceNotFoundException {
final CommunityDetails cd = new CommunityDetails();
final Set<String> current = Sets.newHashSet(cc.getCommunity(id).getSdg());
current.removeAll(sdgs);
cd.setSdg(Lists.newArrayList(current));
setCommunity(id, CommunityWritableProperties.fromDetails(cd));
return cd;
}
public CommunityDetails addCommunityAdvancedConstraint(final String id, final SelectionCriteria advancedCosntraint)
throws CommunityException, ResourceNotFoundException {
final CommunityDetails cd = new CommunityDetails();
cd.setAdvancedConstraints(advancedCosntraint);
setCommunity(id, CommunityWritableProperties.fromDetails(cd));
return cd;
}
public CommunityDetails removeCommunityAdvancedConstraint(final String id) throws ResourceNotFoundException, CommunityException {
removeAdvancedConstraint(id);
return new CommunityDetails();
}
@CacheEvict(value = "community-cache", allEntries = true)
public void removeCommunityZenodoCommunity(final String id, final Integer zenodoCommId) throws CommunityException, ResourceNotFoundException {
final Map<Integer, CommunityZenodoCommunity> zcomms = getZenodoCommunityMap(id);
if (!zcomms.containsKey(zenodoCommId)) {
throw new ResourceNotFoundException(String.format("Zenodo community '%s' doesn't exist within context '%s'", zenodoCommId, id));
}
isClient.removeConcept(id, id + ZENODOCOMMUNITY_ID_SUFFIX, id + ZENODOCOMMUNITY_ID_SUFFIX + ID_SEPARATOR + zenodoCommId);
cc.removeFromCategory(id, ZENODOCOMMUNITY_ID_SUFFIX, String.valueOf(zenodoCommId));
}
@CacheEvict(value = "community-cache", allEntries = true)
public CommunityZenodoCommunity addCommunityZenodoCommunity(final String id, final CommunityZenodoCommunity zc)
throws CommunityException, ResourceNotFoundException {
if (!StringUtils.equalsIgnoreCase(id, zc.getCommunityId())) { throw new CommunityException("parameters 'id' and zc.communityId must be coherent"); }
if (!StringUtils.isNotBlank(zc.getZenodoid())) { throw new CommunityException("parameter zenodoid cannot be null or empty"); }
final TreeMap<Integer, CommunityZenodoCommunity> zcs = getZenodoCommunityMap(id);
for (final CommunityZenodoCommunity czc : zcs.values()) {
if (czc.getZenodoid().equals(zc.getZenodoid())) { throw new CommunityException("Zenodo community already associated to the RCD"); }
}
zc.setId(nextId(!zcs.isEmpty() ? zcs.lastKey() : 0));
isClient.addConcept(id, id + ZENODOCOMMUNITY_ID_SUFFIX, CommunityMappingUtils.asZenodoCommunityXML(id, zc));
cc.updateZenodoCommunity(id, zc);
return zc;
}
public CommunityOpenAIRECommunities getOpenAIRECommunities(final String zenodoId) throws CommunityException, ResourceNotFoundException {
if (cci.getInverseZenodoCommunityMap().containsKey(zenodoId)) {
return new CommunityOpenAIRECommunities().setZenodoid(zenodoId)
.setOpenAirecommunitylist(cci.getInverseZenodoCommunityMap().get(zenodoId).stream().collect(Collectors.toList()));
}
return new CommunityOpenAIRECommunities();
}
// HELPERS
private TreeMap<Integer, CommunityZenodoCommunity> getZenodoCommunityMap(final String id) throws CommunityException, ResourceNotFoundException {

View File

@ -1,16 +0,0 @@
package eu.dnetlib.openaire.community;
import java.util.Map;
import java.util.Set;
import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
import eu.dnetlib.openaire.exporter.exceptions.ResourceNotFoundException;
@Deprecated
public interface CommunityClient {
Map<String, Set<String>> getInverseZenodoCommunityMap() throws CommunityException, ResourceNotFoundException;
void dropCache();
}

View File

@ -1,68 +0,0 @@
package eu.dnetlib.openaire.community;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import eu.dnetlib.openaire.exporter.exceptions.CommunityException;
import eu.dnetlib.openaire.exporter.exceptions.ResourceNotFoundException;
import eu.dnetlib.openaire.exporter.model.community.CommunitySummary;
import eu.dnetlib.openaire.exporter.model.community.CommunityZenodoCommunity;
@Component
@Deprecated
public class CommunityClientImpl implements CommunityClient {
private static final Log log = LogFactory.getLog(CommunityClient.class);
@Autowired
private CommunityCommon communityCommon;
@Override
@Cacheable("community-cache")
public Map<String, Set<String>> getInverseZenodoCommunityMap() throws CommunityException, ResourceNotFoundException {
log.info("Creating the data structure. Not using cache");
final Map<String, Set<String>> inverseListMap = new HashMap<>();
final List<CommunitySummary> communityList = communityCommon.listCommunities();
for (final CommunitySummary cs : communityList) {
final String communityId = cs.getId();
final List<CommunityZenodoCommunity> czc = communityCommon.getCommunityZenodoCommunities(communityId);
for (final CommunityZenodoCommunity zc : czc) {
final String zenodoId = zc.getZenodoid();
if (!inverseListMap.containsKey(zenodoId)) {
inverseListMap.put(zc.getZenodoid(), new HashSet<>());
}
inverseListMap.get(zc.getZenodoid()).add(communityId);
}
final String zenodoMainCommunity = communityCommon.getCommunity(communityId).getZenodoCommunity();
if (!inverseListMap.containsKey(zenodoMainCommunity)) {
inverseListMap.put(zenodoMainCommunity, new HashSet<>());
}
inverseListMap.get(zenodoMainCommunity).add(communityId);
}
return inverseListMap;
}
@Override
@CacheEvict(cacheNames = {
"community-cache", "context-cache-community"
}, allEntries = true)
@Scheduled(fixedDelayString = "${openaire.exporter.cache.ttl}")
public void dropCache() {
log.debug("dropped community cache");
}
}

View File

@ -43,10 +43,9 @@ import eu.dnetlib.openaire.exporter.model.community.selectioncriteria.SelectionC
public class CommunityService {
// TODO
// 1) Subcommunities: modello e repository
// 2) Subcommunities: importazione tramite profili
// 3) Subcommunities: visualizzazione tramite le context api
// 5) ricontrollare se tutti i campi del database sono esposti dalle api (modello + mapping)
// 1) Subcommunities: importazione tramite profili
// 2) Subcommunities: visualizzazione tramite le context api
// 3) ricontrollare se tutti i campi del database sono esposti dalle api (modello + mapping)
@Autowired
private DbCommunityRepository dbCommunityRepository;

View File

@ -36,7 +36,6 @@ import eu.dnetlib.enabling.datasources.common.DsmException;
import eu.dnetlib.enabling.datasources.common.DsmForbiddenException;
import eu.dnetlib.enabling.datasources.common.DsmNotFoundException;
import eu.dnetlib.openaire.common.ISClient;
import eu.dnetlib.openaire.community.CommunityClient;
import eu.dnetlib.openaire.dsm.dao.CountryTermRepository;
import eu.dnetlib.openaire.dsm.dao.DatasourceDao;
import eu.dnetlib.openaire.dsm.dao.MongoLoggerClient;
@ -91,9 +90,6 @@ public class DsmCore {
@Autowired
private JdbcTemplate jdbcTemplate;
@Autowired
private CommunityClient communityClient;
public List<Country> listCountries() throws DsmApiException {
final List<Country> countries = Lists.newArrayList();
final Vocabulary v = vocabularyClient.getCountries();
@ -321,7 +317,6 @@ public class DsmCore {
mongoLoggerClient.dropCache();
isClient.dropCache();
vocabularyClient.dropCache();
communityClient.dropCache();
}
// HELPERS //////////////