D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

new_model_for_communities #15

Merged
michele.artini merged 81 commits from new_model_for_communities into master 2023-10-24 08:12:26 +02:00
Conversation 0 Commits 81 Files Changed 70 +1554 -96
10 changed files with 1554 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 15f0d8cea7 - Show all commits

93
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/importer/CommunityImporterService.java
View File

@ -247,88 +247,63 @@ public class CommunityImporterService {
details.setType(CommunityType.valueOf(c.getType())); details.setType(CommunityType.valueOf(c.getType()));
details.setMembership(CommunityMembershipType.open); details.setMembership(CommunityMembershipType.open);
details.setClaim(CommunityClaimType.all); details.setClaim(CommunityClaimType.all);
details.setDescription(asCsv(CSUMMARY_DESCRIPTION, c.getParams()));
final Map<String, List<Param>> params = c.getParams(); details.setLogoUrl(asCsv(CSUMMARY_LOGOURL, c.getParams()));
if (params.containsKey(CSUMMARY_DESCRIPTION)) { details.setStatus(CommunityStatus.valueOf(firstValue(CSUMMARY_STATUS, c.getParams())));
details.setDescription(asCsv(params.get(CSUMMARY_DESCRIPTION))); details.setName(asCsv(CSUMMARY_NAME, c.getParams()));
}
if (params.containsKey(CSUMMARY_LOGOURL)) {
details.setLogoUrl(asCsv(params.get(CSUMMARY_LOGOURL)));
}
if (params.containsKey(CSUMMARY_STATUS)) {
details.setStatus(CommunityStatus.valueOf(firstValue(params, CSUMMARY_STATUS)));
}
if (params.containsKey(CSUMMARY_NAME)) {
details.setName(asCsv(params.get(CSUMMARY_NAME)));
} else {
details.setName(c.getLabel()); details.setName(c.getLabel());
} details.setZenodoCommunity(asCsv(CSUMMARY_ZENODOC, c.getParams()));
if (params.containsKey(CSUMMARY_ZENODOC)) { details.setSubjects(splitValues(asValues(CPROFILE_SUBJECT, c.getParams()), CSV_DELIMITER));
details.setZenodoCommunity(asCsv(params.get(CSUMMARY_ZENODOC))); details.setFos(splitValues(asValues(CPROFILE_FOS, c.getParams()), CSV_DELIMITER));
} details.setSdg(splitValues(asValues(CPROFILE_SDG, c.getParams()), CSV_DELIMITER));
if (params.containsKey(CPROFILE_SUBJECT)) {
details.setSubjects(splitValues(asValues(params.get(CPROFILE_SUBJECT)), CSV_DELIMITER));
}
if (params.containsKey(CPROFILE_FOS)) {
details.setFos(splitValues(asValues(params.get(CPROFILE_FOS)), CSV_DELIMITER));
}
if (params.containsKey(CPROFILE_SDG)) {
details.setSdg(splitValues(asValues(params.get(CPROFILE_SDG)), CSV_DELIMITER));
}
if (params.containsKey(CPROFILE_ADVANCED_CONSTRAINT)) {
// In the map the string is the serialization of the json representing the selection criteria so it is a valid json // In the map the string is the serialization of the json representing the selection criteria so it is a valid json
details.setAdvancedConstraints(SelectionCriteria.fromJson(asCsv(params.get(CPROFILE_ADVANCED_CONSTRAINT)))); details.setAdvancedConstraints(SelectionCriteria.fromJson(asCsv(CPROFILE_ADVANCED_CONSTRAINT, c.getParams())));
}
if (params.containsKey(CPROFILE_REMOVE_CONSTRAINT)) {
// In the map the string is the serialization of the json representing the selection criteria so it is a valid json // In the map the string is the serialization of the json representing the selection criteria so it is a valid json
details.setRemoveConstraints(SelectionCriteria.fromJson(asCsv(params.get(CPROFILE_REMOVE_CONSTRAINT)))); details.setRemoveConstraints(SelectionCriteria.fromJson(asCsv(CPROFILE_REMOVE_CONSTRAINT, c.getParams())));
}
if (params.containsKey(CPROFILE_CREATIONDATE)) {
try { try {
details.setCreationDate(CommunityMappingUtils.asLocalDateTime(asCsv(params.get(CPROFILE_CREATIONDATE)))); details.setCreationDate(CommunityMappingUtils.asLocalDateTime(asCsv(CPROFILE_CREATIONDATE, c.getParams())));
} catch (final Exception e) { } catch (final Exception e) {
log.debug("Exception on date format: " + e.getMessage()); log.debug("Exception on date format: " + e.getMessage());
} }
}
return details; return details;
} }
private static CommunityProject asCommunityProject(final String communityId, final Concept c) { private static CommunityProject asCommunityProject(final String communityId, final Concept c) {
final Map<String, List<Param>> p = c.getParams(); final List<Param> p = c.getParams();
final CommunityProject project = new CommunityProject(); final CommunityProject project = new CommunityProject();
project.setCommunityId(communityId); project.setCommunityId(communityId);
project.setOpenaireId(firstValue(p, OPENAIRE_ID)); project.setOpenaireId(firstValue(OPENAIRE_ID, p));
project.setFunder(firstValue(p, CPROJECT_FUNDER)); project.setFunder(firstValue(CPROJECT_FUNDER, p));
project.setGrantId(firstValue(p, CPROJECT_NUMBER)); project.setGrantId(firstValue(CPROJECT_NUMBER, p));
project.setName(firstValue(p, CPROJECT_FULLNAME)); project.setName(firstValue(CPROJECT_FULLNAME, p));
project.setAcronym(firstValue(p, CPROJECT_ACRONYM)); project.setAcronym(firstValue(CPROJECT_ACRONYM, p));
return project; return project;
} }
private static CommunityContentprovider asCommunityDataprovider(final String communityId, final Concept c) { private static CommunityContentprovider asCommunityDataprovider(final String communityId, final Concept c) {
final Map<String, List<Param>> p = c.getParams(); final List<Param> p = c.getParams();
final CommunityContentprovider d = new CommunityContentprovider(); final CommunityContentprovider d = new CommunityContentprovider();
d.setCommunityId(communityId); d.setCommunityId(communityId);
d.setOpenaireId(firstValue(p, OPENAIRE_ID)); d.setOpenaireId(firstValue(OPENAIRE_ID, p));
d.setName(firstValue(p, CCONTENTPROVIDER_NAME)); d.setName(firstValue(CCONTENTPROVIDER_NAME, p));
d.setOfficialname(firstValue(p, CCONTENTPROVIDER_OFFICIALNAME)); d.setOfficialname(firstValue(CCONTENTPROVIDER_OFFICIALNAME, p));
d.setSelectioncriteria(SelectionCriteria.fromJson(firstValue(p, CCONTENTPROVIDER_SELCRITERIA))); d.setSelectioncriteria(SelectionCriteria.fromJson(firstValue(CCONTENTPROVIDER_SELCRITERIA, p)));
return d; return d;
} }
private static CommunityOrganization asCommunityOrganization(final String id, final Concept c) { private static CommunityOrganization asCommunityOrganization(final String id, final Concept c) {
final Map<String, List<Param>> p = c.getParams(); final List<Param> p = c.getParams();
final CommunityOrganization o = new CommunityOrganization(); final CommunityOrganization o = new CommunityOrganization();
o.setCommunityId(id); o.setCommunityId(id);
o.setName(firstValue(p, CORGANIZATION_NAME)); o.setName(firstValue(CORGANIZATION_NAME, p));
o.setLogo_url(getDecodedUrl(firstValue(p, CORGANIZATION_LOGOURL))); o.setLogo_url(getDecodedUrl(firstValue(CORGANIZATION_LOGOURL, p)));
o.setWebsite_url(getDecodedUrl(firstValue(p, CORGANIZATION_WEBSITEURL))); o.setWebsite_url(getDecodedUrl(firstValue(CORGANIZATION_WEBSITEURL, p)));
return o; return o;
} }
private static String asZenodoCommunity(final Concept c) { private static String asZenodoCommunity(final Concept c) {
return firstValue(c.getParams(), CZENODOCOMMUNITY_ID); return firstValue(CZENODOCOMMUNITY_ID, c.getParams());
} }
private static List<SubCommunity> asSubCommunities(final String communityId, final String parent, final String category, final List<Concept> concepts) { private static List<SubCommunity> asSubCommunities(final String communityId, final String parent, final String category, final List<Concept> concepts) {
@ -368,18 +343,20 @@ public class CommunityImporterService {
.collect(Collectors.toList()); .collect(Collectors.toList());
} }
private static String firstValue(final Map<String, List<Param>> p, final String paramName) { private static String firstValue(final String name, final List<Param> params) {
return asValues(p.get(paramName)).findFirst().orElse(null); return asValues(name, params).findFirst().orElse(null);
} }
private static String asCsv(final List<Param> params) { private static String asCsv(final String name, final List<Param> params) {
return asValues(params) return asValues(name, params).collect(Collectors.joining(CSV_DELIMITER));
.collect(Collectors.joining(CSV_DELIMITER));
} }
private static Stream<String> asValues(final List<Param> params) { private static Stream<String> asValues(final String name, final List<Param> params) {
return params == null ? Stream.empty() return params == null ? Stream.empty()
: params.stream() : params.stream()
.filter(p -> p != null)
.filter(p -> StringUtils.isNotBlank(p.getName()))
.filter(p -> p.getName().trim().equals(name.trim()))
.map(Param::getValue) .map(Param::getValue)
.map(StringUtils::trim) .map(StringUtils::trim)
.distinct(); .distinct();

9
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/model/DbSubCommunity.java
View File

@ -1,9 +1,8 @@
package eu.dnetlib.openaire.community.model; package eu.dnetlib.openaire.community.model;
import java.io.Serializable; import java.io.Serializable;
import java.util.LinkedHashMap; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map;
import javax.persistence.Column; import javax.persistence.Column;
import javax.persistence.Entity; import javax.persistence.Entity;
@ -46,7 +45,7 @@ public class DbSubCommunity implements Serializable {
@Type(type = "jsonb") @Type(type = "jsonb")
@Column(name = "params") @Column(name = "params")
private Map<String, List<Param>> params = new LinkedHashMap<>(); private List<Param> params = new ArrayList<>();
@Column(name = "parent") @Column(name = "parent")
private String parent; private String parent;
@ -86,11 +85,11 @@ public class DbSubCommunity implements Serializable {
this.category = category; this.category = category;
} }
public Map<String, List<Param>> getParams() { public List<Param> getParams() {
return params; return params;
} }
public void setParams(final Map<String, List<Param>> params) { public void setParams(final List<Param> params) {
this.params = params; this.params = params;
} }

8
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiCore.java
View File

@ -32,13 +32,19 @@ public class ContextApiCore {
private ISClient isClient; private ISClient isClient;
public List<ContextSummary> listContexts(final List<String> type) throws ContextException { public List<ContextSummary> listContexts(final List<String> type) throws ContextException {
return getContextMap(type).values() return getContextMap(type).values()
.stream() .stream()
.map(c -> new ContextSummary() .map(c -> new ContextSummary()
.setId(c.getId()) .setId(c.getId())
.setType(c.getType()) .setType(c.getType())
.setLabel(c.getLabel()) .setLabel(c.getLabel())
.setStatus(c.getParams().containsKey("status") ? c.getParams().get("status").get(0).getValue() : "")) .setStatus(c.getParams()
.stream()
.filter(p -> p.getName().equals("status"))
.map(p -> p.getValue())
.findFirst()
.orElse("")))
.collect(Collectors.toList()); .collect(Collectors.toList());
} }

13
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextMappingUtils.java
View File

@ -1,7 +1,6 @@
package eu.dnetlib.openaire.context; package eu.dnetlib.openaire.context;
import java.text.ParseException; import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -103,18 +102,12 @@ public class ContextMappingUtils {
return BooleanUtils.toBooleanObject(StringUtils.isNotBlank(claim) ? claim : "false"); return BooleanUtils.toBooleanObject(StringUtils.isNotBlank(claim) ? claim : "false");
} }
private static Map<String, List<Param>> parseParams(final Element e) { private static List<Param> parseParams(final Element e) {
final List<Node> params = e.selectNodes("./param"); final List<Node> params = e.selectNodes("./param");
return params.stream() return params.stream()
.map(n -> (Element) n) .map(n -> (Element) n)
.map(p -> new Param() .map(p -> new Param().setName(p.attributeValue("name")).setValue(p.getTextTrim()))
.setName(p.attributeValue("name")) .collect(Collectors.toList());
.setValue(p.getTextTrim()))
.collect(Collectors.toMap(Param::getName, Lists::newArrayList, (p1, p2) -> {
final List<Param> p = new ArrayList<>(p1);
p.addAll(p2);
return p;
}));
} }
public static FunderDetails asFunderDetails(final Context c) { public static FunderDetails asFunderDetails(final Context c) {

47
apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/community/importer/CommunityImporterServiceTest.java
View File

@ -111,4 +111,51 @@ class CommunityImporterServiceTest {
} }
@SuppressWarnings("unchecked")
@Test
public void testImportCommunityFetFp7() throws Exception {
final String profile = IOUtils.toString(getClass().getResourceAsStream("old_community_profile_fet-fp7.xml"), StandardCharsets.UTF_8.toString());
final Queue<Throwable> errors = new LinkedList<>();
final Context context = ContextMappingUtils.parseContext(profile, errors);
assertTrue(errors.isEmpty());
// Mockito.when(jdbcTemplate.queryForList(Mockito.anyString(), Mockito.any(Class.class),
// Mockito.anyString())).thenReturn(Arrays.asList("corda_______"));
importer.importCommunity(context);
final ArgumentCaptor<CommunityDetails> detailsCapture = ArgumentCaptor.forClass(CommunityDetails.class);
final ArgumentCaptor<CommunityProject> projectsCapture = ArgumentCaptor.forClass(CommunityProject.class);
final ArgumentCaptor<CommunityContentprovider> datasourcesCapture = ArgumentCaptor.forClass(CommunityContentprovider.class);
final ArgumentCaptor<CommunityOrganization> orgsCapture = ArgumentCaptor.forClass(CommunityOrganization.class);
final ArgumentCaptor<SubCommunity> subCommunitiesCapture = ArgumentCaptor.forClass(SubCommunity.class);
Mockito.verify(service, Mockito.times(1)).saveCommunity(detailsCapture.capture());
Mockito.verify(service, Mockito.times(1)).addCommunityProjects(Mockito.anyString(), projectsCapture.capture());
Mockito.verify(service, Mockito.times(1)).addCommunityContentProviders(Mockito.anyString(), datasourcesCapture.capture());
Mockito.verify(service, Mockito.times(1)).addCommunityOrganizations(Mockito.anyString(), orgsCapture.capture());
Mockito.verify(service, Mockito.times(1)).addSubCommunities(subCommunitiesCapture.capture());
final CommunityDetails details = detailsCapture.getValue();
assertEquals("fet-fp7", details.getId());
// System.out.println(details);
final List<CommunityProject> projects = projectsCapture.getAllValues();
assertEquals(0, projects.size());
// projects.forEach(System.out::println);
final List<CommunityContentprovider> datasources = datasourcesCapture.getAllValues();
assertEquals(0, datasources.size());
// datasources.forEach(System.out::println);
final List<CommunityOrganization> orgs = orgsCapture.getAllValues();
assertEquals(0, orgs.size());
// orgs.forEach(System.out::println);
final List<SubCommunity> subs = subCommunitiesCapture.getAllValues();
assertEquals(151, subs.size());
subs.forEach(System.out::println);
}
} }

1439
apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/community/importer/old_community_profile_fet-fp7.xml Normal file
View File

File diff suppressed because it is too large Load Diff

9
libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/community/SubCommunity.java
View File

@ -1,8 +1,7 @@
package eu.dnetlib.openaire.exporter.model.community; package eu.dnetlib.openaire.exporter.model.community;
import java.util.LinkedHashMap; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map;
import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect;
@ -28,7 +27,7 @@ public class SubCommunity {
private String category; private String category;
@Schema(description = "the parameters of the subCommunity", required = true) @Schema(description = "the parameters of the subCommunity", required = true)
private Map<String, List<Param>> params = new LinkedHashMap<>(); private List<Param> params = new ArrayList<>();
@Schema(description = "it supports the claims", required = true) @Schema(description = "it supports the claims", required = true)
private boolean claim = false; private boolean claim = false;
@ -73,11 +72,11 @@ public class SubCommunity {
this.category = category; this.category = category;
} }
public Map<String, List<Param>> getParams() { public List<Param> getParams() {
return params; return params;
} }
public void setParams(final Map<String, List<Param>> map) { public void setParams(final List<Param> map) {
this.params = map; this.params = map;
} }

7
libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/context/Category.java
View File

@ -1,7 +1,6 @@
package eu.dnetlib.openaire.exporter.model.context; package eu.dnetlib.openaire.exporter.model.context;
import java.util.List; import java.util.List;
import java.util.Map;
public class Category { public class Category {
@ -11,7 +10,7 @@ public class Category {
private boolean claim; private boolean claim;
private Map<String, List<Param>> params; private List<Param> params;
private List<Concept> concepts; private List<Concept> concepts;
@ -31,7 +30,7 @@ public class Category {
return getConcepts() != null && !getConcepts().isEmpty(); return getConcepts() != null && !getConcepts().isEmpty();
} }
public Map<String, List<Param>> getParams() { public List<Param> getParams() {
return params; return params;
} }
@ -54,7 +53,7 @@ public class Category {
return this; return this;
} }
public Category setParams(final Map<String, List<Param>> params) { public Category setParams(final List<Param> params) {
this.params = params; this.params = params;
return this; return this;
} }

7
libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/context/Concept.java
View File

@ -1,7 +1,6 @@
package eu.dnetlib.openaire.exporter.model.context; package eu.dnetlib.openaire.exporter.model.context;
import java.util.List; import java.util.List;
import java.util.Map;
public class Concept { public class Concept {
@ -11,7 +10,7 @@ public class Concept {
private boolean claim; private boolean claim;
private Map<String, List<Param>> params; private List<Param> params;
private List<Concept> concepts; private List<Concept> concepts;
@ -31,7 +30,7 @@ public class Concept {
return claim; return claim;
} }
public Map<String, List<Param>> getParams() { public List<Param> getParams() {
return params; return params;
} }
@ -54,7 +53,7 @@ public class Concept {
return this; return this;
} }
public Concept setParams(final Map<String, List<Param>> params) { public Concept setParams(final List<Param> params) {
this.params = params; this.params = params;
return this; return this;
} }

6
libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/context/Context.java
View File

@ -16,7 +16,7 @@ public class Context {
private Date lastUpdateDate; private Date lastUpdateDate;
private Map<String, List<Param>> params; private List<Param> params;
private Map<String, Category> categories; private Map<String, Category> categories;
@ -36,7 +36,7 @@ public class Context {
return creationDate; return creationDate;
} }
public Map<String, List<Param>> getParams() { public List<Param> getParams() {
return params; return params;
} }
@ -73,7 +73,7 @@ public class Context {
return this; return this;
} }
public Context setParams(final Map<String, List<Param>> params) { public Context setParams(final List<Param> params) {
this.params = params; this.params = params;
return this; return this;
} }