D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

new_model_for_communities #15

Merged
michele.artini merged 81 commits from new_model_for_communities into master 2023-10-24 08:12:26 +02:00
Conversation 0 Commits 81 Files Changed 70 +1554 -96
10 changed files with 1554 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 15f0d8cea7 - Show all commits

93
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/importer/CommunityImporterService.java
View File

@ -247,88 +247,63 @@ public class CommunityImporterService {
details.setType(CommunityType.valueOf(c.getType()));
details.setMembership(CommunityMembershipType.open);
details.setClaim(CommunityClaimType.all);
final Map<String, List<Param>> params = c.getParams();
if (params.containsKey(CSUMMARY_DESCRIPTION)) {
details.setDescription(asCsv(params.get(CSUMMARY_DESCRIPTION)));
}
if (params.containsKey(CSUMMARY_LOGOURL)) {
details.setLogoUrl(asCsv(params.get(CSUMMARY_LOGOURL)));
}
if (params.containsKey(CSUMMARY_STATUS)) {
details.setStatus(CommunityStatus.valueOf(firstValue(params, CSUMMARY_STATUS)));
}
if (params.containsKey(CSUMMARY_NAME)) {
details.setName(asCsv(params.get(CSUMMARY_NAME)));
} else {
details.setDescription(asCsv(CSUMMARY_DESCRIPTION, c.getParams()));
details.setLogoUrl(asCsv(CSUMMARY_LOGOURL, c.getParams()));
details.setStatus(CommunityStatus.valueOf(firstValue(CSUMMARY_STATUS, c.getParams())));
details.setName(asCsv(CSUMMARY_NAME, c.getParams()));
details.setName(c.getLabel());
}
if (params.containsKey(CSUMMARY_ZENODOC)) {
details.setZenodoCommunity(asCsv(params.get(CSUMMARY_ZENODOC)));
}
if (params.containsKey(CPROFILE_SUBJECT)) {
details.setSubjects(splitValues(asValues(params.get(CPROFILE_SUBJECT)), CSV_DELIMITER));
}
if (params.containsKey(CPROFILE_FOS)) {
details.setFos(splitValues(asValues(params.get(CPROFILE_FOS)), CSV_DELIMITER));
}
if (params.containsKey(CPROFILE_SDG)) {
details.setSdg(splitValues(asValues(params.get(CPROFILE_SDG)), CSV_DELIMITER));
}
if (params.containsKey(CPROFILE_ADVANCED_CONSTRAINT)) {
details.setZenodoCommunity(asCsv(CSUMMARY_ZENODOC, c.getParams()));
details.setSubjects(splitValues(asValues(CPROFILE_SUBJECT, c.getParams()), CSV_DELIMITER));
details.setFos(splitValues(asValues(CPROFILE_FOS, c.getParams()), CSV_DELIMITER));
details.setSdg(splitValues(asValues(CPROFILE_SDG, c.getParams()), CSV_DELIMITER));
// In the map the string is the serialization of the json representing the selection criteria so it is a valid json
details.setAdvancedConstraints(SelectionCriteria.fromJson(asCsv(params.get(CPROFILE_ADVANCED_CONSTRAINT))));
}
if (params.containsKey(CPROFILE_REMOVE_CONSTRAINT)) {
details.setAdvancedConstraints(SelectionCriteria.fromJson(asCsv(CPROFILE_ADVANCED_CONSTRAINT, c.getParams())));
// In the map the string is the serialization of the json representing the selection criteria so it is a valid json
details.setRemoveConstraints(SelectionCriteria.fromJson(asCsv(params.get(CPROFILE_REMOVE_CONSTRAINT))));
}
if (params.containsKey(CPROFILE_CREATIONDATE)) {
details.setRemoveConstraints(SelectionCriteria.fromJson(asCsv(CPROFILE_REMOVE_CONSTRAINT, c.getParams())));
try {
details.setCreationDate(CommunityMappingUtils.asLocalDateTime(asCsv(params.get(CPROFILE_CREATIONDATE))));
details.setCreationDate(CommunityMappingUtils.asLocalDateTime(asCsv(CPROFILE_CREATIONDATE, c.getParams())));
} catch (final Exception e) {
log.debug("Exception on date format: " + e.getMessage());
}
}
return details;
}
private static CommunityProject asCommunityProject(final String communityId, final Concept c) {
final Map<String, List<Param>> p = c.getParams();
final List<Param> p = c.getParams();
final CommunityProject project = new CommunityProject();
project.setCommunityId(communityId);
project.setOpenaireId(firstValue(p, OPENAIRE_ID));
project.setFunder(firstValue(p, CPROJECT_FUNDER));
project.setGrantId(firstValue(p, CPROJECT_NUMBER));
project.setName(firstValue(p, CPROJECT_FULLNAME));
project.setAcronym(firstValue(p, CPROJECT_ACRONYM));
project.setOpenaireId(firstValue(OPENAIRE_ID, p));
project.setFunder(firstValue(CPROJECT_FUNDER, p));
project.setGrantId(firstValue(CPROJECT_NUMBER, p));
project.setName(firstValue(CPROJECT_FULLNAME, p));
project.setAcronym(firstValue(CPROJECT_ACRONYM, p));
return project;
}
private static CommunityContentprovider asCommunityDataprovider(final String communityId, final Concept c) {
final Map<String, List<Param>> p = c.getParams();
final List<Param> p = c.getParams();
final CommunityContentprovider d = new CommunityContentprovider();
d.setCommunityId(communityId);
d.setOpenaireId(firstValue(p, OPENAIRE_ID));
d.setName(firstValue(p, CCONTENTPROVIDER_NAME));
d.setOfficialname(firstValue(p, CCONTENTPROVIDER_OFFICIALNAME));
d.setSelectioncriteria(SelectionCriteria.fromJson(firstValue(p, CCONTENTPROVIDER_SELCRITERIA)));
d.setOpenaireId(firstValue(OPENAIRE_ID, p));
d.setName(firstValue(CCONTENTPROVIDER_NAME, p));
d.setOfficialname(firstValue(CCONTENTPROVIDER_OFFICIALNAME, p));
d.setSelectioncriteria(SelectionCriteria.fromJson(firstValue(CCONTENTPROVIDER_SELCRITERIA, p)));
return d;
}
private static CommunityOrganization asCommunityOrganization(final String id, final Concept c) {
final Map<String, List<Param>> p = c.getParams();
final List<Param> p = c.getParams();
final CommunityOrganization o = new CommunityOrganization();
o.setCommunityId(id);
o.setName(firstValue(p, CORGANIZATION_NAME));
o.setLogo_url(getDecodedUrl(firstValue(p, CORGANIZATION_LOGOURL)));
o.setWebsite_url(getDecodedUrl(firstValue(p, CORGANIZATION_WEBSITEURL)));
o.setName(firstValue(CORGANIZATION_NAME, p));
o.setLogo_url(getDecodedUrl(firstValue(CORGANIZATION_LOGOURL, p)));
o.setWebsite_url(getDecodedUrl(firstValue(CORGANIZATION_WEBSITEURL, p)));
return o;
}
private static String asZenodoCommunity(final Concept c) {
return firstValue(c.getParams(), CZENODOCOMMUNITY_ID);
return firstValue(CZENODOCOMMUNITY_ID, c.getParams());
}
private static List<SubCommunity> asSubCommunities(final String communityId, final String parent, final String category, final List<Concept> concepts) {
@ -368,18 +343,20 @@ public class CommunityImporterService {
.collect(Collectors.toList());
}
private static String firstValue(final Map<String, List<Param>> p, final String paramName) {
return asValues(p.get(paramName)).findFirst().orElse(null);
private static String firstValue(final String name, final List<Param> params) {
return asValues(name, params).findFirst().orElse(null);
}
private static String asCsv(final List<Param> params) {
return asValues(params)
.collect(Collectors.joining(CSV_DELIMITER));
private static String asCsv(final String name, final List<Param> params) {
return asValues(name, params).collect(Collectors.joining(CSV_DELIMITER));
}
private static Stream<String> asValues(final List<Param> params) {
private static Stream<String> asValues(final String name, final List<Param> params) {
return params == null ? Stream.empty()
: params.stream()
.filter(p -> p != null)
.filter(p -> StringUtils.isNotBlank(p.getName()))
.filter(p -> p.getName().trim().equals(name.trim()))
.map(Param::getValue)
.map(StringUtils::trim)
.distinct();

9
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/community/model/DbSubCommunity.java
View File

@ -1,9 +1,8 @@
package eu.dnetlib.openaire.community.model;
import java.io.Serializable;
import java.util.LinkedHashMap;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import javax.persistence.Column;
import javax.persistence.Entity;
@ -46,7 +45,7 @@ public class DbSubCommunity implements Serializable {
@Type(type = "jsonb")
@Column(name = "params")
private Map<String, List<Param>> params = new LinkedHashMap<>();
private List<Param> params = new ArrayList<>();
@Column(name = "parent")
private String parent;
@ -86,11 +85,11 @@ public class DbSubCommunity implements Serializable {
this.category = category;
}
public Map<String, List<Param>> getParams() {
public List<Param> getParams() {
return params;
}
public void setParams(final Map<String, List<Param>> params) {
public void setParams(final List<Param> params) {
this.params = params;
}

8
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextApiCore.java
View File

@ -32,13 +32,19 @@ public class ContextApiCore {
private ISClient isClient;
public List<ContextSummary> listContexts(final List<String> type) throws ContextException {
return getContextMap(type).values()
.stream()
.map(c -> new ContextSummary()
.setId(c.getId())
.setType(c.getType())
.setLabel(c.getLabel())
.setStatus(c.getParams().containsKey("status") ? c.getParams().get("status").get(0).getValue() : ""))
.setStatus(c.getParams()
.stream()
.filter(p -> p.getName().equals("status"))
.map(p -> p.getValue())
.findFirst()
.orElse("")))
.collect(Collectors.toList());
}

13
apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/context/ContextMappingUtils.java
View File

@ -1,7 +1,6 @@
package eu.dnetlib.openaire.context;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
@ -103,18 +102,12 @@ public class ContextMappingUtils {
return BooleanUtils.toBooleanObject(StringUtils.isNotBlank(claim) ? claim : "false");
}
private static Map<String, List<Param>> parseParams(final Element e) {
private static List<Param> parseParams(final Element e) {
final List<Node> params = e.selectNodes("./param");
return params.stream()
.map(n -> (Element) n)
.map(p -> new Param()
.setName(p.attributeValue("name"))
.setValue(p.getTextTrim()))
.collect(Collectors.toMap(Param::getName, Lists::newArrayList, (p1, p2) -> {
final List<Param> p = new ArrayList<>(p1);
p.addAll(p2);
return p;
}));
.map(p -> new Param().setName(p.attributeValue("name")).setValue(p.getTextTrim()))
.collect(Collectors.toList());
}
public static FunderDetails asFunderDetails(final Context c) {

47
apps/dnet-exporter-api/src/test/java/eu/dnetlib/openaire/community/importer/CommunityImporterServiceTest.java
View File

@ -111,4 +111,51 @@ class CommunityImporterServiceTest {
}
@SuppressWarnings("unchecked")
@Test
public void testImportCommunityFetFp7() throws Exception {
final String profile = IOUtils.toString(getClass().getResourceAsStream("old_community_profile_fet-fp7.xml"), StandardCharsets.UTF_8.toString());
final Queue<Throwable> errors = new LinkedList<>();
final Context context = ContextMappingUtils.parseContext(profile, errors);
assertTrue(errors.isEmpty());
// Mockito.when(jdbcTemplate.queryForList(Mockito.anyString(), Mockito.any(Class.class),
// Mockito.anyString())).thenReturn(Arrays.asList("corda_______"));
importer.importCommunity(context);
final ArgumentCaptor<CommunityDetails> detailsCapture = ArgumentCaptor.forClass(CommunityDetails.class);
final ArgumentCaptor<CommunityProject> projectsCapture = ArgumentCaptor.forClass(CommunityProject.class);
final ArgumentCaptor<CommunityContentprovider> datasourcesCapture = ArgumentCaptor.forClass(CommunityContentprovider.class);
final ArgumentCaptor<CommunityOrganization> orgsCapture = ArgumentCaptor.forClass(CommunityOrganization.class);
final ArgumentCaptor<SubCommunity> subCommunitiesCapture = ArgumentCaptor.forClass(SubCommunity.class);
Mockito.verify(service, Mockito.times(1)).saveCommunity(detailsCapture.capture());
Mockito.verify(service, Mockito.times(1)).addCommunityProjects(Mockito.anyString(), projectsCapture.capture());
Mockito.verify(service, Mockito.times(1)).addCommunityContentProviders(Mockito.anyString(), datasourcesCapture.capture());
Mockito.verify(service, Mockito.times(1)).addCommunityOrganizations(Mockito.anyString(), orgsCapture.capture());
Mockito.verify(service, Mockito.times(1)).addSubCommunities(subCommunitiesCapture.capture());
final CommunityDetails details = detailsCapture.getValue();
assertEquals("fet-fp7", details.getId());
// System.out.println(details);
final List<CommunityProject> projects = projectsCapture.getAllValues();
assertEquals(0, projects.size());
// projects.forEach(System.out::println);
final List<CommunityContentprovider> datasources = datasourcesCapture.getAllValues();
assertEquals(0, datasources.size());
// datasources.forEach(System.out::println);
final List<CommunityOrganization> orgs = orgsCapture.getAllValues();
assertEquals(0, orgs.size());
// orgs.forEach(System.out::println);
final List<SubCommunity> subs = subCommunitiesCapture.getAllValues();
assertEquals(151, subs.size());
subs.forEach(System.out::println);
}
}

1439
apps/dnet-exporter-api/src/test/resources/eu/dnetlib/openaire/community/importer/old_community_profile_fet-fp7.xml Normal file
View File

File diff suppressed because it is too large Load Diff

9
libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/community/SubCommunity.java
View File

@ -1,8 +1,7 @@
package eu.dnetlib.openaire.exporter.model.community;
import java.util.LinkedHashMap;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@ -28,7 +27,7 @@ public class SubCommunity {
private String category;
@Schema(description = "the parameters of the subCommunity", required = true)
private Map<String, List<Param>> params = new LinkedHashMap<>();
private List<Param> params = new ArrayList<>();
@Schema(description = "it supports the claims", required = true)
private boolean claim = false;
@ -73,11 +72,11 @@ public class SubCommunity {
this.category = category;
}
public Map<String, List<Param>> getParams() {
public List<Param> getParams() {
return params;
}
public void setParams(final Map<String, List<Param>> map) {
public void setParams(final List<Param> map) {
this.params = map;
}

7
libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/context/Category.java
View File

@ -1,7 +1,6 @@
package eu.dnetlib.openaire.exporter.model.context;
import java.util.List;
import java.util.Map;
public class Category {
@ -11,7 +10,7 @@ public class Category {
private boolean claim;
private Map<String, List<Param>> params;
private List<Param> params;
private List<Concept> concepts;
@ -31,7 +30,7 @@ public class Category {
return getConcepts() != null && !getConcepts().isEmpty();
}
public Map<String, List<Param>> getParams() {
public List<Param> getParams() {
return params;
}
@ -54,7 +53,7 @@ public class Category {
return this;
}
public Category setParams(final Map<String, List<Param>> params) {
public Category setParams(final List<Param> params) {
this.params = params;
return this;
}

7
libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/context/Concept.java
View File

@ -1,7 +1,6 @@
package eu.dnetlib.openaire.exporter.model.context;
import java.util.List;
import java.util.Map;
public class Concept {
@ -11,7 +10,7 @@ public class Concept {
private boolean claim;
private Map<String, List<Param>> params;
private List<Param> params;
private List<Concept> concepts;
@ -31,7 +30,7 @@ public class Concept {
return claim;
}
public Map<String, List<Param>> getParams() {
public List<Param> getParams() {
return params;
}
@ -54,7 +53,7 @@ public class Concept {
return this;
}
public Concept setParams(final Map<String, List<Param>> params) {
public Concept setParams(final List<Param> params) {
this.params = params;
return this;
}

6
libs/dnet-exporter-model/src/main/java/eu/dnetlib/openaire/exporter/model/context/Context.java
View File

@ -16,7 +16,7 @@ public class Context {
private Date lastUpdateDate;
private Map<String, List<Param>> params;
private List<Param> params;
private Map<String, Category> categories;
@ -36,7 +36,7 @@ public class Context {
return creationDate;
}
public Map<String, List<Param>> getParams() {
public List<Param> getParams() {
return params;
}
@ -73,7 +73,7 @@ public class Context {
return this;
}
public Context setParams(final Map<String, List<Param>> params) {
public Context setParams(final List<Param> params) {
this.params = params;
return this;
}