D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

updated public logs api

This commit is contained in:
Michele Artini 2023-12-14 11:10:43 +01:00
parent e4cbfe863b
commit a0e7425379
6 changed files with 184 additions and 244 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/PublicApiController.java
View File

@ -1,35 +1,53 @@
package eu.dnetlib.organizations.controller; package eu.dnetlib.organizations.controller;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.io.IOUtils;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page; import org.springframework.http.HttpStatus;
import org.springframework.data.domain.PageRequest; import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.organizations.model.view.ApiJournalView; import eu.dnetlib.organizations.utils.DatabaseUtils;
import eu.dnetlib.organizations.repository.OrganizationRepository;
import eu.dnetlib.organizations.repository.readonly.ApiJournalViewRepository;
@RestController @RestController
@RequestMapping("/public-api") @RequestMapping("/public-api")
public class PublicApiController { public class PublicApiController {
@Autowired @Autowired
private ApiJournalViewRepository apiJournalViewRepository; private DatabaseUtils dbUtils;
@Autowired @GetMapping("/logs/{year}/{month}/{country}")
private OrganizationRepository organizationRepository; public void findJournalByCountry(@PathVariable final int year,
@PathVariable final int month,
@PathVariable final String country,
final HttpServletResponse res) throws IOException {
if (month < 1 || month > 12) {
res.sendError(HttpStatus.BAD_REQUEST.value(), "Invalid month");
}
if (year < 2020 || year > 2100) {
res.sendError(HttpStatus.BAD_REQUEST.value(), "Invalid year");
}
res.setContentType(MediaType.TEXT_PLAIN.getType());
final ServletOutputStream out = res.getOutputStream();
dbUtils.obtainLogEntries(year, month, country).forEach(s -> {
try {
IOUtils.write(s, out, StandardCharsets.UTF_8);
} catch (final IOException e) {
throw new RuntimeException(e);
}
});
@GetMapping("/logs")
public ApiJournalView findJournalByDsId(@RequestParam final String id) {
return apiJournalViewRepository.findById(id).orElse(organizationRepository.findById(id).map(ApiJournalView::new).orElse(new ApiJournalView()));
} }
@GetMapping("/logs/{country}/{page}/{size}")
public Page<ApiJournalView> findJournalByCountry(@PathVariable final String country, @PathVariable final int page, @PathVariable final int size) {
return apiJournalViewRepository.findByCountry(country, PageRequest.of(page, size));
}
} }

37
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/utils/ApiOperation.java
View File

@ -1,37 +0,0 @@
package eu.dnetlib.organizations.model.utils;
import java.io.Serializable;
import java.time.LocalDateTime;
public class ApiOperation implements Serializable {
private static final long serialVersionUID = -7111524441502889608L;
private String operation;
private String description;
private LocalDateTime date;
public String getOperation() {
return operation;
}
public void setOperation(final String operation) {
this.operation = operation;
}
public String getDescription() {
return description;
}
public void setDescription(final String description) {
this.description = description;
}
public LocalDateTime getDate() {
return date;
}
public void setDate(final LocalDateTime date) {
this.date = date;
}
}

87
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/ApiJournalView.java
View File

@ -1,87 +0,0 @@
package eu.dnetlib.organizations.model.view;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Table;
import org.hibernate.annotations.Type;
import org.hibernate.annotations.TypeDef;
import org.hibernate.annotations.TypeDefs;
import com.vladmihalcea.hibernate.type.json.JsonBinaryType;
import com.vladmihalcea.hibernate.type.json.JsonStringType;
import eu.dnetlib.organizations.model.Organization;
import eu.dnetlib.organizations.model.utils.ApiOperation;
@Entity
@Table(name = "api_journal_view")
@TypeDefs({
@TypeDef(name = "json", typeClass = JsonStringType.class),
@TypeDef(name = "jsonb", typeClass = JsonBinaryType.class)
})
public class ApiJournalView implements Serializable {
private static final long serialVersionUID = 1270660185726854334L;
@Id
@Column(name = "id")
private String id;
@Column(name = "name")
private String name;
@Column(name = "country")
private String country;
@Type(type = "json")
@Column(name = "logs", columnDefinition = "json")
private List<ApiOperation> logs;
public ApiJournalView() {}
public ApiJournalView(final Organization o) {
id = o.getId();
name = o.getName();
country = o.getCountry();
logs = new ArrayList<>();
}
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(final String name) {
this.name = name;
}
public String getCountry() {
return country;
}
public void setCountry(final String country) {
this.country = country;
}
public List<ApiOperation> getLogs() {
return logs;
}
public void setLogs(final List<ApiOperation> logs) {
this.logs = logs;
}
}

14
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/readonly/ApiJournalViewRepository.java
View File

@ -1,14 +0,0 @@
package eu.dnetlib.organizations.repository.readonly;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Repository;
import eu.dnetlib.organizations.model.view.ApiJournalView;
@Repository
public interface ApiJournalViewRepository extends ReadOnlyRepository<ApiJournalView, String> {
Page<ApiJournalView> findByCountry(String country, Pageable page);
}

229
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java
View File

@ -1,5 +1,6 @@
package eu.dnetlib.organizations.utils; package eu.dnetlib.organizations.utils;
import java.io.StringWriter;
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
@ -104,12 +105,7 @@ public class DatabaseUtils {
private static final Log log = LogFactory.getLog(DatabaseUtils.class); private static final Log log = LogFactory.getLog(DatabaseUtils.class);
public enum VocabularyTable { public enum VocabularyTable {
languages, languages, countries, org_types, id_types, rel_types, simrel_types
countries,
org_types,
id_types,
rel_types,
simrel_types
} }
@Transactional @Transactional
@ -118,8 +114,8 @@ public class DatabaseUtils {
final String oldId = StringUtils.isNotBlank(orgView.getId()) ? new String(orgView.getId()) : null; final String oldId = StringUtils.isNotBlank(orgView.getId()) ? new String(orgView.getId()) : null;
final String oldStatus = oldId != null ? organizationRepository.findById(oldId) final String oldStatus = oldId != null ? organizationRepository.findById(oldId)
.map(Organization::getStatus) .map(Organization::getStatus)
.orElse(null) : null; .orElse(null) : null;
final boolean alreadyApproved = StringUtils.equals(oldStatus, OrganizationStatus.approved.toString()); final boolean alreadyApproved = StringUtils.equals(oldStatus, OrganizationStatus.approved.toString());
@ -146,15 +142,15 @@ public class DatabaseUtils {
} }
final Organization org = new Organization(orgView.getId(), final Organization org = new Organization(orgView.getId(),
orgView.getName(), orgView.getName(),
orgView.getType(), orgView.getType(),
orgView.getLat(), orgView.getLng(), orgView.getLat(), orgView.getLng(),
orgView.getCity(), orgView.getCountry(), orgView.getCity(), orgView.getCountry(),
newStatus, newStatus,
orgView.getEcLegalBody(), orgView.getEcLegalBody(),
orgView.getEcLegalPerson(), orgView.getEcNonProfit(), orgView.getEcResearchOrganization(), orgView.getEcHigherEducation(), orgView.getEcLegalPerson(), orgView.getEcNonProfit(), orgView.getEcResearchOrganization(), orgView.getEcHigherEducation(),
orgView.getEcInternationalOrganizationEurInterests(), orgView.getEcInternationalOrganization(), orgView.getEcEnterprise(), orgView.getEcInternationalOrganizationEurInterests(), orgView.getEcInternationalOrganization(), orgView.getEcEnterprise(),
orgView.getEcSmeValidated(), orgView.getEcNutscode()); orgView.getEcSmeValidated(), orgView.getEcNutscode());
final String newId = organizationRepository.save(org).getId(); final String newId = organizationRepository.save(org).getId();
@ -168,9 +164,9 @@ public class DatabaseUtils {
if (oldId != null) { if (oldId != null) {
final List<OpenaireDuplicate> dups = openaireDuplicateRepository.findByLocalId(oldId) final List<OpenaireDuplicate> dups = openaireDuplicateRepository.findByLocalId(oldId)
.stream() .stream()
.map(d -> prepareNewDuplicate(newId, oldId, d)) .map(d -> prepareNewDuplicate(newId, oldId, d))
.collect(Collectors.toList()); .collect(Collectors.toList());
openaireDuplicateRepository.saveAll(dups); openaireDuplicateRepository.saveAll(dups);
@ -194,7 +190,7 @@ public class DatabaseUtils {
if (oldStatus == null) { if (oldStatus == null) {
op = JournalOperations.NEW_SUGG_ORG; op = JournalOperations.NEW_SUGG_ORG;
message = "Created a new suggested org"; message = "Created a new suggested org";
} else if (oldStatus != null) { } else {
op = JournalOperations.EDIT_SUGG_ORG; op = JournalOperations.EDIT_SUGG_ORG;
message = "Metadata updated"; message = "Metadata updated";
} }
@ -227,9 +223,9 @@ public class DatabaseUtils {
d.setOaCollectedFrom(old.getOaCollectedFrom()); d.setOaCollectedFrom(old.getOaCollectedFrom());
if (oldId != null if (oldId != null
&& newId.startsWith(OpenOrgsConstants.OPENORGS_PREFIX) && newId.startsWith(OpenOrgsConstants.OPENORGS_PREFIX)
&& oldId.startsWith(OpenOrgsConstants.OPENORGS_PENDING_PREFIX) && oldId.startsWith(OpenOrgsConstants.OPENORGS_PENDING_PREFIX)
&& StringUtils.substringAfter(oldId, OpenOrgsConstants.OPENORGS_PENDING_PREFIX).equalsIgnoreCase(DigestUtils.md5Hex(d.getOaOriginalId()))) { && StringUtils.substringAfter(oldId, OpenOrgsConstants.OPENORGS_PENDING_PREFIX).equalsIgnoreCase(DigestUtils.md5Hex(d.getOaOriginalId()))) {
d.setRelType(SimilarityType.is_similar.toString()); d.setRelType(SimilarityType.is_similar.toString());
} else { } else {
d.setRelType(SimilarityType.suggested.toString()); d.setRelType(SimilarityType.suggested.toString());
@ -276,16 +272,16 @@ public class DatabaseUtils {
// Updating journal // Updating journal
toSave.stream().collect(Collectors.groupingBy(OpenaireDuplicate::getLocalId)).forEach((id, list) -> { toSave.stream().collect(Collectors.groupingBy(OpenaireDuplicate::getLocalId)).forEach((id, list) -> {
final long sim = list.stream() final long sim = list.stream()
.filter(d -> d.getRelType().equals(SimilarityType.is_similar.toString())) .filter(d -> d.getRelType().equals(SimilarityType.is_similar.toString()))
.count(); .count();
final long diff = list.stream() final long diff = list.stream()
.filter(d -> d.getRelType().equals(SimilarityType.is_different.toString())) .filter(d -> d.getRelType().equals(SimilarityType.is_different.toString()))
.count(); .count();
final long sugg = list.stream() final long sugg = list.stream()
.filter(d -> d.getRelType().equals(SimilarityType.suggested.toString())) .filter(d -> d.getRelType().equals(SimilarityType.suggested.toString()))
.count(); .count();
final String message = String.format("Duplicates updated (%s similars, %s differents, %s suggested)", sim, diff, sugg); final String message = String.format("Duplicates updated (%s similars, %s differents, %s suggested)", sim, diff, sugg);
@ -300,7 +296,7 @@ public class DatabaseUtils {
// delete invalid pending orgs (without simrels) // delete invalid pending orgs (without simrels)
final int n = jdbcTemplate final int n = jdbcTemplate
.update("delete from organizations where id in (select o.id from organizations o left outer join oa_duplicates d on (o.id = d.local_id) where o.status = 'suggested' and o.created_by = 'dedupWf' group by o.id having count(d.local_id) = 0)"); .update("delete from organizations where id in (select o.id from organizations o left outer join oa_duplicates d on (o.id = d.local_id) where o.status = 'suggested' and o.created_by = 'dedupWf' group by o.id having count(d.local_id) = 0)");
if (n > 0) { if (n > 0) {
log.info("Invalid pending orgs deleted: " + n); log.info("Invalid pending orgs deleted: " + n);
@ -335,12 +331,12 @@ public class DatabaseUtils {
@Cacheable("countries_for_user") @Cacheable("countries_for_user")
public List<VocabularyTerm> listCountriesForUser(final String name) { public List<VocabularyTerm> listCountriesForUser(final String name) {
final String sql = final String sql =
"select uc.country as value, c.name as name from user_countries uc left outer join countries c on (c.val = uc.country) where uc.email = ? order by c.name"; "select uc.country as value, c.name as name from user_countries uc left outer join countries c on (c.val = uc.country) where uc.email = ? order by c.name";
return jdbcTemplate.query(sql, new BeanPropertyRowMapper<>(VocabularyTerm.class), name); return jdbcTemplate.query(sql, new BeanPropertyRowMapper<>(VocabularyTerm.class), name);
} }
@CacheEvict(value = { @CacheEvict(value = {
"vocs", "countries_for_user" "vocs", "countries_for_user"
}, allEntries = true) }, allEntries = true)
public void clearCache() { public void clearCache() {
log.info("All caches cleaned"); log.info("All caches cleaned");
@ -359,7 +355,7 @@ public class DatabaseUtils {
userCountryRepository.deleteByEmail(userView.getEmail()); userCountryRepository.deleteByEmail(userView.getEmail());
if (userView.getCountries() != null) { if (userView.getCountries() != null) {
userCountryRepository userCountryRepository
.saveAll(Arrays.stream(userView.getCountries()).map(c -> new UserCountry(userView.getEmail(), c)).collect(Collectors.toList())); .saveAll(Arrays.stream(userView.getCountries()).map(c -> new UserCountry(userView.getEmail(), c)).collect(Collectors.toList()));
} }
} }
@ -371,11 +367,11 @@ public class DatabaseUtils {
@Transactional @Transactional
public void newUser(final String email, public void newUser(final String email,
final String fullname, final String fullname,
final String organization, final String organization,
final String referencePerson, final String referencePerson,
final String requestMessage, final String requestMessage,
final List<String> countries) { final List<String> countries) {
final User user = new User(); final User user = new User();
user.setEmail(email); user.setEmail(email);
@ -405,32 +401,32 @@ public class DatabaseUtils {
// BROWSE BY COUNTRY // BROWSE BY COUNTRY
public List<BrowseEntry> browseCountries() { public List<BrowseEntry> browseCountries() {
final String sql = final String sql =
"select o.country as code, c.name as name, o.status as group, count(o.status) as count from organizations o left outer join countries c on (o.country = c.val) group by o.country, c.name, o.status"; "select o.country as code, c.name as name, o.status as group, count(o.status) as count from organizations o left outer join countries c on (o.country = c.val) group by o.country, c.name, o.status";
return listBrowseEntries(sql); return listBrowseEntries(sql);
} }
// BROWSE BY COUNTRY FOR USER // BROWSE BY COUNTRY FOR USER
public List<BrowseEntry> browseCountriesForUser(final String email) { public List<BrowseEntry> browseCountriesForUser(final String email) {
final String sql = final String sql =
"select o.country as code, c.name as name, o.status as group, count(o.status) as count from user_countries uc left outer join organizations o on (uc.country = o.country) left outer join countries c on (o.country = c.val) where uc.email=? group by o.country, c.name, o.status"; "select o.country as code, c.name as name, o.status as group, count(o.status) as count from user_countries uc left outer join organizations o on (uc.country = o.country) left outer join countries c on (o.country = c.val) where uc.email=? group by o.country, c.name, o.status";
return listBrowseEntries(sql, email); return listBrowseEntries(sql, email);
} }
// BROWSE BY ORG TYPE // BROWSE BY ORG TYPE
public List<BrowseEntry> browseTypes() { public List<BrowseEntry> browseTypes() {
final String sql = final String sql =
"select type as code, type as name, status as group, count(status) as count from organizations group by type, status"; "select type as code, type as name, status as group, count(status) as count from organizations group by type, status";
return listBrowseEntries(sql); return listBrowseEntries(sql);
} }
// BROWSE BY ORG TYPE FOR USER // BROWSE BY ORG TYPE FOR USER
public List<BrowseEntry> browseTypesForUser(final String email) { public List<BrowseEntry> browseTypesForUser(final String email) {
final String sql = "select o.type as code, o.type as name," final String sql = "select o.type as code, o.type as name,"
+ "o.status as group, count(o.status) as count " + "o.status as group, count(o.status) as count "
+ "from organizations o " + "from organizations o "
+ "left outer join user_countries uc on (uc.country = o.country) " + "left outer join user_countries uc on (uc.country = o.country) "
+ "where uc.email=? " + "where uc.email=? "
+ "group by o.type, o.status"; + "group by o.type, o.status";
return listBrowseEntries(sql, email); return listBrowseEntries(sql, email);
} }
@ -454,7 +450,7 @@ public class DatabaseUtils {
public List<OrganizationConflict> listConflictsForId(final String id) { public List<OrganizationConflict> listConflictsForId(final String id) {
final String sql = final String sql =
"select o.id, o.name, o.type, o.city, o.country from oa_conflicts c left outer join organizations o on (c.id2 = o.id) where o.id is not null and c.id1 = ? and c.reltype = 'suggested'"; "select o.id, o.name, o.type, o.city, o.country from oa_conflicts c left outer join organizations o on (c.id2 = o.id) where o.id is not null and c.id1 = ? and c.reltype = 'suggested'";
return jdbcTemplate.query(sql, new BeanPropertyRowMapper<>(OrganizationConflict.class), id); return jdbcTemplate.query(sql, new BeanPropertyRowMapper<>(OrganizationConflict.class), id);
} }
@ -479,15 +475,14 @@ public class DatabaseUtils {
public String fixConflictSimilars(final List<String> similarIds, final String user) { public String fixConflictSimilars(final List<String> similarIds, final String user) {
final List<OrganizationView> views = final List<OrganizationView> views =
similarIds.stream().map(organizationViewRepository::findById).filter(Optional::isPresent).map(Optional::get).collect(Collectors.toList()); similarIds.stream().map(organizationViewRepository::findById).filter(Optional::isPresent).map(Optional::get).collect(Collectors.toList());
final List<OrganizationView> persistents = views.stream().filter(v -> v.getPersistent()).collect(Collectors.toList()); final List<OrganizationView> persistents = views.stream().filter(OrganizationView::getPersistent).collect(Collectors.toList());
final OrganizationView masterOrg = new OrganizationView(); final OrganizationView masterOrg = new OrganizationView();
if (persistents.size() > 1) { if (persistents.size() > 1) { throw new RuntimeException("Too many persintent organizations"); }
throw new RuntimeException("Too many persintent organizations"); if (persistents.size() == 1) {
} else if (persistents.size() == 1) {
backupOrg(persistents.get(0), user); backupOrg(persistents.get(0), user);
masterOrg.setId(persistents.get(0).getId()); masterOrg.setId(persistents.get(0).getId());
masterOrg.setStatus(OrganizationStatus.approved.toString()); masterOrg.setStatus(OrganizationStatus.approved.toString());
@ -513,9 +508,9 @@ public class DatabaseUtils {
organizationRepository.updateStatus(backupId, OrganizationStatus.hidden.toString()); organizationRepository.updateStatus(backupId, OrganizationStatus.hidden.toString());
journalEntryRepository journalEntryRepository
.save(new JournalEntry(origId, JournalOperations.BACKUP_ORG, "Saved a backup copy: " + backupId, user)); .save(new JournalEntry(origId, JournalOperations.BACKUP_ORG, "Saved a backup copy: " + backupId, user));
journalEntryRepository journalEntryRepository
.save(new JournalEntry(backupId, JournalOperations.BACKUP_ORG, "Saved a backup copy of " + origId, user)); .save(new JournalEntry(backupId, JournalOperations.BACKUP_ORG, "Saved a backup copy of " + origId, user));
return backupId; return backupId;
} catch (final Exception e) { } catch (final Exception e) {
@ -529,9 +524,9 @@ public class DatabaseUtils {
final OffsetDateTime now = OffsetDateTime.now(); final OffsetDateTime now = OffsetDateTime.now();
final String finalMessage = (masterOrg.getId() == null ? "New org created merging: " : "Merging in persistent org: ") + final String finalMessage = (masterOrg.getId() == null ? "New org created merging: " : "Merging in persistent org: ") +
orgs.stream() orgs.stream()
.map(OrganizationView::getId) .map(OrganizationView::getId)
.collect(Collectors.joining(", ")); .collect(Collectors.joining(", "));
masterOrg.setName(findFirstString(orgs, OrganizationView::getName)); masterOrg.setName(findFirstString(orgs, OrganizationView::getName));
masterOrg.setType(findFirstString(orgs, OrganizationView::getType)); masterOrg.setType(findFirstString(orgs, OrganizationView::getType));
@ -544,35 +539,35 @@ public class DatabaseUtils {
masterOrg.setAcronyms(findAll(orgs, OrganizationView::getAcronyms)); masterOrg.setAcronyms(findAll(orgs, OrganizationView::getAcronyms));
masterOrg.setUrls(findAll(orgs, OrganizationView::getUrls)); masterOrg.setUrls(findAll(orgs, OrganizationView::getUrls));
masterOrg.setRelations(findAll(orgs, OrganizationView::getRelations, r -> !r.getType().equals(RelationType.Merged_In.toString()) masterOrg.setRelations(findAll(orgs, OrganizationView::getRelations, r -> !r.getType().equals(RelationType.Merged_In.toString())
&& !r.getType().equals(RelationType.Merged_In.toString()))); && !r.getType().equals(RelationType.Merged_In.toString())));
masterOrg.getOtherNames() masterOrg.getOtherNames()
.addAll(orgs.stream() .addAll(orgs.stream()
.map(OrganizationView::getName) .map(OrganizationView::getName)
.filter(StringUtils::isNotBlank) .filter(StringUtils::isNotBlank)
.filter(s -> StringUtils.equalsIgnoreCase(s, masterOrg.getName())) .filter(s -> StringUtils.equalsIgnoreCase(s, masterOrg.getName()))
.map(s -> new eu.dnetlib.organizations.model.view.OtherName(s, "UNKNOWN")) .map(s -> new eu.dnetlib.organizations.model.view.OtherName(s, "UNKNOWN"))
.collect(Collectors.toList())); .collect(Collectors.toList()));
final String masterId = insertOrUpdateOrganization(masterOrg, user, false); final String masterId = insertOrUpdateOrganization(masterOrg, user, false);
// I hide the merged organizations // I hide the merged organizations
orgs.stream() orgs.stream()
.map(OrganizationView::getId) .map(OrganizationView::getId)
.filter(id -> !id.equals(masterId)) .filter(id -> !id.equals(masterId))
.forEach(id -> { .forEach(id -> {
hideConflictOrgs(masterId, id); hideConflictOrgs(masterId, id);
journalEntryRepository journalEntryRepository
.save(new JournalEntry(id, JournalOperations.FIX_CONFLICT, "The org has been hidded and merged in " + masterId, user)); .save(new JournalEntry(id, JournalOperations.FIX_CONFLICT, "The org has been hidded and merged in " + masterId, user));
}); });
// I reassign the duplicates to the new org // I reassign the duplicates to the new org
final List<OpenaireDuplicate> newDuplicates = orgs.stream() final List<OpenaireDuplicate> newDuplicates = orgs.stream()
.map(OrganizationView::getId) .map(OrganizationView::getId)
.map(openaireDuplicateRepository::findByLocalId) .map(openaireDuplicateRepository::findByLocalId)
.flatMap(l -> l.stream()) .flatMap(List::stream)
.map(d -> new OpenaireDuplicate(masterId, d.getOaOriginalId(), d.getRelType(), d.getOaCollectedFrom())) .map(d -> new OpenaireDuplicate(masterId, d.getOaOriginalId(), d.getRelType(), d.getOaCollectedFrom()))
.collect(Collectors.toList()); .collect(Collectors.toList());
openaireDuplicateRepository.saveAll(newDuplicates); openaireDuplicateRepository.saveAll(newDuplicates);
newDuplicates.forEach(d -> { newDuplicates.forEach(d -> {
@ -588,12 +583,12 @@ public class DatabaseUtils {
for (int i = 0; i < orgs.size(); i++) { for (int i = 0; i < orgs.size(); i++) {
for (int j = i + 1; j < orgs.size(); j++) { for (int j = i + 1; j < orgs.size(); j++) {
openaireConflictRepository openaireConflictRepository
.updateStatusAndResetGroup(orgs.get(i).getId(), orgs.get(j).getId(), SimilarityType.is_similar.toString(), user, now); .updateStatusAndResetGroup(orgs.get(i).getId(), orgs.get(j).getId(), SimilarityType.is_similar.toString(), user, now);
} }
} }
journalEntryRepository journalEntryRepository
.save(new JournalEntry(masterId, JournalOperations.FIX_CONFLICT, finalMessage, user)); .save(new JournalEntry(masterId, JournalOperations.FIX_CONFLICT, finalMessage, user));
return masterId; return masterId;
} }
@ -608,7 +603,7 @@ public class DatabaseUtils {
for (int i = 0; i < differentsIds.size(); i++) { for (int i = 0; i < differentsIds.size(); i++) {
for (int j = i + 1; j < differentsIds.size(); j++) { for (int j = i + 1; j < differentsIds.size(); j++) {
openaireConflictRepository openaireConflictRepository
.updateStatusAndResetGroup(differentsIds.get(i), differentsIds.get(j), SimilarityType.is_different.toString(), user, now); .updateStatusAndResetGroup(differentsIds.get(i), differentsIds.get(j), SimilarityType.is_different.toString(), user, now);
} }
journalEntryRepository.save(new JournalEntry(differentsIds.get(i), JournalOperations.NO_CONFLICT, message, user)); journalEntryRepository.save(new JournalEntry(differentsIds.get(i), JournalOperations.NO_CONFLICT, message, user));
} }
@ -633,11 +628,11 @@ public class DatabaseUtils {
} }
private <T> Set<T> findAll(final List<OrganizationView> views, final Function<OrganizationView, Set<T>> mapper) { private <T> Set<T> findAll(final List<OrganizationView> views, final Function<OrganizationView, Set<T>> mapper) {
return views.stream().map(mapper).flatMap(s -> s.stream()).collect(Collectors.toCollection(LinkedHashSet::new)); return views.stream().map(mapper).flatMap(Set::stream).collect(Collectors.toCollection(LinkedHashSet::new));
} }
private <T> Set<T> findAll(final List<OrganizationView> views, final Function<OrganizationView, Set<T>> mapper, final Predicate<T> filter) { private <T> Set<T> findAll(final List<OrganizationView> views, final Function<OrganizationView, Set<T>> mapper, final Predicate<T> filter) {
return views.stream().map(mapper).flatMap(s -> s.stream()).filter(filter).collect(Collectors.toCollection(LinkedHashSet::new)); return views.stream().map(mapper).flatMap(Set::stream).filter(filter).collect(Collectors.toCollection(LinkedHashSet::new));
} }
private List<Relationship> hideConflictOrgs(final String masterId, final String otherId) { private List<Relationship> hideConflictOrgs(final String masterId, final String otherId) {
@ -649,7 +644,7 @@ public class DatabaseUtils {
public List<String> invalidCountriesInSuggestions() { public List<String> invalidCountriesInSuggestions() {
final String sql = final String sql =
" select distinct t.oa_country from tmp_dedup_events t left outer join countries c on (t.oa_country = c.val) where c.val is null order by t.oa_country"; " select distinct t.oa_country from tmp_dedup_events t left outer join countries c on (t.oa_country = c.val) where c.val is null order by t.oa_country";
return jdbcTemplate.queryForList(sql, String.class); return jdbcTemplate.queryForList(sql, String.class);
} }
@ -665,27 +660,81 @@ public class DatabaseUtils {
if (id.length() == 46) { if (id.length() == 46) {
final Optional<OrganizationView> orgView = organizationViewRepository.findByOpenaireId(id); final Optional<OrganizationView> orgView = organizationViewRepository.findByOpenaireId(id);
valid = orgView.map(OrganizationView::getStatus) valid = orgView.map(OrganizationView::getStatus)
.filter(s -> s.equals(OrganizationStatus.approved.toString())) .filter(s -> s.equals(OrganizationStatus.approved.toString()))
.isPresent(); .isPresent();
ooid = orgView.get().getId(); ooid = orgView.get().getId();
} else { } else {
valid = organizationRepository.findById(id) valid = organizationRepository.findById(id)
.map(Organization::getStatus) .map(Organization::getStatus)
.filter(s -> s.equals(OrganizationStatus.approved.toString())) .filter(s -> s.equals(OrganizationStatus.approved.toString()))
.isPresent(); .isPresent();
ooid = id; ooid = id;
} }
if (valid) { if (valid) {
persistentOrganizationRepository.save(new PersistentOrganization(ooid)); persistentOrganizationRepository.save(new PersistentOrganization(ooid));
return ooid; return ooid;
} else {
throw new RuntimeException("The ID does not refer to an approved Organization");
} }
throw new RuntimeException("The ID does not refer to an approved Organization");
} }
public void deletePersistentOrgs(final String id) { public void deletePersistentOrgs(final String id) {
persistentOrganizationRepository.deleteById(id); persistentOrganizationRepository.deleteById(id);
} }
public List<String> obtainLogEntries(final int year, final int month, final String country) {
final String query = "SELECT o.id, o.name, j.operation, j.description, date(j.op_date) as op_date "
+ "FROM organizations o JOIN journal j ON o.id = j.id "
+ "WHERE o.country=? AND extract(year FROM j.op_date)=? AND extract(month FROM j.op_date)=? "
+ "ORDER BY date(j.op_date), o,id";
return jdbcTemplate.queryForList(query, country, year, month)
.stream()
.map(this::asLogEntry)
.collect(Collectors.toList());
}
private String asLogEntry(final Map<String, Object> map) {
final JournalOperations op = JournalOperations.valueOf("" + map.get("operation"));
final StringWriter sw = new StringWriter();
sw.write("On %s a curator ");
switch (op) {
case APPROVE_SUGG_ORG:
sw.write("approved the suggested organisation %s (ID: %s)");
break;
case NEW_ORG:
sw.write("created the organisation %s (ID: %s)");
break;
case NEW_SUGG_ORG:
sw.write("suggested the organisation %s (ID: %s)");
break;
case EDIT_ORG:
sw.write("updated the metadata of %s (ID: %s)");
break;
case EDIT_SUGG_ORG:
sw.write("updated the metadata of the suggested organisation %s (ID: %s)");
break;
case DUPLICATES:
sw.write("updated the list of duplicates for %s (ID: %s)");
break;
case FIX_CONFLICT:
sw.write("chose as master the organisation %s (id: %s), which has been involved in a conflict resolution");
break;
case NO_CONFLICT:
sw.write("stated that the conflict where the organisation %s (id: %s) was involved was a false positive");
break;
case BACKUP_ORG:
sw.write("resolved a conflict and the organisation %s (id: %s) involved has been hidden, because another organisation has been chosen as master");
break;
case UNKNOWN:
default:
sw.write("performed an unknown operation on %s (ID: %s)");
}
sw.write(". \"%s\"\n");
return String.format(sw.toString(), map.get("op_date"), map.get("name"), map.get("id"), map.get("description"));
}
} }

11
apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql
View File

@ -749,4 +749,15 @@ $$;
CREATE TRIGGER insert_or_update_index_search_trigger AFTER INSERT OR UPDATE ON organizations FOR EACH ROW EXECUTE PROCEDURE insert_or_update_index_search_trigger(); CREATE TRIGGER insert_or_update_index_search_trigger AFTER INSERT OR UPDATE ON organizations FOR EACH ROW EXECUTE PROCEDURE insert_or_update_index_search_trigger();
CREATE TRIGGER delete_index_search_trigger BEFORE DELETE ON organizations FOR EACH ROW EXECUTE PROCEDURE delete_index_search(); CREATE TRIGGER delete_index_search_trigger BEFORE DELETE ON organizations FOR EACH ROW EXECUTE PROCEDURE delete_index_search();
-- PUBLIC VIEW
CREATE VIEW api_journal_view AS SELECT
o.id AS org_id,
o.name AS org_name,
o.country AS country,
j.operation AS operation,
j.description AS description,
j.op_date AS op_date
FROM organizations o JOIN journal j ON o.id = j.id;