From 4b8488efe1b510ee0a06d49e8cf8e8bd72b2b201 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Fri, 9 Oct 2020 16:00:12 +0200 Subject: [PATCH] moved some fields from oa_duplicate to organizations --- .../OpenaireInternalApiController.java | 6 +- .../controller/OrganizationController.java | 16 ++- .../model/OpenaireDuplicate.java | 55 --------- .../model/view/OpenaireDuplicateView.java | 113 ++++++++++++++++++ .../OpenaireDuplicateViewRepository.java | 15 +++ .../organizations/utils/DatabaseUtils.java | 16 +-- .../utils/OrganizationStatus.java | 3 +- .../organizations/utils/SimilarityType.java | 4 +- .../src/main/resources/sql/schema.sql | 24 +++- .../resources/static/resources/html/edit.html | 3 +- .../resources/html/forms/all_conflicts.html | 2 +- .../resources/html/forms/all_duplicates.html | 2 +- .../resources/html/forms/org_conflicts.html | 2 +- .../resources/html/forms/org_metadata.html | 18 ++- .../resources/html/forms/pending_orgs.html | 2 +- .../resources/html/parts/org_details.html | 2 +- .../html/parts/org_results_page.html | 2 +- 17 files changed, 197 insertions(+), 88 deletions(-) create mode 100644 apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/OpenaireDuplicateView.java create mode 100644 apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/readonly/OpenaireDuplicateViewRepository.java diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OpenaireInternalApiController.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OpenaireInternalApiController.java index 992a9d6d..65228be5 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OpenaireInternalApiController.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OpenaireInternalApiController.java @@ -27,13 +27,13 @@ public class OpenaireInternalApiController { private static final Log log = LogFactory.getLog(OpenaireInternalApiController.class); - @GetMapping("/import/simrels") - public List importSimRels(final HttpServletRequest req) { + @GetMapping("/import/dedupEvents") + public List importDedupEvents(final HttpServletRequest req) { if (req.getRemoteAddr().equals(httpsProxy)) { log.warn("Call received by blaklisted ip (https proxy): " + req.getRemoteAddr()); throw new RuntimeException("Call received by blaklisted ip (https proxy): " + req.getRemoteAddr()); } - new Thread(databaseUtils::importSimRels).run(); + new Thread(databaseUtils::importDedupEvents).run(); return Arrays.asList("Importing simrels (request from " + req.getRemoteAddr() + ") ..."); } } diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java index 2fe7fa02..1a8aaefe 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java @@ -31,13 +31,14 @@ import eu.dnetlib.organizations.model.utils.BrowseEntry; import eu.dnetlib.organizations.model.utils.OrganizationConflict; import eu.dnetlib.organizations.model.view.ConflictGroupView; import eu.dnetlib.organizations.model.view.DuplicateGroupView; +import eu.dnetlib.organizations.model.view.OpenaireDuplicateView; import eu.dnetlib.organizations.model.view.OrganizationInfoView; import eu.dnetlib.organizations.model.view.OrganizationSimpleView; import eu.dnetlib.organizations.model.view.OrganizationView; -import eu.dnetlib.organizations.repository.OpenaireDuplicateRepository; import eu.dnetlib.organizations.repository.UserCountryRepository; import eu.dnetlib.organizations.repository.readonly.ConflictGroupViewRepository; import eu.dnetlib.organizations.repository.readonly.DuplicateGroupViewRepository; +import eu.dnetlib.organizations.repository.readonly.OpenaireDuplicateViewRepository; import eu.dnetlib.organizations.repository.readonly.OrganizationInfoViewRepository; import eu.dnetlib.organizations.repository.readonly.OrganizationSimpleViewRepository; import eu.dnetlib.organizations.repository.readonly.OrganizationViewRepository; @@ -55,7 +56,7 @@ public class OrganizationController { @Autowired private OrganizationSimpleViewRepository organizationSimpleViewRepository; @Autowired - private OpenaireDuplicateRepository openaireDuplicateRepository; + private OpenaireDuplicateViewRepository openaireDuplicateViewRepository; @Autowired private ConflictGroupViewRepository conflictGroupViewRepository; @Autowired @@ -127,9 +128,9 @@ public class OrganizationController { } @GetMapping("/duplicates") - public List duplicates(@RequestParam final String id, final Authentication authentication) { + public List duplicates(@RequestParam final String id, final Authentication authentication) { if (UserInfo.isSuperAdmin(authentication) || userCountryRepository.verifyAuthorizationForId(id, authentication.getName())) { - return openaireDuplicateRepository.findByLocalId(id); + return openaireDuplicateViewRepository.findByLocalId(id); } else { throw new RuntimeException("User not authorized"); } @@ -187,7 +188,9 @@ public class OrganizationController { } @PostMapping("/duplicates") - public List duplicates(@RequestBody final List simrels, final Authentication authentication) { + public List duplicates(@RequestBody final List simrels, final Authentication authentication) { + + if (simrels.isEmpty()) { return new ArrayList<>(); } final boolean b = UserInfo.isSuperAdmin(authentication) || simrels.stream() @@ -196,7 +199,8 @@ public class OrganizationController { .allMatch(id -> userCountryRepository.verifyAuthorizationForId(id, authentication.getName())); if (b) { - return databaseUtils.saveDuplicates(simrels, authentication.getName()); + databaseUtils.saveDuplicates(simrels, authentication.getName()); + return openaireDuplicateViewRepository.findByLocalId(simrels.get(0).getLocalId()); } else { throw new RuntimeException("User not authorized"); } diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/OpenaireDuplicate.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/OpenaireDuplicate.java index ccd6ae7a..de546cc0 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/OpenaireDuplicate.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/OpenaireDuplicate.java @@ -26,21 +26,6 @@ public class OpenaireDuplicate implements Serializable { @Column(name = "oa_original_id") private String oaOriginalId; - @Column(name = "oa_name") - private String oaName; - - @Column(name = "oa_acronym") - private String oaAcronym; - - @Column(name = "oa_country") - private String oaCountry; - - @Column(name = "oa_url") - private String oaUrl; - - @Column(name = "oa_collectedfrom") - private String oaCollectedFrom; - @Column(name = "reltype") private String relType; @@ -60,46 +45,6 @@ public class OpenaireDuplicate implements Serializable { this.oaOriginalId = oaOriginalId; } - public String getOaName() { - return oaName; - } - - public void setOaName(final String oaName) { - this.oaName = oaName; - } - - public String getOaAcronym() { - return oaAcronym; - } - - public void setOaAcronym(final String oaAcronym) { - this.oaAcronym = oaAcronym; - } - - public String getOaCountry() { - return oaCountry; - } - - public void setOaCountry(final String oaCountry) { - this.oaCountry = oaCountry; - } - - public String getOaUrl() { - return oaUrl; - } - - public void setOaUrl(final String oaUrl) { - this.oaUrl = oaUrl; - } - - public String getOaCollectedFrom() { - return oaCollectedFrom; - } - - public void setOaCollectedFrom(final String oaCollectedFrom) { - this.oaCollectedFrom = oaCollectedFrom; - } - public String getRelType() { return relType; } diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/OpenaireDuplicateView.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/OpenaireDuplicateView.java new file mode 100644 index 00000000..3fb24132 --- /dev/null +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/OpenaireDuplicateView.java @@ -0,0 +1,113 @@ +package eu.dnetlib.organizations.model.view; + +import java.io.Serializable; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.IdClass; +import javax.persistence.Table; + +import eu.dnetlib.organizations.model.OpenaireDuplicatePK; + +@Entity +@Table(name = "oa_duplicates_view") +@IdClass(OpenaireDuplicatePK.class) +public class OpenaireDuplicateView implements Serializable { + + /** + * + */ + private static final long serialVersionUID = 2673168234221945705L; + + @Id + @Column(name = "local_id") + private String localId; + + @Id + @Column(name = "oa_original_id") + private String oaOriginalId; + + @Column(name = "oa_name") + private String oaName; + + @Column(name = "oa_acronym") + private String oaAcronym; + + @Column(name = "oa_country") + private String oaCountry; + + @Column(name = "oa_url") + private String oaUrl; + + @Column(name = "oa_collectedfrom") + private String oaCollectedFrom; + + @Column(name = "reltype") + private String relType; + + public String getLocalId() { + return localId; + } + + public void setLocalId(final String localId) { + this.localId = localId; + } + + public String getOaOriginalId() { + return oaOriginalId; + } + + public void setOaOriginalId(final String oaOriginalId) { + this.oaOriginalId = oaOriginalId; + } + + public String getOaName() { + return oaName; + } + + public void setOaName(final String oaName) { + this.oaName = oaName; + } + + public String getOaAcronym() { + return oaAcronym; + } + + public void setOaAcronym(final String oaAcronym) { + this.oaAcronym = oaAcronym; + } + + public String getOaCountry() { + return oaCountry; + } + + public void setOaCountry(final String oaCountry) { + this.oaCountry = oaCountry; + } + + public String getOaUrl() { + return oaUrl; + } + + public void setOaUrl(final String oaUrl) { + this.oaUrl = oaUrl; + } + + public String getOaCollectedFrom() { + return oaCollectedFrom; + } + + public void setOaCollectedFrom(final String oaCollectedFrom) { + this.oaCollectedFrom = oaCollectedFrom; + } + + public String getRelType() { + return relType; + } + + public void setRelType(final String relType) { + this.relType = relType; + } + +} diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/readonly/OpenaireDuplicateViewRepository.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/readonly/OpenaireDuplicateViewRepository.java new file mode 100644 index 00000000..a48826cd --- /dev/null +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/readonly/OpenaireDuplicateViewRepository.java @@ -0,0 +1,15 @@ +package eu.dnetlib.organizations.repository.readonly; + +import java.util.List; + +import org.springframework.stereotype.Repository; + +import eu.dnetlib.organizations.model.OpenaireDuplicatePK; +import eu.dnetlib.organizations.model.view.OpenaireDuplicateView; + +@Repository +public interface OpenaireDuplicateViewRepository extends ReadOnlyRepository { + + List findByLocalId(String id); + +} diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java index ba5327b7..61af9b8e 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java @@ -14,7 +14,6 @@ import java.util.stream.Collectors; import javax.transaction.Transactional; -import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -134,12 +133,13 @@ public class DatabaseUtils { } @Transactional - public List saveDuplicates(final List simrels, final String email) { + public void saveDuplicates(final List simrels, final String email) { final OffsetDateTime now = OffsetDateTime.now(); final List list = openaireDuplicateRepository.saveAll(simrels); + list.forEach(d -> openaireDuplicateRepository.updateModificationDate(d.getLocalId(), d.getOaOriginalId(), email, now)); - return list; + } private void makeNewRelations(final String orgId, final OrganizationView orgView) { @@ -319,12 +319,12 @@ public class DatabaseUtils { } @Transactional - public void importSimRels() { + public void importDedupEvents() { try { - log.info("Importing conflicts and duplicates..."); - jdbcTemplate.update(IOUtils.toString(getClass().getResourceAsStream("/sql/importNewRels.sql"))); - log.info("...done"); - verifyConflictGroups(true); + // log.info("Importing conflicts and duplicates..."); + // jdbcTemplate.update(IOUtils.toString(getClass().getResourceAsStream("/sql/importNewRels.sql"))); + // log.info("...done"); + // verifyConflictGroups(true); } catch (final Exception e) { log.error("Error importing conflicts and duplicates", e); } diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/OrganizationStatus.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/OrganizationStatus.java index d97b922b..6ae54092 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/OrganizationStatus.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/OrganizationStatus.java @@ -5,5 +5,6 @@ public enum OrganizationStatus { approved, discarded, hidden, - deleted + deleted, + duplicate } diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/SimilarityType.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/SimilarityType.java index 29b65d82..e18f233f 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/SimilarityType.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/SimilarityType.java @@ -1,5 +1,7 @@ package eu.dnetlib.organizations.utils; public enum SimilarityType { - suggested, is_similar, is_different + suggested, + is_similar, + is_different } diff --git a/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql b/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql index d6843c71..116e328d 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql +++ b/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql @@ -381,7 +381,7 @@ CREATE INDEX urls_id_idx ON urls(id); CREATE TABLE oa_duplicates ( local_id text REFERENCES organizations(id) ON UPDATE CASCADE, - oa_original_id text NOT NULL, + oa_original_id text REFERENCES organizations(id) ON UPDATE CASCADE, oa_name text NOT NULL, oa_acronym text, oa_country text, @@ -396,6 +396,28 @@ CREATE TABLE oa_duplicates ( CREATE INDEX oa_duplicates_local_id_idx ON oa_duplicates(local_id); +CREATE VIEW oa_duplicates_view AS + d.local_id as local_id, + d.oa_original_id as oa_original_id, + o.name as oa_name, + array_to_string(array_agg(a.acronym), ', ') as oa_acronym, + o.country as oa_country, + array_to_string(array_agg(u.url), ', ') as oa_url, + d.oa_collectedfrom as oa_collectedfrom, + d.reltype as reltype +FROM + oa_duplicates d + LEFT OUTER JOIN organizations o ON (o.id = d.oa_original_id) + LEFT OUTER JOIN acronyms a ON (o.id = a.id) + LEFT OUTER JOIN urls u ON (o.id = u.id) +GROUP BY + d.local_id, + d.oa_original_id, + o.name, + o.country, + d.oa_collectedfrom, + d.reltype; + CREATE TABLE oa_conflicts ( id1 text REFERENCES organizations(id) ON UPDATE CASCADE, id2 text REFERENCES organizations(id) ON UPDATE CASCADE, diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/edit.html b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/edit.html index 0d56e173..c9a2038e 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/edit.html +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/edit.html @@ -12,6 +12,7 @@ + - + diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/forms/all_conflicts.html b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/forms/all_conflicts.html index 1077c4a1..49f72213 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/forms/all_conflicts.html +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/forms/all_conflicts.html @@ -25,7 +25,7 @@ #{{$index+1}} {{o.name}} - {{o.city}}, {{o.country}} + {{o.city || '-'}}, {{o.country}} {{o.type}} diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/forms/all_duplicates.html b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/forms/all_duplicates.html index 4055ab2b..84754aa5 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/forms/all_duplicates.html +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/forms/all_duplicates.html @@ -33,7 +33,7 @@ {{d.name}} - {{d.city}}, {{d.country}} + {{d.city || '-'}}, {{d.country}} {{d.numberOfDuplicates}} diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/forms/org_conflicts.html b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/forms/org_conflicts.html index 99dd8eff..199e7001 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/forms/org_conflicts.html +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/forms/org_conflicts.html @@ -15,7 +15,7 @@ {{sr.name}}
{{sr.id}} {{sr.type}} - {{sr.city}}, {{sr.country}} + {{sr.city || '-'}}, {{sr.country}} diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/forms/org_metadata.html b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/forms/org_metadata.html index d8ed4e96..f56d2b7f 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/forms/org_metadata.html +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/forms/org_metadata.html @@ -4,8 +4,12 @@
This is a pending organization. Please evaluate it before approving.
+
+ This organization is managed by the system. You can not edit. +
-
+ +
Official name and type
@@ -29,7 +33,7 @@
-
+
Geographical location
@@ -63,7 +67,7 @@
-
+
Other names and identifiers
@@ -229,7 +233,7 @@
-
+
Relations
@@ -261,7 +265,9 @@ + data-toggle="modal" data-target="#selectRelatedOrgModal" ng-hide="mode == 'readonly'"/> +