From e71bb3657abc203f78a7f08fc1f5678980c22f17 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 19 Apr 2021 16:00:29 +0200 Subject: [PATCH] new root org for all suggested orgs --- .../controller/OrganizationController.java | 36 ++-- .../organizations/model/OrganizationBase.java | 37 ++++ .../model/view/DuplicateSearchView.java | 158 ++++++++++++++++++ .../model/view/OrganizationSimpleView.java | 20 ++- .../OpenaireDuplicateRepository.java | 2 + .../repository/OrganizationRepository.java | 4 + .../DuplicateSearchViewRepository.java | 22 +++ .../organizations/utils/DatabaseUtils.java | 66 +++++--- .../utils/OpenOrgsConstants.java | 1 + .../main/resources/sql/importDedupEvents.sql | 70 ++++++-- .../src/main/resources/sql/schema.sql | 24 +++ .../html/pages/search/resultsByCountry.html | 3 +- .../html/pages/search/resultsByType.html | 3 +- .../html/pages/search/searchResults.html | 3 +- .../resources/html/parts/org_duplicates.html | 2 +- .../html/parts/org_results_page.html | 6 +- .../html/parts/select_org.modal.html | 3 +- .../static/resources/js/organizations.js | 3 +- 18 files changed, 407 insertions(+), 56 deletions(-) create mode 100644 apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/OrganizationBase.java create mode 100644 apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/DuplicateSearchView.java create mode 100644 apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/readonly/DuplicateSearchViewRepository.java diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java index f94c8945..79f23673 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java @@ -29,6 +29,7 @@ import eu.dnetlib.common.controller.AbstractDnetController; import eu.dnetlib.organizations.model.JournalEntry; import eu.dnetlib.organizations.model.Note; import eu.dnetlib.organizations.model.OpenaireDuplicate; +import eu.dnetlib.organizations.model.OrganizationBase; import eu.dnetlib.organizations.model.utils.BrowseEntry; import eu.dnetlib.organizations.model.utils.OrganizationConflict; import eu.dnetlib.organizations.model.view.ConflictGroupView; @@ -42,6 +43,7 @@ import eu.dnetlib.organizations.repository.NoteRepository; import eu.dnetlib.organizations.repository.UserCountryRepository; import eu.dnetlib.organizations.repository.readonly.ConflictGroupViewRepository; import eu.dnetlib.organizations.repository.readonly.DuplicateGroupViewRepository; +import eu.dnetlib.organizations.repository.readonly.DuplicateSearchViewRepository; import eu.dnetlib.organizations.repository.readonly.OpenaireDuplicateViewRepository; import eu.dnetlib.organizations.repository.readonly.OrganizationInfoViewRepository; import eu.dnetlib.organizations.repository.readonly.OrganizationSimpleViewRepository; @@ -63,6 +65,8 @@ public class OrganizationController extends AbstractDnetController { @Autowired private OpenaireDuplicateViewRepository openaireDuplicateViewRepository; @Autowired + private DuplicateSearchViewRepository duplicateSearchViewRepository; + @Autowired private ConflictGroupViewRepository conflictGroupViewRepository; @Autowired private SuggestionInfoViewByCountryRepository suggestionInfoViewByCountryRepository; @@ -217,26 +221,32 @@ public class OrganizationController extends AbstractDnetController { } @GetMapping("/search/{page}/{size}") - public Page search(@PathVariable final int page, + public Page search(@PathVariable final int page, @PathVariable final int size, @RequestParam final String q, @RequestParam(required = false, defaultValue = "") final String status, final Authentication authentication) { - final List statuses; - - if (StringUtils.isNotBlank(status)) { - statuses = Arrays.asList(status.split(",")); - } else if (UserInfo.isSimpleUser(authentication)) { - statuses = Arrays.asList(OrganizationStatus.approved.toString()); + if (StringUtils.isNotBlank(status) && status.equals("search_duplicate")) { + return UserInfo.isSuperAdmin(authentication) + ? duplicateSearchViewRepository.search(q, PageRequest.of(page, size)) + : duplicateSearchViewRepository.searchForUser(q, UserInfo.getEmail(authentication), PageRequest.of(page, size)); } else { - statuses = Arrays.asList(OrganizationStatus.approved.toString(), OrganizationStatus.suggested.toString()); + + final List statuses; + + if (StringUtils.isNotBlank(status)) { + statuses = Arrays.asList(status.split(",")); + } else if (UserInfo.isSimpleUser(authentication)) { + statuses = Arrays.asList(OrganizationStatus.approved.toString()); + } else { + statuses = Arrays.asList(OrganizationStatus.approved.toString(), OrganizationStatus.suggested.toString()); + } + + return UserInfo.isSuperAdmin(authentication) + ? organizationSimpleViewRepository.search(q, statuses, PageRequest.of(page, size)) + : organizationSimpleViewRepository.searchForUser(q, UserInfo.getEmail(authentication), statuses, PageRequest.of(page, size)); } - - return UserInfo.isSuperAdmin(authentication) - ? organizationSimpleViewRepository.search(q, statuses, PageRequest.of(page, size)) - : organizationSimpleViewRepository.searchForUser(q, UserInfo.getEmail(authentication), statuses, PageRequest.of(page, size)); - } @GetMapping("/byCountry/{status}/{code}/{page}/{size}") diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/OrganizationBase.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/OrganizationBase.java new file mode 100644 index 00000000..34cecead --- /dev/null +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/OrganizationBase.java @@ -0,0 +1,37 @@ +package eu.dnetlib.organizations.model; + +public interface OrganizationBase { + + String getId(); + + void setId(final String id); + + String getName(); + + void setName(final String name); + + String getType(); + + void setType(final String type); + + String getCity(); + + void setCity(final String city); + + String getCountry(); + + void setCountry(final String country); + + String[] getAcronyms(); + + void setAcronyms(final String[] acronyms); + + String getStatus(); + + void setStatus(final String status); + + String[] getUrls(); + + void setUrls(final String[] urls); + +} diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/DuplicateSearchView.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/DuplicateSearchView.java new file mode 100644 index 00000000..c9a76819 --- /dev/null +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/DuplicateSearchView.java @@ -0,0 +1,158 @@ +package eu.dnetlib.organizations.model.view; + +import java.io.Serializable; +import java.util.Objects; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; + +import org.hibernate.annotations.Type; +import org.hibernate.annotations.TypeDef; +import org.hibernate.annotations.TypeDefs; + +import com.vladmihalcea.hibernate.type.array.StringArrayType; + +import eu.dnetlib.organizations.model.OrganizationBase; + +@Entity +@Table(name = "duplicate_search_view") +@TypeDefs({ + @TypeDef(name = "string-array", typeClass = StringArrayType.class) +}) +public class DuplicateSearchView implements OrganizationBase, Serializable, Comparable { + + /** + * + */ + private static final long serialVersionUID = -2790140170438505458L; + + @Id + @Column(name = "id") + private String id; + + @Column(name = "name") + private String name; + + @Column(name = "type") + private String type; + + @Column(name = "city") + private String city; + + @Column(name = "country") + private String country; + + @Type(type = "string-array") + @Column(name = "acronyms", columnDefinition = "text[]") + private String[] acronyms; + + @Type(type = "string-array") + @Column(name = "urls", columnDefinition = "text[]") + private String[] urls; + + @Column(name = "status") + private String status; + + public DuplicateSearchView() {} + + public DuplicateSearchView(final String id) { + this.id = id; + } + + public DuplicateSearchView(final String id, final String name, final String type, final String city, final String country, final String[] acronyms, + final String status) { + this.id = id; + this.name = name; + this.type = type; + this.city = city; + this.country = country; + this.acronyms = acronyms; + this.status = status; + } + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(final String type) { + this.type = type; + } + + public String getCity() { + return city; + } + + public void setCity(final String city) { + this.city = city; + } + + public String getCountry() { + return country; + } + + public void setCountry(final String country) { + this.country = country; + } + + public String[] getAcronyms() { + return acronyms; + } + + public void setAcronyms(final String[] acronyms) { + this.acronyms = acronyms; + } + + public String getStatus() { + return status; + } + + public void setStatus(final String status) { + this.status = status; + } + + public String[] getUrls() { + return urls; + } + + public void setUrls(final String[] urls) { + this.urls = urls; + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) { return true; } + if (obj == null) { return false; } + if (!(obj instanceof DuplicateSearchView)) { return false; } + final DuplicateSearchView other = (DuplicateSearchView) obj; + return Objects.equals(id, other.id); + } + + @Override + public int compareTo(final DuplicateSearchView o) { + return id.compareTo(o.getId()); + } + +} diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/OrganizationSimpleView.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/OrganizationSimpleView.java index e30a32df..cf3af004 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/OrganizationSimpleView.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/OrganizationSimpleView.java @@ -14,12 +14,14 @@ import org.hibernate.annotations.TypeDefs; import com.vladmihalcea.hibernate.type.array.StringArrayType; +import eu.dnetlib.organizations.model.OrganizationBase; + @Entity @Table(name = "organizations_simple_view") @TypeDefs({ @TypeDef(name = "string-array", typeClass = StringArrayType.class) }) -public class OrganizationSimpleView implements Serializable, Comparable { +public class OrganizationSimpleView implements OrganizationBase, Serializable, Comparable { /** * @@ -70,66 +72,82 @@ public class OrganizationSimpleView implements Serializable, Comparable findByOaOriginalId(String oaOriginalId); + } diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/OrganizationRepository.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/OrganizationRepository.java index 593e60c2..cd9a50c7 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/OrganizationRepository.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/OrganizationRepository.java @@ -22,4 +22,8 @@ public interface OrganizationRepository extends JpaRepository { + + // SEARCH + @Query(value = "select o.* from duplicate_search_view o left outer join org_index_search idx on (idx.id = o.id) where idx.txt @@ plainto_tsquery(:text) order by o.name", nativeQuery = true) + Page search(@Param("text") String text, Pageable pageable); + + // SEARCH FOR USER + @Query(value = "select o.* from duplicate_search_view o left outer join org_index_search idx on (idx.id = o.id) left outer join user_countries uc on (uc.country = o.country) where idx.txt @@ plainto_tsquery(:text) and uc.email = :email order by o.name", nativeQuery = true) + Page searchForUser(@Param("text") String text, @Param("email") String email, Pageable pageable); + +} diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java index 517bfaf0..7a843317 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java @@ -1,7 +1,6 @@ package eu.dnetlib.organizations.utils; import java.time.OffsetDateTime; -import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.LinkedHashSet; @@ -10,6 +9,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.function.Function; import java.util.stream.Collectors; @@ -102,7 +102,9 @@ public class DatabaseUtils { @Transactional public String insertOrUpdateOrganization(final OrganizationView orgView, final String user, final boolean isSimpleUser) { - final String oldStatus = orgView.getId() != null ? organizationRepository.findById(orgView.getId()) + final String oldId = StringUtils.isNotBlank(orgView.getId()) ? new String(orgView.getId()) : null; + + final String oldStatus = oldId != null ? organizationRepository.findById(oldId) .map(Organization::getStatus) .orElse(null) : null; @@ -119,10 +121,15 @@ public class DatabaseUtils { throw new RuntimeException("User not authorized"); } - final String oldId = StringUtils.isNotBlank(orgView.getId()) ? new String(orgView.getId()) : null; - if (oldId == null || !oldId.startsWith(OpenOrgsConstants.OPENORGS_PREFIX)) { - orgView.setId(null); + if (isSimpleUser) { + final String pendingId = OpenOrgsConstants.OPENORGS_PENDING_PREFIX + UUID.randomUUID(); + orgView.setId(pendingId); + // to override the generation strategy of the ID + organizationRepository.preparePendingOrg(pendingId); + } else { + orgView.setId(null); // The ID is generated by the DB + } } final Organization org = new Organization(orgView.getId(), @@ -147,22 +154,21 @@ public class DatabaseUtils { makeRelations(newId, orgView, false); if (oldId != null) { - final List dups = new ArrayList<>(); - - dups.add(new OpenaireDuplicate(newId, oldId, SimilarityType.is_similar.toString(), "")); - dups.addAll(openaireDuplicateRepository.findByLocalId(oldId) + final List dups = openaireDuplicateRepository.findByLocalId(oldId) .stream() .map(d -> new OpenaireDuplicate(newId, d.getOaOriginalId(), SimilarityType.suggested.toString(), d.getOaCollectedFrom())) - .collect(Collectors.toList())); + .collect(Collectors.toList()); openaireDuplicateRepository.saveAll(dups); + dups.forEach(d -> { openaireDuplicateRepository.updateCreatedByIfMissing(d.getLocalId(), d.getOaOriginalId(), user); openaireDuplicateRepository.updateModificationDate(d.getLocalId(), d.getOaOriginalId(), user, now); }); - organizationRepository.updateStatus(oldId, OrganizationStatus.duplicate.toString()); - organizationRepository.updateModificationDate(oldId, user, now); + if (oldId.startsWith(OpenOrgsConstants.OPENORGS_PENDING_PREFIX)) { + organizationRepository.deleteById(oldId); + } } } @@ -209,14 +215,33 @@ public class DatabaseUtils { list.forEach(d -> { openaireDuplicateRepository.updateCreatedByIfMissing(d.getLocalId(), d.getOaOriginalId(), user); openaireDuplicateRepository.updateModificationDate(d.getLocalId(), d.getOaOriginalId(), user, now); - - if (d.getRelType().equals(SimilarityType.is_different.toString())) { - updateStatus(d.getOaOriginalId(), OrganizationStatus.suggested, user, now); - } else { - updateStatus(d.getOaOriginalId(), OrganizationStatus.duplicate, user, now); - } }); + for (final OpenaireDuplicate curr : list) { + if (curr.getRelType().equals(SimilarityType.is_similar.toString())) { + openaireDuplicateRepository.findByOaOriginalId(curr.getOaOriginalId()) + .stream() + .filter(d -> !d.getLocalId().equals(curr.getLocalId())) + .forEach(d -> { + if (d.getLocalId().startsWith(OpenOrgsConstants.OPENORGS_PENDING_PREFIX)) { + log.info("Removing useless pending org: " + d.getLocalId()); + organizationRepository.deleteById(d.getLocalId()); + } else if (d.getRelType().equals(SimilarityType.suggested.toString())) { + log.info("Removing invalid suggestion: " + d); + openaireDuplicateRepository.delete(d); + } else if (d.getRelType().equals(SimilarityType.is_similar.toString())) { + log.warn("***"); + log.warn("More is_similar relations"); + log.warn("rel1 ->" + d); + log.warn("rel1 ->" + curr); + log.warn("***"); + } else { + // is_different: nothing todo + } + }); + } + } + final String message = String.format("Duplicates updated (%s similars, %s differents, %s suggested)", list.stream() .filter(d -> d.getRelType().equals(SimilarityType.is_similar.toString())) .count(), list.stream() @@ -380,11 +405,6 @@ public class DatabaseUtils { } } - private void updateStatus(final String id, final OrganizationStatus status, final String user, final OffsetDateTime now) { - organizationRepository.updateStatus(id, status.toString()); - organizationRepository.updateModificationDate(id, user, now); - } - @Transactional public String fixConflictSimilars(final List similarIds, final String user) { diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/OpenOrgsConstants.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/OpenOrgsConstants.java index 0c210076..bbf7733b 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/OpenOrgsConstants.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/OpenOrgsConstants.java @@ -6,6 +6,7 @@ public class OpenOrgsConstants { public static final String OPENORGS_PREFIX = "openorgs____::"; public static final String OPENORGS_MESH_PREFIX = "openorgsmesh::"; + public static final String OPENORGS_PENDING_PREFIX = "pending_org_::"; public static final String OPENORGS_ROLE_PREFIX = "OPENORGS_"; diff --git a/apps/dnet-orgs-database-application/src/main/resources/sql/importDedupEvents.sql b/apps/dnet-orgs-database-application/src/main/resources/sql/importDedupEvents.sql index 71784890..24f97d8c 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/sql/importDedupEvents.sql +++ b/apps/dnet-orgs-database-application/src/main/resources/sql/importDedupEvents.sql @@ -4,36 +4,86 @@ DELETE FROM oa_conflicts WHERE created_by = 'dedupWf' and reltype = 'suggested' DELETE FROM oa_duplicates WHERE created_by = 'dedupWf' and reltype = 'suggested'; DELETE FROM organizations WHERE created_by = 'dedupWf' and modified_by = 'dedupWf'; +-- FIX ORIGINAL DATA +UPDATE organizations SET id = 'pending_org_::'||MD5(id) WHERE status = 'suggested' AND id NOT LIKE 'pending_org_::%'; + -- FIX IMPORT DATA +DELETE FROM tmp_dedup_events WHERE oa_original_id = '' OR oa_original_id IS NULL; +UPDATE tmp_dedup_events SET local_id = oa_original_id WHERE local_id = '' OR local_id IS NULL; UPDATE tmp_dedup_events SET oa_country = 'UNKNOWN' WHERE oa_country = '' OR oa_country IS NULL; UPDATE tmp_dedup_events SET oa_name = oa_acronym WHERE oa_name = '' OR oa_name IS NULL; DELETE FROM tmp_dedup_events WHERE oa_name = '' OR oa_name IS NULL; --- NEW ORGANIZATIONS +-- IMPORT MISSING TERMS +INSERT INTO id_types(val, name) SELECT distinct arr[2], arr[2] FROM (SELECT string_to_array(unnest(string_to_array(pid_list, '@@@')), '###') AS arr FROM tmp_dedup_events WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%') as c ON CONFLICT DO NOTHING; + +-- NEW ORGANIZATIONS (suggested) INSERT INTO organizations(id, name, country, status, ec_legalbody, ec_legalperson, ec_nonprofit, ec_researchorganization, ec_highereducation, ec_internationalorganizationeurinterests, ec_internationalorganization, ec_enterprise, ec_smevalidated, ec_nutscode, created_by, modified_by) -SELECT oa_original_id, oa_name, oa_country, 'suggested', ec_legalbody, ec_legalperson, ec_nonprofit, ec_researchorganization, ec_highereducation, ec_internationalorganizationeurinterests, ec_internationalorganization, ec_enterprise, ec_smevalidated, ec_nutscode, 'dedupWf', 'dedupWf' +SELECT 'pending_org_::'||MD5(local_id), oa_name, oa_country, 'suggested', ec_legalbody, ec_legalperson, ec_nonprofit, ec_researchorganization, ec_highereducation, ec_internationalorganizationeurinterests, ec_internationalorganization, ec_enterprise, ec_smevalidated, ec_nutscode, 'dedupWf', 'dedupWf' FROM tmp_dedup_events -WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' AND (oa_original_id = local_id OR local_id = '' OR local_id IS NULL) +WHERE local_id NOT LIKE 'openorgs\_\_\_\_::%' AND local_id = oa_original_id ON CONFLICT DO NOTHING; +INSERT INTO acronyms(id, acronym) +SELECT 'pending_org_::'||MD5(local_id), oa_acronym FROM tmp_dedup_events +WHERE local_id NOT LIKE 'openorgs\_\_\_\_::%' AND local_id = oa_original_id +ON CONFLICT DO NOTHING; + +INSERT INTO urls(id, url) +SELECT 'pending_org_::'||MD5(local_id), oa_url FROM tmp_dedup_events +WHERE local_id NOT LIKE 'openorgs\_\_\_\_::%' AND local_id = oa_original_id +ON CONFLICT DO NOTHING; + +INSERT INTO other_ids(id, otherid, type) +SELECT 'pending_org_::'||MD5(local_id), arr[1] AS otherid, arr[2] AS type +FROM ( + SELECT local_id, string_to_array(unnest(string_to_array(pid_list, '@@@')), '###') AS arr + FROM tmp_dedup_events + WHERE local_id NOT LIKE 'openorgs\_\_\_\_::%' AND local_id = oa_original_id +) as c +ON CONFLICT DO NOTHING; + +-- NEW ORGANIZATIONS (duplicate) INSERT INTO organizations(id, name, country, status, ec_legalbody, ec_legalperson, ec_nonprofit, ec_researchorganization, ec_highereducation, ec_internationalorganizationeurinterests, ec_internationalorganization, ec_enterprise, ec_smevalidated, ec_nutscode, created_by, modified_by) SELECT oa_original_id, oa_name, oa_country, 'duplicate', ec_legalbody, ec_legalperson, ec_nonprofit, ec_researchorganization, ec_highereducation, ec_internationalorganizationeurinterests, ec_internationalorganization, ec_enterprise, ec_smevalidated, ec_nutscode, 'dedupWf', 'dedupWf' FROM tmp_dedup_events -WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' AND local_id is NOT NULL AND local_id != '' AND local_id != oa_original_id +WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' ON CONFLICT DO NOTHING; -INSERT INTO acronyms(id, acronym) SELECT oa_original_id, oa_acronym FROM tmp_dedup_events WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' ON CONFLICT DO NOTHING; -INSERT INTO urls(id, url) SELECT oa_original_id, oa_url FROM tmp_dedup_events WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' ON CONFLICT DO NOTHING; -INSERT INTO id_types(val, name) SELECT distinct arr[2], arr[2] FROM (SELECT string_to_array(unnest(string_to_array(pid_list, '@@@')), '###') AS arr FROM tmp_dedup_events WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%') as c ON CONFLICT DO NOTHING; -INSERT INTO other_ids(id, otherid, type) SELECT oa_original_id, arr[1] AS otherid, arr[2] AS type FROM (SELECT oa_original_id, string_to_array(unnest(string_to_array(pid_list, '@@@')), '###') AS arr FROM tmp_dedup_events WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%') as c ON CONFLICT DO NOTHING; +INSERT INTO acronyms(id, acronym) +SELECT oa_original_id, oa_acronym FROM tmp_dedup_events +WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' +ON CONFLICT DO NOTHING; --- DUPLICATES +INSERT INTO urls(id, url) +SELECT oa_original_id, oa_url +FROM tmp_dedup_events +WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' +ON CONFLICT DO NOTHING; + +INSERT INTO other_ids(id, otherid, type) +SELECT oa_original_id, arr[1] AS otherid, arr[2] AS type +FROM ( + SELECT oa_original_id, + string_to_array(unnest(string_to_array(pid_list, '@@@')), '###') AS arr + FROM tmp_dedup_events + WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' +) as c +ON CONFLICT DO NOTHING; + +-- DUPLICATES (relations to openorgs) INSERT INTO oa_duplicates (local_id, oa_original_id, oa_collectedfrom, created_by, modified_by) SELECT local_id, oa_original_id, oa_collectedfrom, 'dedupWf', 'dedupWf' FROM tmp_dedup_events -WHERE local_id IS NOT NULL AND local_id != '' AND oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' AND local_id != oa_original_id +WHERE local_id LIKE 'openorgs\_\_\_\_::%' AND oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' ON CONFLICT DO NOTHING; +-- DUPLICATES (relations to suggested) +INSERT INTO oa_duplicates (local_id, oa_original_id, oa_collectedfrom, created_by, modified_by) +SELECT 'pending_org_::'||MD5(local_id), oa_original_id, oa_collectedfrom, 'dedupWf', 'dedupWf' +FROM tmp_dedup_events +WHERE local_id NOT LIKE 'openorgs\_\_\_\_::%' AND oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' +ON CONFLICT DO NOTHING; -- CONFLICTS (I generate all the couples) CREATE TEMPORARY TABLE tmp_conflict_groups AS diff --git a/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql b/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql index 0836892e..91378b0c 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql +++ b/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql @@ -572,6 +572,30 @@ GROUP BY org.country, org.status; +CREATE VIEW duplicate_search_view AS SELECT + org.id, + org.name, + org.type, + org.city, + org.country, + org.status, + array_remove(array_agg(DISTINCT a.acronym), NULL::text) AS acronyms, + array_remove(array_agg(DISTINCT u.url), NULL::text) AS urls +FROM organizations org + LEFT JOIN acronyms a ON org.id = a.id + LEFT JOIN urls u ON org.id = u.id + LEFT OUTER JOIN oa_duplicates d ON (org.id = d.oa_original_id) +WHERE + org.status = 'duplicate' +GROUP BY + org.id, + org.name, + org.type, + org.city, + org.country, + org.status +HAVING not('is_similar' = ANY(array_agg(d.reltype))); + CREATE VIEW users_view AS SELECT u.email, u.valid, diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/search/resultsByCountry.html b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/search/resultsByCountry.html index 7c99aac5..2d587359 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/search/resultsByCountry.html +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/search/resultsByCountry.html @@ -1,4 +1,5 @@ + next-function="next()" + show-status="true"> diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/search/resultsByType.html b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/search/resultsByType.html index 07dfb328..a31a5742 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/search/resultsByType.html +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/search/resultsByType.html @@ -1,4 +1,5 @@ + next-function="next()" + show-status="true"> diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/search/searchResults.html b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/search/searchResults.html index 275c8438..4662a5a9 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/search/searchResults.html +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/search/searchResults.html @@ -1,4 +1,5 @@ + next-function="next()" + show-status="true"> diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/org_duplicates.html b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/org_duplicates.html index b2958ee7..38998eb1 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/org_duplicates.html +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/org_duplicates.html @@ -68,4 +68,4 @@ - + diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/org_results_page.html b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/org_results_page.html index ff4139af..d486bac7 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/org_results_page.html +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/org_results_page.html @@ -46,11 +46,11 @@ - + - {{o.name}} + {{o.name}} {{o.name}} - {{o.status}} + {{o.status}} {{o.city || '-'}}, {{o.country}} {{o.acronyms.join()}} diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/select_org.modal.html b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/select_org.modal.html index 243eede8..c401e8c0 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/select_org.modal.html +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/select_org.modal.html @@ -18,7 +18,8 @@ next-function="search(searchValue, searchOrgs.number + 1, searchOrgs.size)" on-select="selectOrg()" selected-org="selectedOrg" - mode="select-modal"> + mode="select-modal" + show-status="true"> diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/js/organizations.js b/apps/dnet-orgs-database-application/src/main/resources/static/resources/js/organizations.js index 56ca6c06..2b4130be 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/static/resources/js/organizations.js +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/js/organizations.js @@ -212,7 +212,8 @@ orgsModule.directive('orgResultsPage', function($http, $location, $route) { 'prevFunction' : '&', 'onSelect' : '&', 'selectedOrg' : '=', - 'mode' : '@' + 'mode' : '@', + 'showStatus' : '@' }, templateUrl: 'resources/html/parts/org_results_page.html', link: function(scope, element, attrs, ctrl) {