D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

new root org for all suggested orgs

This commit is contained in:
Michele Artini 2021-04-19 16:00:29 +02:00
parent 85fa189f2f
commit e71bb3657a
18 changed files with 407 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java
View File

@ -29,6 +29,7 @@ import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.organizations.model.JournalEntry;
import eu.dnetlib.organizations.model.Note;
import eu.dnetlib.organizations.model.OpenaireDuplicate;
import eu.dnetlib.organizations.model.OrganizationBase;
import eu.dnetlib.organizations.model.utils.BrowseEntry;
import eu.dnetlib.organizations.model.utils.OrganizationConflict;
import eu.dnetlib.organizations.model.view.ConflictGroupView;
@ -42,6 +43,7 @@ import eu.dnetlib.organizations.repository.NoteRepository;
import eu.dnetlib.organizations.repository.UserCountryRepository;
import eu.dnetlib.organizations.repository.readonly.ConflictGroupViewRepository;
import eu.dnetlib.organizations.repository.readonly.DuplicateGroupViewRepository;
import eu.dnetlib.organizations.repository.readonly.DuplicateSearchViewRepository;
import eu.dnetlib.organizations.repository.readonly.OpenaireDuplicateViewRepository;
import eu.dnetlib.organizations.repository.readonly.OrganizationInfoViewRepository;
import eu.dnetlib.organizations.repository.readonly.OrganizationSimpleViewRepository;
@ -63,6 +65,8 @@ public class OrganizationController extends AbstractDnetController {
@Autowired
private OpenaireDuplicateViewRepository openaireDuplicateViewRepository;
@Autowired
private DuplicateSearchViewRepository duplicateSearchViewRepository;
@Autowired
private ConflictGroupViewRepository conflictGroupViewRepository;
@Autowired
private SuggestionInfoViewByCountryRepository suggestionInfoViewByCountryRepository;
@ -217,26 +221,32 @@ public class OrganizationController extends AbstractDnetController {
}
@GetMapping("/search/{page}/{size}")
public Page<OrganizationSimpleView> search(@PathVariable final int page,
public Page<? extends OrganizationBase> search(@PathVariable final int page,
@PathVariable final int size,
@RequestParam final String q,
@RequestParam(required = false, defaultValue = "") final String status,
final Authentication authentication) {
final List<String> statuses;
if (StringUtils.isNotBlank(status)) {
statuses = Arrays.asList(status.split(","));
} else if (UserInfo.isSimpleUser(authentication)) {
statuses = Arrays.asList(OrganizationStatus.approved.toString());
if (StringUtils.isNotBlank(status) && status.equals("search_duplicate")) {
return UserInfo.isSuperAdmin(authentication)
? duplicateSearchViewRepository.search(q, PageRequest.of(page, size))
: duplicateSearchViewRepository.searchForUser(q, UserInfo.getEmail(authentication), PageRequest.of(page, size));
} else {
statuses = Arrays.asList(OrganizationStatus.approved.toString(), OrganizationStatus.suggested.toString());
final List<String> statuses;
if (StringUtils.isNotBlank(status)) {
statuses = Arrays.asList(status.split(","));
} else if (UserInfo.isSimpleUser(authentication)) {
statuses = Arrays.asList(OrganizationStatus.approved.toString());
} else {
statuses = Arrays.asList(OrganizationStatus.approved.toString(), OrganizationStatus.suggested.toString());
}
return UserInfo.isSuperAdmin(authentication)
? organizationSimpleViewRepository.search(q, statuses, PageRequest.of(page, size))
: organizationSimpleViewRepository.searchForUser(q, UserInfo.getEmail(authentication), statuses, PageRequest.of(page, size));
}
return UserInfo.isSuperAdmin(authentication)
? organizationSimpleViewRepository.search(q, statuses, PageRequest.of(page, size))
: organizationSimpleViewRepository.searchForUser(q, UserInfo.getEmail(authentication), statuses, PageRequest.of(page, size));
}
@GetMapping("/byCountry/{status}/{code}/{page}/{size}")

37
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/OrganizationBase.java Normal file
View File

@ -0,0 +1,37 @@
package eu.dnetlib.organizations.model;
public interface OrganizationBase {
String getId();
void setId(final String id);
String getName();
void setName(final String name);
String getType();
void setType(final String type);
String getCity();
void setCity(final String city);
String getCountry();
void setCountry(final String country);
String[] getAcronyms();
void setAcronyms(final String[] acronyms);
String getStatus();
void setStatus(final String status);
String[] getUrls();
void setUrls(final String[] urls);
}

158
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/DuplicateSearchView.java Normal file
View File

@ -0,0 +1,158 @@
package eu.dnetlib.organizations.model.view;
import java.io.Serializable;
import java.util.Objects;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Table;
import org.hibernate.annotations.Type;
import org.hibernate.annotations.TypeDef;
import org.hibernate.annotations.TypeDefs;
import com.vladmihalcea.hibernate.type.array.StringArrayType;
import eu.dnetlib.organizations.model.OrganizationBase;
@Entity
@Table(name = "duplicate_search_view")
@TypeDefs({
@TypeDef(name = "string-array", typeClass = StringArrayType.class)
})
public class DuplicateSearchView implements OrganizationBase, Serializable, Comparable<DuplicateSearchView> {
/**
*
*/
private static final long serialVersionUID = -2790140170438505458L;
@Id
@Column(name = "id")
private String id;
@Column(name = "name")
private String name;
@Column(name = "type")
private String type;
@Column(name = "city")
private String city;
@Column(name = "country")
private String country;
@Type(type = "string-array")
@Column(name = "acronyms", columnDefinition = "text[]")
private String[] acronyms;
@Type(type = "string-array")
@Column(name = "urls", columnDefinition = "text[]")
private String[] urls;
@Column(name = "status")
private String status;
public DuplicateSearchView() {}
public DuplicateSearchView(final String id) {
this.id = id;
}
public DuplicateSearchView(final String id, final String name, final String type, final String city, final String country, final String[] acronyms,
final String status) {
this.id = id;
this.name = name;
this.type = type;
this.city = city;
this.country = country;
this.acronyms = acronyms;
this.status = status;
}
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(final String name) {
this.name = name;
}
public String getType() {
return type;
}
public void setType(final String type) {
this.type = type;
}
public String getCity() {
return city;
}
public void setCity(final String city) {
this.city = city;
}
public String getCountry() {
return country;
}
public void setCountry(final String country) {
this.country = country;
}
public String[] getAcronyms() {
return acronyms;
}
public void setAcronyms(final String[] acronyms) {
this.acronyms = acronyms;
}
public String getStatus() {
return status;
}
public void setStatus(final String status) {
this.status = status;
}
public String[] getUrls() {
return urls;
}
public void setUrls(final String[] urls) {
this.urls = urls;
}
@Override
public int hashCode() {
return Objects.hash(id);
}
@Override
public boolean equals(final Object obj) {
if (this == obj) { return true; }
if (obj == null) { return false; }
if (!(obj instanceof DuplicateSearchView)) { return false; }
final DuplicateSearchView other = (DuplicateSearchView) obj;
return Objects.equals(id, other.id);
}
@Override
public int compareTo(final DuplicateSearchView o) {
return id.compareTo(o.getId());
}
}

20
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/OrganizationSimpleView.java
View File

@ -14,12 +14,14 @@ import org.hibernate.annotations.TypeDefs;
import com.vladmihalcea.hibernate.type.array.StringArrayType;
import eu.dnetlib.organizations.model.OrganizationBase;
@Entity
@Table(name = "organizations_simple_view")
@TypeDefs({
@TypeDef(name = "string-array", typeClass = StringArrayType.class)
})
public class OrganizationSimpleView implements Serializable, Comparable<OrganizationSimpleView> {
public class OrganizationSimpleView implements OrganizationBase, Serializable, Comparable<OrganizationSimpleView> {
/**
*
@ -70,66 +72,82 @@ public class OrganizationSimpleView implements Serializable, Comparable<Organiza
this.status = status;
}
@Override
public String getId() {
return id;
}
@Override
public void setId(final String id) {
this.id = id;
}
@Override
public String getName() {
return name;
}
@Override
public void setName(final String name) {
this.name = name;
}
@Override
public String getType() {
return type;
}
@Override
public void setType(final String type) {
this.type = type;
}
@Override
public String getCity() {
return city;
}
@Override
public void setCity(final String city) {
this.city = city;
}
@Override
public String getCountry() {
return country;
}
@Override
public void setCountry(final String country) {
this.country = country;
}
@Override
public String[] getAcronyms() {
return acronyms;
}
@Override
public void setAcronyms(final String[] acronyms) {
this.acronyms = acronyms;
}
@Override
public String getStatus() {
return status;
}
@Override
public void setStatus(final String status) {
this.status = status;
}
@Override
public String[] getUrls() {
return urls;
}
@Override
public void setUrls(final String[] urls) {
this.urls = urls;
}

2
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/OpenaireDuplicateRepository.java
View File

@ -24,4 +24,6 @@ public interface OpenaireDuplicateRepository extends JpaRepository<OpenaireDupli
@Query(value = "update oa_duplicates set created_by = ?3 where ((local_id = ?1 and oa_original_id = ?2) or (local_id = ?2 and oa_original_id = ?1)) and (created_by is null or created_by = '')", nativeQuery = true)
void updateCreatedByIfMissing(String id1, String id2, String user);
List<OpenaireDuplicate> findByOaOriginalId(String oaOriginalId);
}

4
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/OrganizationRepository.java
View File

@ -22,4 +22,8 @@ public interface OrganizationRepository extends JpaRepository<Organization, Stri
@Query("update Organization set status = ?2 where id = ?1")
void updateStatus(String id, String status);
@Modifying
@Query(value = "insert into organizations(id) values (?1)", nativeQuery = true)
void preparePendingOrg(String id);
}

22
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/readonly/DuplicateSearchViewRepository.java Normal file
View File

@ -0,0 +1,22 @@
package eu.dnetlib.organizations.repository.readonly;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository;
import eu.dnetlib.organizations.model.view.DuplicateSearchView;
@Repository
public interface DuplicateSearchViewRepository extends ReadOnlyRepository<DuplicateSearchView, String> {
// SEARCH
@Query(value = "select o.* from duplicate_search_view o left outer join org_index_search idx on (idx.id = o.id) where idx.txt @@ plainto_tsquery(:text) order by o.name", nativeQuery = true)
Page<DuplicateSearchView> search(@Param("text") String text, Pageable pageable);
// SEARCH FOR USER
@Query(value = "select o.* from duplicate_search_view o left outer join org_index_search idx on (idx.id = o.id) left outer join user_countries uc on (uc.country = o.country) where idx.txt @@ plainto_tsquery(:text) and uc.email = :email order by o.name", nativeQuery = true)
Page<DuplicateSearchView> searchForUser(@Param("text") String text, @Param("email") String email, Pageable pageable);
}

66
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java
View File

@ -1,7 +1,6 @@
package eu.dnetlib.organizations.utils;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashSet;
@ -10,6 +9,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.function.Function;
import java.util.stream.Collectors;
@ -102,7 +102,9 @@ public class DatabaseUtils {
@Transactional
public String insertOrUpdateOrganization(final OrganizationView orgView, final String user, final boolean isSimpleUser) {
final String oldStatus = orgView.getId() != null ? organizationRepository.findById(orgView.getId())
final String oldId = StringUtils.isNotBlank(orgView.getId()) ? new String(orgView.getId()) : null;
final String oldStatus = oldId != null ? organizationRepository.findById(oldId)
.map(Organization::getStatus)
.orElse(null) : null;
@ -119,10 +121,15 @@ public class DatabaseUtils {
throw new RuntimeException("User not authorized");
}
final String oldId = StringUtils.isNotBlank(orgView.getId()) ? new String(orgView.getId()) : null;
if (oldId == null || !oldId.startsWith(OpenOrgsConstants.OPENORGS_PREFIX)) {
orgView.setId(null);
if (isSimpleUser) {
final String pendingId = OpenOrgsConstants.OPENORGS_PENDING_PREFIX + UUID.randomUUID();
orgView.setId(pendingId);
// to override the generation strategy of the ID
organizationRepository.preparePendingOrg(pendingId);
} else {
orgView.setId(null); // The ID is generated by the DB
}
}
final Organization org = new Organization(orgView.getId(),
@ -147,22 +154,21 @@ public class DatabaseUtils {
makeRelations(newId, orgView, false);
if (oldId != null) {
final List<OpenaireDuplicate> dups = new ArrayList<>();
dups.add(new OpenaireDuplicate(newId, oldId, SimilarityType.is_similar.toString(), ""));
dups.addAll(openaireDuplicateRepository.findByLocalId(oldId)
final List<OpenaireDuplicate> dups = openaireDuplicateRepository.findByLocalId(oldId)
.stream()
.map(d -> new OpenaireDuplicate(newId, d.getOaOriginalId(), SimilarityType.suggested.toString(), d.getOaCollectedFrom()))
.collect(Collectors.toList()));
.collect(Collectors.toList());
openaireDuplicateRepository.saveAll(dups);
dups.forEach(d -> {
openaireDuplicateRepository.updateCreatedByIfMissing(d.getLocalId(), d.getOaOriginalId(), user);
openaireDuplicateRepository.updateModificationDate(d.getLocalId(), d.getOaOriginalId(), user, now);
});
organizationRepository.updateStatus(oldId, OrganizationStatus.duplicate.toString());
organizationRepository.updateModificationDate(oldId, user, now);
if (oldId.startsWith(OpenOrgsConstants.OPENORGS_PENDING_PREFIX)) {
organizationRepository.deleteById(oldId);
}
}
}
@ -209,14 +215,33 @@ public class DatabaseUtils {
list.forEach(d -> {
openaireDuplicateRepository.updateCreatedByIfMissing(d.getLocalId(), d.getOaOriginalId(), user);
openaireDuplicateRepository.updateModificationDate(d.getLocalId(), d.getOaOriginalId(), user, now);
if (d.getRelType().equals(SimilarityType.is_different.toString())) {
updateStatus(d.getOaOriginalId(), OrganizationStatus.suggested, user, now);
} else {
updateStatus(d.getOaOriginalId(), OrganizationStatus.duplicate, user, now);
}
});
for (final OpenaireDuplicate curr : list) {
if (curr.getRelType().equals(SimilarityType.is_similar.toString())) {
openaireDuplicateRepository.findByOaOriginalId(curr.getOaOriginalId())
.stream()
.filter(d -> !d.getLocalId().equals(curr.getLocalId()))
.forEach(d -> {
if (d.getLocalId().startsWith(OpenOrgsConstants.OPENORGS_PENDING_PREFIX)) {
log.info("Removing useless pending org: " + d.getLocalId());
organizationRepository.deleteById(d.getLocalId());
} else if (d.getRelType().equals(SimilarityType.suggested.toString())) {
log.info("Removing invalid suggestion: " + d);
openaireDuplicateRepository.delete(d);
} else if (d.getRelType().equals(SimilarityType.is_similar.toString())) {
log.warn("***");
log.warn("More is_similar relations");
log.warn("rel1 ->" + d);
log.warn("rel1 ->" + curr);
log.warn("***");
} else {
// is_different: nothing todo
}
});
}
}
final String message = String.format("Duplicates updated (%s similars, %s differents, %s suggested)", list.stream()
.filter(d -> d.getRelType().equals(SimilarityType.is_similar.toString()))
.count(), list.stream()
@ -380,11 +405,6 @@ public class DatabaseUtils {
}
}
private void updateStatus(final String id, final OrganizationStatus status, final String user, final OffsetDateTime now) {
organizationRepository.updateStatus(id, status.toString());
organizationRepository.updateModificationDate(id, user, now);
}
@Transactional
public String fixConflictSimilars(final List<String> similarIds, final String user) {

1
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/OpenOrgsConstants.java
View File

@ -6,6 +6,7 @@ public class OpenOrgsConstants {
public static final String OPENORGS_PREFIX = "openorgs____::";
public static final String OPENORGS_MESH_PREFIX = "openorgsmesh::";
public static final String OPENORGS_PENDING_PREFIX = "pending_org_::";
public static final String OPENORGS_ROLE_PREFIX = "OPENORGS_";

70
apps/dnet-orgs-database-application/src/main/resources/sql/importDedupEvents.sql
View File

@ -4,36 +4,86 @@ DELETE FROM oa_conflicts WHERE created_by = 'dedupWf' and reltype = 'suggested'
DELETE FROM oa_duplicates WHERE created_by = 'dedupWf' and reltype = 'suggested';
DELETE FROM organizations WHERE created_by = 'dedupWf' and modified_by = 'dedupWf';
-- FIX ORIGINAL DATA
UPDATE organizations SET id = 'pending_org_::'||MD5(id) WHERE status = 'suggested' AND id NOT LIKE 'pending_org_::%';
-- FIX IMPORT DATA
DELETE FROM tmp_dedup_events WHERE oa_original_id = '' OR oa_original_id IS NULL;
UPDATE tmp_dedup_events SET local_id = oa_original_id WHERE local_id = '' OR local_id IS NULL;
UPDATE tmp_dedup_events SET oa_country = 'UNKNOWN' WHERE oa_country = '' OR oa_country IS NULL;
UPDATE tmp_dedup_events SET oa_name = oa_acronym WHERE oa_name = '' OR oa_name IS NULL;
DELETE FROM tmp_dedup_events WHERE oa_name = '' OR oa_name IS NULL;
-- NEW ORGANIZATIONS
-- IMPORT MISSING TERMS
INSERT INTO id_types(val, name) SELECT distinct arr[2], arr[2] FROM (SELECT string_to_array(unnest(string_to_array(pid_list, '@@@')), '###') AS arr FROM tmp_dedup_events WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%') as c ON CONFLICT DO NOTHING;
-- NEW ORGANIZATIONS (suggested)
INSERT INTO organizations(id, name, country, status, ec_legalbody, ec_legalperson, ec_nonprofit, ec_researchorganization, ec_highereducation, ec_internationalorganizationeurinterests, ec_internationalorganization, ec_enterprise, ec_smevalidated, ec_nutscode, created_by, modified_by)
SELECT oa_original_id, oa_name, oa_country, 'suggested', ec_legalbody, ec_legalperson, ec_nonprofit, ec_researchorganization, ec_highereducation, ec_internationalorganizationeurinterests, ec_internationalorganization, ec_enterprise, ec_smevalidated, ec_nutscode, 'dedupWf', 'dedupWf'
SELECT 'pending_org_::'||MD5(local_id), oa_name, oa_country, 'suggested', ec_legalbody, ec_legalperson, ec_nonprofit, ec_researchorganization, ec_highereducation, ec_internationalorganizationeurinterests, ec_internationalorganization, ec_enterprise, ec_smevalidated, ec_nutscode, 'dedupWf', 'dedupWf'
FROM tmp_dedup_events
WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' AND (oa_original_id = local_id OR local_id = '' OR local_id IS NULL)
WHERE local_id NOT LIKE 'openorgs\_\_\_\_::%' AND local_id = oa_original_id
ON CONFLICT DO NOTHING;
INSERT INTO acronyms(id, acronym)
SELECT 'pending_org_::'||MD5(local_id), oa_acronym FROM tmp_dedup_events
WHERE local_id NOT LIKE 'openorgs\_\_\_\_::%' AND local_id = oa_original_id
ON CONFLICT DO NOTHING;
INSERT INTO urls(id, url)
SELECT 'pending_org_::'||MD5(local_id), oa_url FROM tmp_dedup_events
WHERE local_id NOT LIKE 'openorgs\_\_\_\_::%' AND local_id = oa_original_id
ON CONFLICT DO NOTHING;
INSERT INTO other_ids(id, otherid, type)
SELECT 'pending_org_::'||MD5(local_id), arr[1] AS otherid, arr[2] AS type
FROM (
SELECT local_id, string_to_array(unnest(string_to_array(pid_list, '@@@')), '###') AS arr
FROM tmp_dedup_events
WHERE local_id NOT LIKE 'openorgs\_\_\_\_::%' AND local_id = oa_original_id
) as c
ON CONFLICT DO NOTHING;
-- NEW ORGANIZATIONS (duplicate)
INSERT INTO organizations(id, name, country, status, ec_legalbody, ec_legalperson, ec_nonprofit, ec_researchorganization, ec_highereducation, ec_internationalorganizationeurinterests, ec_internationalorganization, ec_enterprise, ec_smevalidated, ec_nutscode, created_by, modified_by)
SELECT oa_original_id, oa_name, oa_country, 'duplicate', ec_legalbody, ec_legalperson, ec_nonprofit, ec_researchorganization, ec_highereducation, ec_internationalorganizationeurinterests, ec_internationalorganization, ec_enterprise, ec_smevalidated, ec_nutscode, 'dedupWf', 'dedupWf'
FROM tmp_dedup_events
WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' AND local_id is NOT NULL AND local_id != '' AND local_id != oa_original_id
WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%'
ON CONFLICT DO NOTHING;
INSERT INTO acronyms(id, acronym) SELECT oa_original_id, oa_acronym FROM tmp_dedup_events WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' ON CONFLICT DO NOTHING;
INSERT INTO urls(id, url) SELECT oa_original_id, oa_url FROM tmp_dedup_events WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' ON CONFLICT DO NOTHING;
INSERT INTO id_types(val, name) SELECT distinct arr[2], arr[2] FROM (SELECT string_to_array(unnest(string_to_array(pid_list, '@@@')), '###') AS arr FROM tmp_dedup_events WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%') as c ON CONFLICT DO NOTHING;
INSERT INTO other_ids(id, otherid, type) SELECT oa_original_id, arr[1] AS otherid, arr[2] AS type FROM (SELECT oa_original_id, string_to_array(unnest(string_to_array(pid_list, '@@@')), '###') AS arr FROM tmp_dedup_events WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%') as c ON CONFLICT DO NOTHING;
INSERT INTO acronyms(id, acronym)
SELECT oa_original_id, oa_acronym FROM tmp_dedup_events
WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%'
ON CONFLICT DO NOTHING;
-- DUPLICATES
INSERT INTO urls(id, url)
SELECT oa_original_id, oa_url
FROM tmp_dedup_events
WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%'
ON CONFLICT DO NOTHING;
INSERT INTO other_ids(id, otherid, type)
SELECT oa_original_id, arr[1] AS otherid, arr[2] AS type
FROM (
SELECT oa_original_id,
string_to_array(unnest(string_to_array(pid_list, '@@@')), '###') AS arr
FROM tmp_dedup_events
WHERE oa_original_id NOT LIKE 'openorgs\_\_\_\_::%'
) as c
ON CONFLICT DO NOTHING;
-- DUPLICATES (relations to openorgs)
INSERT INTO oa_duplicates (local_id, oa_original_id, oa_collectedfrom, created_by, modified_by)
SELECT local_id, oa_original_id, oa_collectedfrom, 'dedupWf', 'dedupWf'
FROM tmp_dedup_events
WHERE local_id IS NOT NULL AND local_id != '' AND oa_original_id NOT LIKE 'openorgs\_\_\_\_::%' AND local_id != oa_original_id
WHERE local_id LIKE 'openorgs\_\_\_\_::%' AND oa_original_id NOT LIKE 'openorgs\_\_\_\_::%'
ON CONFLICT DO NOTHING;
-- DUPLICATES (relations to suggested)
INSERT INTO oa_duplicates (local_id, oa_original_id, oa_collectedfrom, created_by, modified_by)
SELECT 'pending_org_::'||MD5(local_id), oa_original_id, oa_collectedfrom, 'dedupWf', 'dedupWf'
FROM tmp_dedup_events
WHERE local_id NOT LIKE 'openorgs\_\_\_\_::%' AND oa_original_id NOT LIKE 'openorgs\_\_\_\_::%'
ON CONFLICT DO NOTHING;
-- CONFLICTS (I generate all the couples)
CREATE TEMPORARY TABLE tmp_conflict_groups AS

24
apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql
View File

@ -572,6 +572,30 @@ GROUP BY
org.country,
org.status;
CREATE VIEW duplicate_search_view AS SELECT
org.id,
org.name,
org.type,
org.city,
org.country,
org.status,
array_remove(array_agg(DISTINCT a.acronym), NULL::text) AS acronyms,
array_remove(array_agg(DISTINCT u.url), NULL::text) AS urls
FROM organizations org
LEFT JOIN acronyms a ON org.id = a.id
LEFT JOIN urls u ON org.id = u.id
LEFT OUTER JOIN oa_duplicates d ON (org.id = d.oa_original_id)
WHERE
org.status = 'duplicate'
GROUP BY
org.id,
org.name,
org.type,
org.city,
org.country,
org.status
HAVING not('is_similar' = ANY(array_agg(d.reltype)));
CREATE VIEW users_view AS SELECT
u.email,
u.valid,

3
apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/search/resultsByCountry.html
View File

@ -1,4 +1,5 @@
<org-results-page search-message="Searching for country: {{fieldValue}}"
orgs="orgs"
prev-function="prev()"
next-function="next()"></org-results-page>
next-function="next()"
show-status="true"></org-results-page>

3
apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/search/resultsByType.html
View File

@ -1,4 +1,5 @@
<org-results-page search-message="Searching for type: {{fieldValue}}"
orgs="orgs"
prev-function="prev()"
next-function="next()"></org-results-page>
next-function="next()"
show-status="true"></org-results-page>

3
apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/search/searchResults.html
View File

@ -1,4 +1,5 @@
<org-results-page search-message="Searching for: {{searchText}}"
orgs="orgs"
prev-function="prev()"
next-function="next()"></org-results-page>
next-function="next()"
show-status="true"></org-results-page>

2
apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/org_duplicates.html
View File

@ -68,4 +68,4 @@
</div>
<select-org-modal modal-id="addDuplicateModal" selected-org="newDuplicate" filter-status="suggested" on-select="addDuplicate()"></select-org-modal>
<select-org-modal modal-id="addDuplicateModal" selected-org="newDuplicate" filter-status="search_duplicate" on-select="addDuplicate()"></select-org-modal>

6
apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/org_results_page.html
View File

@ -46,11 +46,11 @@
</tr>
</thead>
<tbody>
<tr ng-repeat="o in orgs.content" class="d-flex" ng-class="{'table-warning' : o.status != 'approved'}">
<tr ng-repeat="o in orgs.content" class="d-flex" ng-class="{'table-warning' : o.showStatus && o.status != 'approved'}">
<td class="col-6">
<a ng-if="mode == 'select-modal'" href="javascript:void(0)" title="select" ng-click="selectOrg(o)" data-dismiss="modal">{{o.name}}</a>
<a ng-if="mode == 'select-modal'" href="javascript:void(0)" title="{{o.id}}" ng-click="selectOrg(o)" data-dismiss="modal">{{o.name}}</a>
<a ng-if="mode != 'select-modal'" href="#!/edit/0/{{o.id}}" title="{{o.id}}">{{o.name}}</a>
<span class="badge badge-warning" ng-if="o.status != 'approved'">{{o.status}}</span>
<span class="badge badge-warning" ng-if="o.showStatus && o.status != 'approved'">{{o.status}}</span>
</td>
<td class="col-4"><img ng-src="resources/images/flags/{{o.country}}.gif" /> {{o.city || '-'}}, {{o.country}}</td>
<td class="col-1 text-center">{{o.acronyms.join()}}</td>

3
apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/select_org.modal.html
View File

@ -18,7 +18,8 @@
next-function="search(searchValue, searchOrgs.number + 1, searchOrgs.size)"
on-select="selectOrg()"
selected-org="selectedOrg"
mode="select-modal"></org-results-page>
mode="select-modal"
show-status="true"></org-results-page>
</div>
</div>

3
apps/dnet-orgs-database-application/src/main/resources/static/resources/js/organizations.js
View File

@ -212,7 +212,8 @@ orgsModule.directive('orgResultsPage', function($http, $location, $route) {
'prevFunction' : '&',
'onSelect' : '&',
'selectedOrg' : '=',
'mode' : '@'
'mode' : '@',
'showStatus' : '@'
},
templateUrl: 'resources/html/parts/org_results_page.html',
link: function(scope, element, attrs, ctrl) {