moved some fields from oa_duplicate to organizations

This commit is contained in:
Michele Artini 2020-10-09 16:00:12 +02:00
parent 35609798b3
commit 5a76d32f15
17 changed files with 197 additions and 88 deletions

View File

@ -27,13 +27,13 @@ public class OpenaireInternalApiController {
private static final Log log = LogFactory.getLog(OpenaireInternalApiController.class);
@GetMapping("/import/simrels")
public List<String> importSimRels(final HttpServletRequest req) {
@GetMapping("/import/dedupEvents")
public List<String> importDedupEvents(final HttpServletRequest req) {
if (req.getRemoteAddr().equals(httpsProxy)) {
log.warn("Call received by blaklisted ip (https proxy): " + req.getRemoteAddr());
throw new RuntimeException("Call received by blaklisted ip (https proxy): " + req.getRemoteAddr());
}
new Thread(databaseUtils::importSimRels).run();
new Thread(databaseUtils::importDedupEvents).run();
return Arrays.asList("Importing simrels (request from " + req.getRemoteAddr() + ") ...");
}
}

View File

@ -31,13 +31,14 @@ import eu.dnetlib.organizations.model.utils.BrowseEntry;
import eu.dnetlib.organizations.model.utils.OrganizationConflict;
import eu.dnetlib.organizations.model.view.ConflictGroupView;
import eu.dnetlib.organizations.model.view.DuplicateGroupView;
import eu.dnetlib.organizations.model.view.OpenaireDuplicateView;
import eu.dnetlib.organizations.model.view.OrganizationInfoView;
import eu.dnetlib.organizations.model.view.OrganizationSimpleView;
import eu.dnetlib.organizations.model.view.OrganizationView;
import eu.dnetlib.organizations.repository.OpenaireDuplicateRepository;
import eu.dnetlib.organizations.repository.UserCountryRepository;
import eu.dnetlib.organizations.repository.readonly.ConflictGroupViewRepository;
import eu.dnetlib.organizations.repository.readonly.DuplicateGroupViewRepository;
import eu.dnetlib.organizations.repository.readonly.OpenaireDuplicateViewRepository;
import eu.dnetlib.organizations.repository.readonly.OrganizationInfoViewRepository;
import eu.dnetlib.organizations.repository.readonly.OrganizationSimpleViewRepository;
import eu.dnetlib.organizations.repository.readonly.OrganizationViewRepository;
@ -55,7 +56,7 @@ public class OrganizationController {
@Autowired
private OrganizationSimpleViewRepository organizationSimpleViewRepository;
@Autowired
private OpenaireDuplicateRepository openaireDuplicateRepository;
private OpenaireDuplicateViewRepository openaireDuplicateViewRepository;
@Autowired
private ConflictGroupViewRepository conflictGroupViewRepository;
@Autowired
@ -127,9 +128,9 @@ public class OrganizationController {
}
@GetMapping("/duplicates")
public List<OpenaireDuplicate> duplicates(@RequestParam final String id, final Authentication authentication) {
public List<OpenaireDuplicateView> duplicates(@RequestParam final String id, final Authentication authentication) {
if (UserInfo.isSuperAdmin(authentication) || userCountryRepository.verifyAuthorizationForId(id, authentication.getName())) {
return openaireDuplicateRepository.findByLocalId(id);
return openaireDuplicateViewRepository.findByLocalId(id);
} else {
throw new RuntimeException("User not authorized");
}
@ -187,7 +188,9 @@ public class OrganizationController {
}
@PostMapping("/duplicates")
public List<OpenaireDuplicate> duplicates(@RequestBody final List<OpenaireDuplicate> simrels, final Authentication authentication) {
public List<OpenaireDuplicateView> duplicates(@RequestBody final List<OpenaireDuplicate> simrels, final Authentication authentication) {
if (simrels.isEmpty()) { return new ArrayList<>(); }
final boolean b = UserInfo.isSuperAdmin(authentication)
|| simrels.stream()
@ -196,7 +199,8 @@ public class OrganizationController {
.allMatch(id -> userCountryRepository.verifyAuthorizationForId(id, authentication.getName()));
if (b) {
return databaseUtils.saveDuplicates(simrels, authentication.getName());
databaseUtils.saveDuplicates(simrels, authentication.getName());
return openaireDuplicateViewRepository.findByLocalId(simrels.get(0).getLocalId());
} else {
throw new RuntimeException("User not authorized");
}

View File

@ -26,21 +26,6 @@ public class OpenaireDuplicate implements Serializable {
@Column(name = "oa_original_id")
private String oaOriginalId;
@Column(name = "oa_name")
private String oaName;
@Column(name = "oa_acronym")
private String oaAcronym;
@Column(name = "oa_country")
private String oaCountry;
@Column(name = "oa_url")
private String oaUrl;
@Column(name = "oa_collectedfrom")
private String oaCollectedFrom;
@Column(name = "reltype")
private String relType;
@ -60,46 +45,6 @@ public class OpenaireDuplicate implements Serializable {
this.oaOriginalId = oaOriginalId;
}
public String getOaName() {
return oaName;
}
public void setOaName(final String oaName) {
this.oaName = oaName;
}
public String getOaAcronym() {
return oaAcronym;
}
public void setOaAcronym(final String oaAcronym) {
this.oaAcronym = oaAcronym;
}
public String getOaCountry() {
return oaCountry;
}
public void setOaCountry(final String oaCountry) {
this.oaCountry = oaCountry;
}
public String getOaUrl() {
return oaUrl;
}
public void setOaUrl(final String oaUrl) {
this.oaUrl = oaUrl;
}
public String getOaCollectedFrom() {
return oaCollectedFrom;
}
public void setOaCollectedFrom(final String oaCollectedFrom) {
this.oaCollectedFrom = oaCollectedFrom;
}
public String getRelType() {
return relType;
}

View File

@ -0,0 +1,113 @@
package eu.dnetlib.organizations.model.view;
import java.io.Serializable;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.IdClass;
import javax.persistence.Table;
import eu.dnetlib.organizations.model.OpenaireDuplicatePK;
@Entity
@Table(name = "oa_duplicates_view")
@IdClass(OpenaireDuplicatePK.class)
public class OpenaireDuplicateView implements Serializable {
/**
*
*/
private static final long serialVersionUID = 2673168234221945705L;
@Id
@Column(name = "local_id")
private String localId;
@Id
@Column(name = "oa_original_id")
private String oaOriginalId;
@Column(name = "oa_name")
private String oaName;
@Column(name = "oa_acronym")
private String oaAcronym;
@Column(name = "oa_country")
private String oaCountry;
@Column(name = "oa_url")
private String oaUrl;
@Column(name = "oa_collectedfrom")
private String oaCollectedFrom;
@Column(name = "reltype")
private String relType;
public String getLocalId() {
return localId;
}
public void setLocalId(final String localId) {
this.localId = localId;
}
public String getOaOriginalId() {
return oaOriginalId;
}
public void setOaOriginalId(final String oaOriginalId) {
this.oaOriginalId = oaOriginalId;
}
public String getOaName() {
return oaName;
}
public void setOaName(final String oaName) {
this.oaName = oaName;
}
public String getOaAcronym() {
return oaAcronym;
}
public void setOaAcronym(final String oaAcronym) {
this.oaAcronym = oaAcronym;
}
public String getOaCountry() {
return oaCountry;
}
public void setOaCountry(final String oaCountry) {
this.oaCountry = oaCountry;
}
public String getOaUrl() {
return oaUrl;
}
public void setOaUrl(final String oaUrl) {
this.oaUrl = oaUrl;
}
public String getOaCollectedFrom() {
return oaCollectedFrom;
}
public void setOaCollectedFrom(final String oaCollectedFrom) {
this.oaCollectedFrom = oaCollectedFrom;
}
public String getRelType() {
return relType;
}
public void setRelType(final String relType) {
this.relType = relType;
}
}

View File

@ -0,0 +1,15 @@
package eu.dnetlib.organizations.repository.readonly;
import java.util.List;
import org.springframework.stereotype.Repository;
import eu.dnetlib.organizations.model.OpenaireDuplicatePK;
import eu.dnetlib.organizations.model.view.OpenaireDuplicateView;
@Repository
public interface OpenaireDuplicateViewRepository extends ReadOnlyRepository<OpenaireDuplicateView, OpenaireDuplicatePK> {
List<OpenaireDuplicateView> findByLocalId(String id);
}

View File

@ -14,7 +14,6 @@ import java.util.stream.Collectors;
import javax.transaction.Transactional;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -134,12 +133,13 @@ public class DatabaseUtils {
}
@Transactional
public List<OpenaireDuplicate> saveDuplicates(final List<OpenaireDuplicate> simrels, final String email) {
public void saveDuplicates(final List<OpenaireDuplicate> simrels, final String email) {
final OffsetDateTime now = OffsetDateTime.now();
final List<OpenaireDuplicate> list = openaireDuplicateRepository.saveAll(simrels);
list.forEach(d -> openaireDuplicateRepository.updateModificationDate(d.getLocalId(), d.getOaOriginalId(), email, now));
return list;
}
private void makeNewRelations(final String orgId, final OrganizationView orgView) {
@ -319,12 +319,12 @@ public class DatabaseUtils {
}
@Transactional
public void importSimRels() {
public void importDedupEvents() {
try {
log.info("Importing conflicts and duplicates...");
jdbcTemplate.update(IOUtils.toString(getClass().getResourceAsStream("/sql/importNewRels.sql")));
log.info("...done");
verifyConflictGroups(true);
// log.info("Importing conflicts and duplicates...");
// jdbcTemplate.update(IOUtils.toString(getClass().getResourceAsStream("/sql/importNewRels.sql")));
// log.info("...done");
// verifyConflictGroups(true);
} catch (final Exception e) {
log.error("Error importing conflicts and duplicates", e);
}

View File

@ -5,5 +5,6 @@ public enum OrganizationStatus {
approved,
discarded,
hidden,
deleted
deleted,
duplicate
}

View File

@ -1,5 +1,7 @@
package eu.dnetlib.organizations.utils;
public enum SimilarityType {
suggested, is_similar, is_different
suggested,
is_similar,
is_different
}

View File

@ -381,7 +381,7 @@ CREATE INDEX urls_id_idx ON urls(id);
CREATE TABLE oa_duplicates (
local_id text REFERENCES organizations(id) ON UPDATE CASCADE,
oa_original_id text NOT NULL,
oa_original_id text REFERENCES organizations(id) ON UPDATE CASCADE,
oa_name text NOT NULL,
oa_acronym text,
oa_country text,
@ -396,6 +396,28 @@ CREATE TABLE oa_duplicates (
CREATE INDEX oa_duplicates_local_id_idx ON oa_duplicates(local_id);
CREATE VIEW oa_duplicates_view AS
d.local_id as local_id,
d.oa_original_id as oa_original_id,
o.name as oa_name,
array_to_string(array_agg(a.acronym), ', ') as oa_acronym,
o.country as oa_country,
array_to_string(array_agg(u.url), ', ') as oa_url,
d.oa_collectedfrom as oa_collectedfrom,
d.reltype as reltype
FROM
oa_duplicates d
LEFT OUTER JOIN organizations o ON (o.id = d.oa_original_id)
LEFT OUTER JOIN acronyms a ON (o.id = a.id)
LEFT OUTER JOIN urls u ON (o.id = u.id)
GROUP BY
d.local_id,
d.oa_original_id,
o.name,
o.country,
d.oa_collectedfrom,
d.reltype;
CREATE TABLE oa_conflicts (
id1 text REFERENCES organizations(id) ON UPDATE CASCADE,
id2 text REFERENCES organizations(id) ON UPDATE CASCADE,

View File

@ -12,6 +12,7 @@
<org-form-metadata org-id="{{orgId}}" org="org" vocabularies="vocabularies" info-method="getInfo()" ng-if="currentTab == 1 && org.status == 'approved'" mode="update"></org-form-metadata>
<org-form-metadata org-id="{{orgId}}" org="org" vocabularies="vocabularies" info-method="getInfo()" ng-if="currentTab == 1 && org.status == 'pending'" mode="approve"></org-form-metadata>
<org-form-metadata org-id="{{orgId}}" org="org" vocabularies="vocabularies" info-method="getInfo()" ng-if="currentTab == 1 && org.status != 'approved' && org.status != 'pending'" mode="readonly"></org-form-metadata>
<org-dedup-events org-id="{{orgId}}" events="events" vocabularies="vocabularies" info-method="getInfo()" ng-if="currentTab == 2"></org-dedup-events>
<org-dedup-events org-id="{{orgId}}" events="events" vocabularies="vocabularies" info-method="getInfo()" ng-if="currentTab == 2 && (org.status == 'approved' || org.status == 'pending')"></org-dedup-events>
</div>

View File

@ -25,7 +25,7 @@
<tr ng-repeat="o in w">
<th style="width:40px" class="text-center">#{{$index+1}}</th>
<td><a href="#!/edit/0/{{o.id}}" title="{{o.id}}">{{o.name}}</a></td>
<td style="width:250px"><img ng-src="resources/images/flags/{{o.country}}.gif" /> {{o.city}}, {{o.country}}</td>
<td style="width:250px"><img ng-src="resources/images/flags/{{o.country}}.gif" /> {{o.city || '-'}}, {{o.country}}</td>
<td style="width:80px" class="text-right">{{o.type}}</td>
</tr>
</table>

View File

@ -33,7 +33,7 @@
<a href="javascript:void(0)" title="{{d.id}}" ng-click="prepareDuplicatesModal(d)" data-toggle="modal" data-target="#duplicatesModal">{{d.name}}</a>
<a href="#!/edit/0/{{d.id}}" title="edit"><i class="fa fa-edit"></i></a>
</td>
<td class="col-3"><img ng-src="resources/images/flags/{{d.country}}.gif" /> {{d.city}}, {{d.country}}</td>
<td class="col-3"><img ng-src="resources/images/flags/{{d.country}}.gif" /> {{d.city || '-'}}, {{d.country}}</td>
<td class="col-1 text-right">{{d.numberOfDuplicates}}</td>
</tr>
</tbody>

View File

@ -15,7 +15,7 @@
<tr class="d-flex" ng-repeat="sr in conflicts">
<td class="col-6 pl-3">{{sr.name}}<br /><a href="#!/edit/0/{{sr.id}}" class="small text-monospace">{{sr.id}}</a></td>
<td class="col-2 text-center small">{{sr.type}}</td>
<td class="col-4 text-right small"><img ng-src="resources/images/flags/{{sr.country}}.gif" /> {{sr.city}}, {{sr.country}}</td>
<td class="col-4 text-right small"><img ng-src="resources/images/flags/{{sr.country}}.gif" /> {{sr.city || '-'}}, {{sr.country}}</td>
</tr>
</tbody>
</table>

View File

@ -4,8 +4,12 @@
<div ng-if="mode == 'approve'" class="alert alert-warning">
This is a pending organization. Please evaluate it before approving.
</div>
<div ng-if="mode == 'readonly'" class="alert alert-secondary">
This organization is managed by the system. You can not edit.
</div>
<fieldset>
<fieldset ng-disabled="mode == 'readonly'">
<legend>Official name and type</legend>
<div class="form-group">
@ -29,7 +33,7 @@
</div>
</div>
</fieldset>
<fieldset class="mt-4">
<fieldset class="mt-4" ng-disabled="mode == 'readonly'">
<legend>Geographical location</legend>
<div class="form-group">
@ -63,7 +67,7 @@
</div>
</fieldset>
<fieldset class="mt-4">
<fieldset class="mt-4" ng-disabled="mode == 'readonly'">
<legend>Other names and identifiers</legend>
<div class="form-group row">
@ -229,7 +233,7 @@
</div>
</fieldset>
<fieldset class="mt-4">
<fieldset class="mt-4" ng-disabled="mode == 'readonly'">
<legend>Relations</legend>
<div class="form-group row">
<div class="col-lg-8 mb-2">
@ -261,7 +265,9 @@
<input type="text" placeholder="add a related organization..." readonly="readonly"
class="form-control form-control-sm" style="background-color: white; color: #007bff;"
ng-model="newRelation.name" ng-click="resetSelectedRelation()"
data-toggle="modal" data-target="#selectRelatedOrgModal"/>
data-toggle="modal" data-target="#selectRelatedOrgModal" ng-hide="mode == 'readonly'"/>
<input type="text" placeholder="add a related organization..." disabled="disabled"
class="form-control form-control-sm" ng-model="newRelation.name" ng-show="mode == 'readonly'"/>
</td>
<td>
<select class="custom-select custom-select-sm" ng-model="newRelType">
@ -280,7 +286,7 @@
</div>
</fieldset>
<button type="submit" class="btn" ng-class="{'btn-primary' : mode != 'approve', 'btn-warning' : mode == 'approve'}" ng-click="save()" ng-disabled="organizationForm.$invalid">
<button type="submit" class="btn" ng-class="{'btn-primary' : mode != 'approve', 'btn-warning' : mode == 'approve'}" ng-click="save()" ng-disabled="organizationForm.$invalid" ng-hide="mode == 'readonly'">
<span ng-if="mode != 'approve'">Save</span>
<span ng-if="mode == 'approve'">Register as new Organization</span>
</button>

View File

@ -33,7 +33,7 @@
<td class="col-6">
<a href="#!/edit/0/{{o.id}}" title="{{o.id}}">{{o.name}}</a>
</td>
<td class="col-4"><img ng-src="resources/images/flags/{{o.country}}.gif" /> {{o.city}}, {{o.country}}</td>
<td class="col-4"><img ng-src="resources/images/flags/{{o.country}}.gif" /> {{o.city || '-'}}, {{o.country}}</td>
<td class="col-1 text-center">{{o.acronyms.join()}}</td>
<td class="col-1 text-right">{{o.type}}</td>
</tr>

View File

@ -16,7 +16,7 @@
</tr>
<tr class="d-flex">
<th class="col-4 pl-3">Place</th>
<td class="col-8">{{org.city}}, {{org.country}}</td>
<td class="col-8">{{org.city || '-'}}, {{org.country}}</td>
</tr>
<tr class="d-flex">
<th class="col-4 pl-3">Acronyms</th>

View File

@ -54,7 +54,7 @@
<a ng-if="mode != 'select-modal'" href="#!/edit/0/{{o.id}}" title="{{o.id}}">{{o.name}}</a>
<span class="badge badge-warning" ng-if="o.status != 'approved'">{{o.status}}</span>
</td>
<td class="col-4"><img ng-src="resources/images/flags/{{o.country}}.gif" /> {{o.city}}, {{o.country}}</td>
<td class="col-4"><img ng-src="resources/images/flags/{{o.country}}.gif" /> {{o.city || '-'}}, {{o.country}}</td>
<td class="col-1 text-center">{{o.acronyms.join()}}</td>
<td class="col-1 text-right">{{o.type}}</td>
</tr>