D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fixed a bug with hibernate and added api to import simrels

This commit is contained in:
Michele Artini 2020-09-28 16:53:20 +02:00
parent a28a5284fd
commit 7465134e8b
12 changed files with 177 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/dnet-orgs-database-application/pom.xml
View File

@ -1,5 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>eu.dnetlib.dhp</groupId>

6
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/HomeController.java
View File

@ -2,8 +2,6 @@ package eu.dnetlib.organizations.controller;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
@Controller
public class HomeController {
@ -28,7 +26,9 @@ public class HomeController {
return "alreadyRegistered";
}
@RequestMapping(value = { "/doc", "/swagger" }, method = RequestMethod.GET)
@GetMapping({
"/doc", "/swagger"
})
public String apiDoc() {
return "redirect:swagger-ui.html";
}

65
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java
View File

@ -1,5 +1,6 @@
package eu.dnetlib.organizations.controller;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@ -17,10 +18,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.security.core.Authentication;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
@ -28,13 +30,11 @@ import eu.dnetlib.organizations.model.OpenaireDuplicate;
import eu.dnetlib.organizations.model.Relationship;
import eu.dnetlib.organizations.model.utils.BrowseEntry;
import eu.dnetlib.organizations.model.utils.OrganizationConflict;
import eu.dnetlib.organizations.model.utils.OrganizationConflictImpl;
import eu.dnetlib.organizations.model.view.ConflictGroupView;
import eu.dnetlib.organizations.model.view.DuplicateGroupView;
import eu.dnetlib.organizations.model.view.OrganizationInfoView;
import eu.dnetlib.organizations.model.view.OrganizationSimpleView;
import eu.dnetlib.organizations.model.view.OrganizationView;
import eu.dnetlib.organizations.repository.OpenaireConflictRepository;
import eu.dnetlib.organizations.repository.OpenaireDuplicateRepository;
import eu.dnetlib.organizations.repository.UserCountryRepository;
import eu.dnetlib.organizations.repository.readonly.ConflictGroupViewRepository;
@ -59,8 +59,6 @@ public class OrganizationController {
@Autowired
private OpenaireDuplicateRepository openaireDuplicateRepository;
@Autowired
private OpenaireConflictRepository openaireConflictRepository;
@Autowired
private ConflictGroupViewRepository conflictGroupViewRepository;
@Autowired
private SuggestionInfoViewByCountryRepository suggestionInfoViewByCountryRepository;
@ -71,7 +69,7 @@ public class OrganizationController {
@Autowired
private DatabaseUtils databaseUtils;
@RequestMapping(value = "/save", method = RequestMethod.POST)
@PostMapping("/save")
public List<String> save(@RequestBody final OrganizationView org, final Authentication authentication) {
if (StringUtils.isBlank(org.getName())) {
@ -88,12 +86,12 @@ public class OrganizationController {
}
}
@RequestMapping(value = "/info", method = RequestMethod.GET)
@GetMapping("/info")
public OrganizationInfoView infoById(@RequestParam final String id, final Authentication authentication) {
return organizationInfoViewRepository.findById(id).get();
}
@RequestMapping(value = "/suggestionsInfo", method = RequestMethod.GET)
@GetMapping("/suggestionsInfo")
public SuggestionInfo suggestionsInfo(final Authentication authentication) {
final SuggestionInfo info = new SuggestionInfo();
@ -110,7 +108,7 @@ public class OrganizationController {
return info;
}
@RequestMapping(value = "/get", method = RequestMethod.GET)
@GetMapping("/get")
public OrganizationView findById(@RequestParam final String id, final Authentication authentication) {
final OrganizationView org = organizationViewRepository.findById(id).get();
@ -121,16 +119,16 @@ public class OrganizationController {
}
}
@RequestMapping(value = "/conflicts", method = RequestMethod.GET)
@GetMapping("/conflicts")
public List<OrganizationConflict> conflicts(@RequestParam final String id, final Authentication authentication) {
if (UserInfo.isSuperAdmin(authentication) || userCountryRepository.verifyAuthorizationForId(id, authentication.getName())) {
return openaireConflictRepository.getConflictsForId(id);
return databaseUtils.listConflictsForId(id);
} else {
throw new RuntimeException("User not authorized");
}
}
@RequestMapping(value = "/duplicates", method = RequestMethod.GET)
@GetMapping("/duplicates")
public List<OpenaireDuplicate> duplicates(@RequestParam final String id, final Authentication authentication) {
if (UserInfo.isSuperAdmin(authentication) || userCountryRepository.verifyAuthorizationForId(id, authentication.getName())) {
return openaireDuplicateRepository.findByLocalId(id);
@ -139,8 +137,8 @@ public class OrganizationController {
}
}
@RequestMapping(value = "/conflicts/byCountry/{country}", method = RequestMethod.GET)
public Collection<Set<OrganizationConflictImpl>> findConflictsByCountry(@PathVariable final String country, final Authentication authentication) {
@GetMapping("/conflicts/byCountry/{country}")
public Collection<Set<OrganizationConflict>> findConflictsByCountry(@PathVariable final String country, final Authentication authentication) {
databaseUtils.verifyConflictGroups(false);
@ -160,7 +158,7 @@ public class OrganizationController {
}
@RequestMapping(value = "/duplicates/byCountry/{country}", method = RequestMethod.GET)
@GetMapping("/duplicates/byCountry/{country}")
public Iterable<DuplicateGroupView> findDuplicatesByCountry(@PathVariable final String country, final Authentication authentication) {
if (UserInfo.isSuperAdmin(authentication)) {
@ -178,19 +176,19 @@ public class OrganizationController {
}
private Collection<Set<OrganizationConflictImpl>> groupConflicts(final Stream<ConflictGroupView> stream) {
final Map<String, Set<OrganizationConflictImpl>> map = new TreeMap<>();
private Collection<Set<OrganizationConflict>> groupConflicts(final Stream<ConflictGroupView> stream) {
final Map<String, Set<OrganizationConflict>> map = new TreeMap<>();
stream.forEach(c -> {
if (!map.containsKey(c.getGroup())) {
map.put(c.getGroup(), new TreeSet<>());
}
map.get(c.getGroup()).add(new OrganizationConflictImpl(c.getId1(), c.getName1(), c.getType1(), c.getCity1(), c.getCountry1()));
map.get(c.getGroup()).add(new OrganizationConflictImpl(c.getId2(), c.getName2(), c.getType2(), c.getCity2(), c.getCountry2()));
map.get(c.getGroup()).add(new OrganizationConflict(c.getId1(), c.getName1(), c.getType1(), c.getCity1(), c.getCountry1()));
map.get(c.getGroup()).add(new OrganizationConflict(c.getId2(), c.getName2(), c.getType2(), c.getCity2(), c.getCountry2()));
});
return map.values();
}
@RequestMapping(value = "/duplicates", method = RequestMethod.POST)
@PostMapping("/duplicates")
public List<OpenaireDuplicate> duplicates(@RequestBody final List<OpenaireDuplicate> simrels, final Authentication authentication) {
final boolean b = UserInfo.isSuperAdmin(authentication)
@ -200,13 +198,20 @@ public class OrganizationController {
.allMatch(id -> userCountryRepository.verifyAuthorizationForId(id, authentication.getName()));
if (b) {
return openaireDuplicateRepository.saveAll(simrels);
final OffsetDateTime now = OffsetDateTime.now();
final String email = authentication.getName();
final List<OpenaireDuplicate> list = openaireDuplicateRepository.saveAll(simrels);
list.forEach(d -> openaireDuplicateRepository.updateModificationDate(d.getLocalId(), d.getOaOriginalId(), email, now));
return list;
} else {
throw new RuntimeException("User not authorized");
}
}
@RequestMapping(value = "/search/{page}/{size}", method = RequestMethod.GET)
@GetMapping("/search/{page}/{size}")
public Page<OrganizationSimpleView> search(@PathVariable final int page,
@PathVariable final int size,
@RequestParam final String q,
@ -216,7 +221,7 @@ public class OrganizationController {
: organizationSimpleViewRepository.findByNameForUser(q, authentication.getName(), PageRequest.of(page, size));
}
@RequestMapping(value = "/byCountry/{code}/{page}/{size}", method = RequestMethod.GET)
@GetMapping("/byCountry/{code}/{page}/{size}")
public Page<OrganizationSimpleView> findByCountry(@PathVariable final String code,
@PathVariable final int page,
@PathVariable final int size,
@ -228,7 +233,7 @@ public class OrganizationController {
}
}
@RequestMapping(value = "/byType/{type}/{page}/{size}", method = RequestMethod.GET)
@GetMapping("/byType/{type}/{page}/{size}")
public Page<OrganizationSimpleView> findByType(@PathVariable final String type,
@PathVariable final int page,
@PathVariable final int size,
@ -238,21 +243,21 @@ public class OrganizationController {
: organizationSimpleViewRepository.findByTypeForUser(type, authentication.getName(), PageRequest.of(page, size));
}
@RequestMapping(value = "/browse/countries", method = RequestMethod.GET)
@GetMapping("/browse/countries")
public List<BrowseEntry> browseCountries(final Authentication authentication) {
return UserInfo.isSuperAdmin(authentication)
? databaseUtils.browseCountries()
: databaseUtils.browseCountriesForUser(authentication.getName());
}
@RequestMapping(value = "/browse/types", method = RequestMethod.GET)
@GetMapping("/browse/types")
public List<BrowseEntry> browseOrganizationTypes(final Authentication authentication) {
return UserInfo.isSuperAdmin(authentication)
? databaseUtils.browseTypes()
: databaseUtils.browseTypesForUser(authentication.getName());
}
@RequestMapping(value = "/conflicts/fix/{masterId}", method = RequestMethod.POST)
@PostMapping("/conflicts/fix/{masterId}")
public List<Relationship> fixConflicts(final Authentication authentication, @PathVariable final String masterId, @RequestBody final List<String> otherIds) {
if (UserInfo.isSuperAdmin(authentication) || userCountryRepository.verifyAuthorizationForId(masterId, authentication.getName())) {
@ -268,4 +273,10 @@ public class OrganizationController {
}
@GetMapping("/import/simrels")
public List<String> importSimRels() {
new Thread(databaseUtils::importSimRels).run();
return Arrays.asList("Importing...");
}
}

84
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/utils/OrganizationConflict.java
View File

@ -1,15 +1,87 @@
package eu.dnetlib.organizations.model.utils;
public interface OrganizationConflict {
import java.io.Serializable;
import java.util.Objects;
String getId();
public class OrganizationConflict implements Serializable, Comparable<OrganizationConflict> {
String getName();
/**
*
*/
private static final long serialVersionUID = 310162000309825019L;
String getType();
private String id;
private String name;
private String type;
private String city;
private String country;
String getCity();
public OrganizationConflict() {}
String getCountry();
public OrganizationConflict(final String id, final String name, final String type, final String city, final String country) {
this.id = id;
this.name = name;
this.type = type;
this.city = city;
this.country = country;
}
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(final String name) {
this.name = name;
}
public String getType() {
return type;
}
public void setType(final String type) {
this.type = type;
}
public String getCity() {
return city;
}
public void setCity(final String city) {
this.city = city;
}
public String getCountry() {
return country;
}
public void setCountry(final String country) {
this.country = country;
}
@Override
public int compareTo(final OrganizationConflict o) {
return id.compareTo(o.id);
}
@Override
public int hashCode() {
return Objects.hash(city, country, id, name, type);
}
@Override
public boolean equals(final Object obj) {
if (this == obj) { return true; }
if (!(obj instanceof OrganizationConflict)) { return false; }
final OrganizationConflict other = (OrganizationConflict) obj;
return Objects.equals(city, other.city) && Objects.equals(country, other.country) && Objects.equals(id, other.id) && Objects.equals(name, other.name)
&& Objects.equals(type, other.type);
}
}

91
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/utils/OrganizationConflictImpl.java
View File

@ -1,91 +0,0 @@
package eu.dnetlib.organizations.model.utils;
import java.util.Objects;
public class OrganizationConflictImpl implements OrganizationConflict, Comparable<OrganizationConflict> {
private String id;
private String name;
private String type;
private String city;
private String country;
public OrganizationConflictImpl() {}
public OrganizationConflictImpl(final String id) {
this.id = id;
}
public OrganizationConflictImpl(final String id, final String name, final String type, final String city, final String country) {
this.id = id;
this.name = name;
this.type = type;
this.city = city;
this.country = country;
}
@Override
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
@Override
public String getName() {
return name;
}
public void setName(final String name) {
this.name = name;
}
@Override
public String getType() {
return type;
}
public void setType(final String type) {
this.type = type;
}
@Override
public String getCity() {
return city;
}
public void setCity(final String city) {
this.city = city;
}
@Override
public String getCountry() {
return country;
}
public void setCountry(final String country) {
this.country = country;
}
@Override
public int compareTo(final OrganizationConflict o) {
return getId().compareTo(o.getId());
}
@Override
public int hashCode() {
return Objects.hash(id);
}
@Override
public boolean equals(final Object obj) {
if (this == obj) { return true; }
if (obj == null) { return false; }
if (!(obj instanceof OrganizationConflictImpl)) { return false; }
final OrganizationConflictImpl other = (OrganizationConflictImpl) obj;
return Objects.equals(id, other.id);
}
}

10
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/OpenaireConflictRepository.java
View File

@ -1,6 +1,6 @@
package eu.dnetlib.organizations.repository;
import java.util.List;
import java.time.OffsetDateTime;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
@ -9,14 +9,10 @@ import org.springframework.stereotype.Repository;
import eu.dnetlib.organizations.model.OpenaireConflict;
import eu.dnetlib.organizations.model.OpenaireConflictPK;
import eu.dnetlib.organizations.model.utils.OrganizationConflict;
@Repository
public interface OpenaireConflictRepository extends JpaRepository<OpenaireConflict, OpenaireConflictPK> {
@Query(value = "select o.id, o.name, o.type, o.city, o.country from oa_conflicts c left outer join organizations o on (c.id2 = o.id) where o.id is not null and c.id1 = ?", nativeQuery = true)
List<OrganizationConflict> getConflictsForId(String id);
Iterable<OpenaireConflict> findById1AndGroupIsNull(String id);
Iterable<OpenaireConflict> findById2AndGroupIsNull(String id);
@ -25,6 +21,10 @@ public interface OpenaireConflictRepository extends JpaRepository<OpenaireConfli
@Query(value = "update oa_conflicts set idgroup = null", nativeQuery = true)
void resetGroupIds();
@Modifying
@Query(value = "update oa_conflicts set modified_by = ?3, modification_date = ?4 where (id1 = ?1 and id2 = ?2) or (id1 = ?2 and id2 = ?1)", nativeQuery = true)
void updateModificationDate(String id1, String id2, String user, OffsetDateTime now);
long countByGroupNull();
}

8
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/OpenaireDuplicateRepository.java
View File

@ -1,8 +1,11 @@
package eu.dnetlib.organizations.repository;
import java.time.OffsetDateTime;
import java.util.List;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.stereotype.Repository;
import eu.dnetlib.organizations.model.OpenaireDuplicate;
@ -12,4 +15,9 @@ import eu.dnetlib.organizations.model.OpenaireDuplicatePK;
public interface OpenaireDuplicateRepository extends JpaRepository<OpenaireDuplicate, OpenaireDuplicatePK> {
List<OpenaireDuplicate> findByLocalId(String localId);
@Modifying
@Query(value = "update oa_duplicates set modified_by = ?3, modification_date = ?4 where (local_id = ?1 and oa_original_id = ?2) or (local_id = ?2 and oa_original_id = ?1)", nativeQuery = true)
void updateModificationDate(String id1, String id2, String user, OffsetDateTime now);
}

20
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java
View File

@ -14,6 +14,9 @@ import java.util.stream.Collectors;
import javax.transaction.Transactional;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.jdbc.core.BeanPropertyRowMapper;
@ -33,6 +36,7 @@ import eu.dnetlib.organizations.model.Url;
import eu.dnetlib.organizations.model.User;
import eu.dnetlib.organizations.model.UserCountry;
import eu.dnetlib.organizations.model.utils.BrowseEntry;
import eu.dnetlib.organizations.model.utils.OrganizationConflict;
import eu.dnetlib.organizations.model.view.OrganizationView;
import eu.dnetlib.organizations.model.view.UserView;
import eu.dnetlib.organizations.repository.AcronymRepository;
@ -69,6 +73,8 @@ public class DatabaseUtils {
@Autowired
private JdbcTemplate jdbcTemplate;
private static final Log log = LogFactory.getLog(DatabaseUtils.class);
public enum VocabularyTable {
languages,
countries,
@ -275,4 +281,18 @@ public class DatabaseUtils {
}
public List<OrganizationConflict> listConflictsForId(final String id) {
final String sql =
"select o.id, o.name, o.type, o.city, o.country from oa_conflicts c left outer join organizations o on (c.id2 = o.id) where o.id is not null and c.id1 = ?";
return jdbcTemplate.query(sql, new BeanPropertyRowMapper<>(OrganizationConflict.class), id);
}
public void importSimRels() {
try {
jdbcTemplate.update(IOUtils.toString(getClass().getResourceAsStream("/sql/importNewRels.sql")));
} catch (final Exception e) {
log.error("Error importing simrels", e);
}
}
}

12
apps/dnet-orgs-database-application/src/main/resources/sql/importNewRels.sql
View File

@ -1,15 +1,3 @@
CREATE TEMPORARY TABLE tmp_simrels (
local_id text NOT NULL,
oa_original_id text NOT NULL,
oa_name text NOT NULL,
oa_acronym text,
oa_country text,
oa_url text,
oa_collectedfrom text
);
COPY tmp_simrels(local_id, oa_original_id, oa_name, oa_acronym, oa_country, oa_url, oa_collectedfrom) FROM '/Users/michele/Develop/dnet45/dnet-orgs-database-application/src/main/resources/tmp_data/rels.part-r-00000.tsv' DELIMITER E'\t';;
DELETE FROM oa_duplicates WHERE reltype = 'suggested';
DELETE FROM oa_conflicts WHERE reltype = 'suggested';
UPDATE oa_conflicts SET idgroup = NULL;

30
apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql
View File

@ -126,24 +126,30 @@ CREATE TABLE urls (
CREATE INDEX urls_id_idx ON urls(id);
CREATE TABLE oa_duplicates (
local_id text REFERENCES organizations(id) ON UPDATE CASCADE,
oa_original_id text NOT NULL,
oa_name text NOT NULL,
oa_acronym text,
oa_country text,
oa_url text,
oa_collectedfrom text,
reltype text NOT NULL DEFAULT 'suggested',
local_id text REFERENCES organizations(id) ON UPDATE CASCADE,
oa_original_id text NOT NULL,
oa_name text NOT NULL,
oa_acronym text,
oa_country text,
oa_url text,
oa_collectedfrom text,
reltype text NOT NULL DEFAULT 'suggested',
creation_date timestamp DEFAULT NOW(),
modification_date timestamp,
modified_by text,
PRIMARY KEY (local_id, oa_original_id)
);
CREATE INDEX oa_duplicates_local_id_idx ON oa_duplicates(local_id);
CREATE TABLE oa_conflicts (
id1 text REFERENCES organizations(id) ON UPDATE CASCADE,
id2 text REFERENCES organizations(id) ON UPDATE CASCADE,
reltype text NOT NULL DEFAULT 'suggested',
idgroup text,
id1 text REFERENCES organizations(id) ON UPDATE CASCADE,
id2 text REFERENCES organizations(id) ON UPDATE CASCADE,
reltype text NOT NULL DEFAULT 'suggested',
idgroup text,
creation_date timestamp DEFAULT NOW(),
modification_date timestamp,
modified_by text,
PRIMARY KEY (id1, id2)
);
CREATE INDEX oa_conflicts_id1_idx ON oa_conflicts(id1);

3
apps/dnet-orgs-database-application/ssh_tunnel_postgres.sh Executable file
View File

@ -0,0 +1,3 @@
#!/bin/bash
ssh -vNL 5432:10.19.65.40:5432 michele.artini@iis-cdh5-test-gw.ocean.icm.edu.pl

2
pom.xml
View File

@ -3,7 +3,7 @@
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.3.2.RELEASE</version>
<version>2.3.4.RELEASE</version>
<relativePath />
</parent>