D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fixed a bug with hibernate and added api to import simrels

This commit is contained in:
Michele Artini 2020-09-28 16:53:20 +02:00
parent a28a5284fd
commit 7465134e8b
12 changed files with 177 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/dnet-orgs-database-application/pom.xml
View File

@ -1,5 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>eu.dnetlib.dhp</groupId>

6
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/HomeController.java
View File

@ -2,8 +2,6 @@ package eu.dnetlib.organizations.controller;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
@Controller
public class HomeController {
@ -28,7 +26,9 @@ public class HomeController {
return "alreadyRegistered";
}
@RequestMapping(value = { "/doc", "/swagger" }, method = RequestMethod.GET)
@GetMapping({
"/doc", "/swagger"
})
public String apiDoc() {
return "redirect:swagger-ui.html";
}

65
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java
View File

@ -1,5 +1,6 @@
package eu.dnetlib.organizations.controller;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@ -17,10 +18,11 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.security.core.Authentication;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
@ -28,13 +30,11 @@ import eu.dnetlib.organizations.model.OpenaireDuplicate;
import eu.dnetlib.organizations.model.Relationship;
import eu.dnetlib.organizations.model.utils.BrowseEntry;
import eu.dnetlib.organizations.model.utils.OrganizationConflict;
import eu.dnetlib.organizations.model.utils.OrganizationConflictImpl;
import eu.dnetlib.organizations.model.view.ConflictGroupView;
import eu.dnetlib.organizations.model.view.DuplicateGroupView;
import eu.dnetlib.organizations.model.view.OrganizationInfoView;
import eu.dnetlib.organizations.model.view.OrganizationSimpleView;
import eu.dnetlib.organizations.model.view.OrganizationView;
import eu.dnetlib.organizations.repository.OpenaireConflictRepository;
import eu.dnetlib.organizations.repository.OpenaireDuplicateRepository;
import eu.dnetlib.organizations.repository.UserCountryRepository;
import eu.dnetlib.organizations.repository.readonly.ConflictGroupViewRepository;
@ -59,8 +59,6 @@ public class OrganizationController {
@Autowired
private OpenaireDuplicateRepository openaireDuplicateRepository;
@Autowired
private OpenaireConflictRepository openaireConflictRepository;
@Autowired
private ConflictGroupViewRepository conflictGroupViewRepository;
@Autowired
private SuggestionInfoViewByCountryRepository suggestionInfoViewByCountryRepository;
@ -71,7 +69,7 @@ public class OrganizationController {
@Autowired
private DatabaseUtils databaseUtils;
@RequestMapping(value = "/save", method = RequestMethod.POST)
@PostMapping("/save")
public List<String> save(@RequestBody final OrganizationView org, final Authentication authentication) {
if (StringUtils.isBlank(org.getName())) {
@ -88,12 +86,12 @@ public class OrganizationController {
}
}
@RequestMapping(value = "/info", method = RequestMethod.GET)
@GetMapping("/info")
public OrganizationInfoView infoById(@RequestParam final String id, final Authentication authentication) {
return organizationInfoViewRepository.findById(id).get();
}
@RequestMapping(value = "/suggestionsInfo", method = RequestMethod.GET)
@GetMapping("/suggestionsInfo")
public SuggestionInfo suggestionsInfo(final Authentication authentication) {
final SuggestionInfo info = new SuggestionInfo();
@ -110,7 +108,7 @@ public class OrganizationController {
return info;
}
@RequestMapping(value = "/get", method = RequestMethod.GET)
@GetMapping("/get")
public OrganizationView findById(@RequestParam final String id, final Authentication authentication) {
final OrganizationView org = organizationViewRepository.findById(id).get();
@ -121,16 +119,16 @@ public class OrganizationController {
}
}
@RequestMapping(value = "/conflicts", method = RequestMethod.GET)
@GetMapping("/conflicts")
public List<OrganizationConflict> conflicts(@RequestParam final String id, final Authentication authentication) {
if (UserInfo.isSuperAdmin(authentication) || userCountryRepository.verifyAuthorizationForId(id, authentication.getName())) {
return openaireConflictRepository.getConflictsForId(id);
return databaseUtils.listConflictsForId(id);
} else {
throw new RuntimeException("User not authorized");
}
}
@RequestMapping(value = "/duplicates", method = RequestMethod.GET)
@GetMapping("/duplicates")
public List<OpenaireDuplicate> duplicates(@RequestParam final String id, final Authentication authentication) {
if (UserInfo.isSuperAdmin(authentication) || userCountryRepository.verifyAuthorizationForId(id, authentication.getName())) {
return openaireDuplicateRepository.findByLocalId(id);
@ -139,8 +137,8 @@ public class OrganizationController {
}
}
@RequestMapping(value = "/conflicts/byCountry/{country}", method = RequestMethod.GET)
public Collection<Set<OrganizationConflictImpl>> findConflictsByCountry(@PathVariable final String country, final Authentication authentication) {
@GetMapping("/conflicts/byCountry/{country}")
public Collection<Set<OrganizationConflict>> findConflictsByCountry(@PathVariable final String country, final Authentication authentication) {
databaseUtils.verifyConflictGroups(false);
@ -160,7 +158,7 @@ public class OrganizationController {
}
@RequestMapping(value = "/duplicates/byCountry/{country}", method = RequestMethod.GET)
@GetMapping("/duplicates/byCountry/{country}")
public Iterable<DuplicateGroupView> findDuplicatesByCountry(@PathVariable final String country, final Authentication authentication) {
if (UserInfo.isSuperAdmin(authentication)) {
@ -178,19 +176,19 @@ public class OrganizationController {
}
private Collection<Set<OrganizationConflictImpl>> groupConflicts(final Stream<ConflictGroupView> stream) {
final Map<String, Set<OrganizationConflictImpl>> map = new TreeMap<>();
private Collection<Set<OrganizationConflict>> groupConflicts(final Stream<ConflictGroupView> stream) {
final Map<String, Set<OrganizationConflict>> map = new TreeMap<>();
stream.forEach(c -> {
if (!map.containsKey(c.getGroup())) {
map.put(c.getGroup(), new TreeSet<>());
}
map.get(c.getGroup()).add(new OrganizationConflictImpl(c.getId1(), c.getName1(), c.getType1(), c.getCity1(), c.getCountry1()));
map.get(c.getGroup()).add(new OrganizationConflictImpl(c.getId2(), c.getName2(), c.getType2(), c.getCity2(), c.getCountry2()));
map.get(c.getGroup()).add(new OrganizationConflict(c.getId1(), c.getName1(), c.getType1(), c.getCity1(), c.getCountry1()));
map.get(c.getGroup()).add(new OrganizationConflict(c.getId2(), c.getName2(), c.getType2(), c.getCity2(), c.getCountry2()));
});
return map.values();
}
@RequestMapping(value = "/duplicates", method = RequestMethod.POST)
@PostMapping("/duplicates")
public List<OpenaireDuplicate> duplicates(@RequestBody final List<OpenaireDuplicate> simrels, final Authentication authentication) {
final boolean b = UserInfo.isSuperAdmin(authentication)
@ -200,13 +198,20 @@ public class OrganizationController {
.allMatch(id -> userCountryRepository.verifyAuthorizationForId(id, authentication.getName()));
if (b) {
return openaireDuplicateRepository.saveAll(simrels);
final OffsetDateTime now = OffsetDateTime.now();
final String email = authentication.getName();
final List<OpenaireDuplicate> list = openaireDuplicateRepository.saveAll(simrels);
list.forEach(d -> openaireDuplicateRepository.updateModificationDate(d.getLocalId(), d.getOaOriginalId(), email, now));
return list;
} else {
throw new RuntimeException("User not authorized");
}
}
@RequestMapping(value = "/search/{page}/{size}", method = RequestMethod.GET)
@GetMapping("/search/{page}/{size}")
public Page<OrganizationSimpleView> search(@PathVariable final int page,
@PathVariable final int size,
@RequestParam final String q,
@ -216,7 +221,7 @@ public class OrganizationController {
: organizationSimpleViewRepository.findByNameForUser(q, authentication.getName(), PageRequest.of(page, size));
}
@RequestMapping(value = "/byCountry/{code}/{page}/{size}", method = RequestMethod.GET)
@GetMapping("/byCountry/{code}/{page}/{size}")
public Page<OrganizationSimpleView> findByCountry(@PathVariable final String code,
@PathVariable final int page,
@PathVariable final int size,
@ -228,7 +233,7 @@ public class OrganizationController {
}
}
@RequestMapping(value = "/byType/{type}/{page}/{size}", method = RequestMethod.GET)
@GetMapping("/byType/{type}/{page}/{size}")
public Page<OrganizationSimpleView> findByType(@PathVariable final String type,
@PathVariable final int page,
@PathVariable final int size,
@ -238,21 +243,21 @@ public class OrganizationController {
: organizationSimpleViewRepository.findByTypeForUser(type, authentication.getName(), PageRequest.of(page, size));
}
@RequestMapping(value = "/browse/countries", method = RequestMethod.GET)
@GetMapping("/browse/countries")
public List<BrowseEntry> browseCountries(final Authentication authentication) {
return UserInfo.isSuperAdmin(authentication)
? databaseUtils.browseCountries()
: databaseUtils.browseCountriesForUser(authentication.getName());
}
@RequestMapping(value = "/browse/types", method = RequestMethod.GET)
@GetMapping("/browse/types")
public List<BrowseEntry> browseOrganizationTypes(final Authentication authentication) {
return UserInfo.isSuperAdmin(authentication)
? databaseUtils.browseTypes()
: databaseUtils.browseTypesForUser(authentication.getName());
}
@RequestMapping(value = "/conflicts/fix/{masterId}", method = RequestMethod.POST)
@PostMapping("/conflicts/fix/{masterId}")
public List<Relationship> fixConflicts(final Authentication authentication, @PathVariable final String masterId, @RequestBody final List<String> otherIds) {
if (UserInfo.isSuperAdmin(authentication) || userCountryRepository.verifyAuthorizationForId(masterId, authentication.getName())) {
@ -268,4 +273,10 @@ public class OrganizationController {
}
@GetMapping("/import/simrels")
public List<String> importSimRels() {
new Thread(databaseUtils::importSimRels).run();
return Arrays.asList("Importing...");
}
}

84
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/utils/OrganizationConflict.java
View File

@ -1,15 +1,87 @@
package eu.dnetlib.organizations.model.utils;
public interface OrganizationConflict {
import java.io.Serializable;
import java.util.Objects;
String getId();
public class OrganizationConflict implements Serializable, Comparable<OrganizationConflict> {
String getName();
/**
*
*/
private static final long serialVersionUID = 310162000309825019L;
String getType();
private String id;
private String name;
private String type;
private String city;
private String country;
String getCity();
public OrganizationConflict() {}
String getCountry();
public OrganizationConflict(final String id, final String name, final String type, final String city, final String country) {
this.id = id;
this.name = name;
this.type = type;
this.city = city;
this.country = country;
}
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(final String name) {
this.name = name;
}
public String getType() {
return type;
}
public void setType(final String type) {
this.type = type;
}
public String getCity() {
return city;
}
public void setCity(final String city) {
this.city = city;
}
public String getCountry() {
return country;
}
public void setCountry(final String country) {
this.country = country;
}
@Override
public int compareTo(final OrganizationConflict o) {
return id.compareTo(o.id);
}
@Override
public int hashCode() {
return Objects.hash(city, country, id, name, type);
}
@Override
public boolean equals(final Object obj) {
if (this == obj) { return true; }
if (!(obj instanceof OrganizationConflict)) { return false; }
final OrganizationConflict other = (OrganizationConflict) obj;
return Objects.equals(city, other.city) && Objects.equals(country, other.country) && Objects.equals(id, other.id) && Objects.equals(name, other.name)
&& Objects.equals(type, other.type);
}
}

91
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/utils/OrganizationConflictImpl.java
View File

@ -1,91 +0,0 @@
package eu.dnetlib.organizations.model.utils;
import java.util.Objects;
public class OrganizationConflictImpl implements OrganizationConflict, Comparable<OrganizationConflict> {
private String id;
private String name;
private String type;
private String city;
private String country;
public OrganizationConflictImpl() {}
public OrganizationConflictImpl(final String id) {
this.id = id;
}
public OrganizationConflictImpl(final String id, final String name, final String type, final String city, final String country) {
this.id = id;
this.name = name;
this.type = type;
this.city = city;
this.country = country;
}
@Override
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
@Override
public String getName() {
return name;
}
public void setName(final String name) {
this.name = name;
}
@Override
public String getType() {
return type;
}
public void setType(final String type) {
this.type = type;
}
@Override
public String getCity() {
return city;
}
public void setCity(final String city) {
this.city = city;
}
@Override
public String getCountry() {
return country;
}
public void setCountry(final String country) {
this.country = country;
}
@Override
public int compareTo(final OrganizationConflict o) {
return getId().compareTo(o.getId());
}
@Override
public int hashCode() {
return Objects.hash(id);
}
@Override
public boolean equals(final Object obj) {
if (this == obj) { return true; }
if (obj == null) { return false; }
if (!(obj instanceof OrganizationConflictImpl)) { return false; }
final OrganizationConflictImpl other = (OrganizationConflictImpl) obj;
return Objects.equals(id, other.id);
}
}

10
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/OpenaireConflictRepository.java
View File

@ -1,6 +1,6 @@
package eu.dnetlib.organizations.repository;
import java.util.List;
import java.time.OffsetDateTime;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
@ -9,14 +9,10 @@ import org.springframework.stereotype.Repository;
import eu.dnetlib.organizations.model.OpenaireConflict;
import eu.dnetlib.organizations.model.OpenaireConflictPK;
import eu.dnetlib.organizations.model.utils.OrganizationConflict;
@Repository
public interface OpenaireConflictRepository extends JpaRepository<OpenaireConflict, OpenaireConflictPK> {
@Query(value = "select o.id, o.name, o.type, o.city, o.country from oa_conflicts c left outer join organizations o on (c.id2 = o.id) where o.id is not null and c.id1 = ?", nativeQuery = true)
List<OrganizationConflict> getConflictsForId(String id);
Iterable<OpenaireConflict> findById1AndGroupIsNull(String id);
Iterable<OpenaireConflict> findById2AndGroupIsNull(String id);
@ -25,6 +21,10 @@ public interface OpenaireConflictRepository extends JpaRepository<OpenaireConfli
@Query(value = "update oa_conflicts set idgroup = null", nativeQuery = true)
void resetGroupIds();
@Modifying
@Query(value = "update oa_conflicts set modified_by = ?3, modification_date = ?4 where (id1 = ?1 and id2 = ?2) or (id1 = ?2 and id2 = ?1)", nativeQuery = true)
void updateModificationDate(String id1, String id2, String user, OffsetDateTime now);
long countByGroupNull();
}

8
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/OpenaireDuplicateRepository.java
View File

@ -1,8 +1,11 @@
package eu.dnetlib.organizations.repository;
import java.time.OffsetDateTime;
import java.util.List;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.stereotype.Repository;
import eu.dnetlib.organizations.model.OpenaireDuplicate;
@ -12,4 +15,9 @@ import eu.dnetlib.organizations.model.OpenaireDuplicatePK;
public interface OpenaireDuplicateRepository extends JpaRepository<OpenaireDuplicate, OpenaireDuplicatePK> {
List<OpenaireDuplicate> findByLocalId(String localId);
@Modifying
@Query(value = "update oa_duplicates set modified_by = ?3, modification_date = ?4 where (local_id = ?1 and oa_original_id = ?2) or (local_id = ?2 and oa_original_id = ?1)", nativeQuery = true)
void updateModificationDate(String id1, String id2, String user, OffsetDateTime now);
}

20
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java
View File

@ -14,6 +14,9 @@ import java.util.stream.Collectors;
import javax.transaction.Transactional;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.jdbc.core.BeanPropertyRowMapper;
@ -33,6 +36,7 @@ import eu.dnetlib.organizations.model.Url;
import eu.dnetlib.organizations.model.User;
import eu.dnetlib.organizations.model.UserCountry;
import eu.dnetlib.organizations.model.utils.BrowseEntry;
import eu.dnetlib.organizations.model.utils.OrganizationConflict;
import eu.dnetlib.organizations.model.view.OrganizationView;
import eu.dnetlib.organizations.model.view.UserView;
import eu.dnetlib.organizations.repository.AcronymRepository;
@ -69,6 +73,8 @@ public class DatabaseUtils {
@Autowired
private JdbcTemplate jdbcTemplate;
private static final Log log = LogFactory.getLog(DatabaseUtils.class);
public enum VocabularyTable {
languages,
countries,
@ -275,4 +281,18 @@ public class DatabaseUtils {
}
public List<OrganizationConflict> listConflictsForId(final String id) {
final String sql =
"select o.id, o.name, o.type, o.city, o.country from oa_conflicts c left outer join organizations o on (c.id2 = o.id) where o.id is not null and c.id1 = ?";
return jdbcTemplate.query(sql, new BeanPropertyRowMapper<>(OrganizationConflict.class), id);
}
public void importSimRels() {
try {
jdbcTemplate.update(IOUtils.toString(getClass().getResourceAsStream("/sql/importNewRels.sql")));
} catch (final Exception e) {
log.error("Error importing simrels", e);
}
}
}

12
apps/dnet-orgs-database-application/src/main/resources/sql/importNewRels.sql
View File

@ -1,15 +1,3 @@
CREATE TEMPORARY TABLE tmp_simrels (
local_id text NOT NULL,
oa_original_id text NOT NULL,
oa_name text NOT NULL,
oa_acronym text,
oa_country text,
oa_url text,
oa_collectedfrom text
);
COPY tmp_simrels(local_id, oa_original_id, oa_name, oa_acronym, oa_country, oa_url, oa_collectedfrom) FROM '/Users/michele/Develop/dnet45/dnet-orgs-database-application/src/main/resources/tmp_data/rels.part-r-00000.tsv' DELIMITER E'\t';;
DELETE FROM oa_duplicates WHERE reltype = 'suggested';
DELETE FROM oa_conflicts WHERE reltype = 'suggested';
UPDATE oa_conflicts SET idgroup = NULL;

6
apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql
View File

@ -134,6 +134,9 @@ CREATE TABLE oa_duplicates (
oa_url text,
oa_collectedfrom text,
reltype text NOT NULL DEFAULT 'suggested',
creation_date timestamp DEFAULT NOW(),
modification_date timestamp,
modified_by text,
PRIMARY KEY (local_id, oa_original_id)
);
CREATE INDEX oa_duplicates_local_id_idx ON oa_duplicates(local_id);
@ -144,6 +147,9 @@ CREATE TABLE oa_conflicts (
id2 text REFERENCES organizations(id) ON UPDATE CASCADE,
reltype text NOT NULL DEFAULT 'suggested',
idgroup text,
creation_date timestamp DEFAULT NOW(),
modification_date timestamp,
modified_by text,
PRIMARY KEY (id1, id2)
);
CREATE INDEX oa_conflicts_id1_idx ON oa_conflicts(id1);

3
apps/dnet-orgs-database-application/ssh_tunnel_postgres.sh Executable file
View File

@ -0,0 +1,3 @@
#!/bin/bash
ssh -vNL 5432:10.19.65.40:5432 michele.artini@iis-cdh5-test-gw.ocean.icm.edu.pl

2
pom.xml
View File

@ -3,7 +3,7 @@
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.3.2.RELEASE</version>
<version>2.3.4.RELEASE</version>
<relativePath />
</parent>