From 75d002c8ba90fdcf54d4b3154de29761e88793b6 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Fri, 23 Sep 2022 14:55:25 +0200 Subject: [PATCH 01/20] form to add persintent organizations --- .../controller/AdminController.java | 28 ++++++++ .../model/PersistentOrganization.java | 34 +++++++++ .../view/PersistentOrganizationView.java | 72 +++++++++++++++++++ .../PersistentOrganizationRepository.java | 11 +++ .../PersistentOrganizationViewRepository.java | 10 +++ .../src/main/resources/sql/schema.sql | 18 +++++ .../html/pages/admin/persistentOrgs.html | 47 ++++++++++++ .../static/resources/js/organizations.js | 21 ++++++ .../src/main/resources/templates/main.html | 2 + 9 files changed, 243 insertions(+) create mode 100644 apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/PersistentOrganization.java create mode 100644 apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/PersistentOrganizationView.java create mode 100644 apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/PersistentOrganizationRepository.java create mode 100644 apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/readonly/PersistentOrganizationViewRepository.java create mode 100644 apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/admin/persistentOrgs.html diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/AdminController.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/AdminController.java index 0e835d19..33635269 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/AdminController.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/AdminController.java @@ -20,11 +20,15 @@ import org.springframework.web.bind.annotation.RestController; import eu.dnetlib.common.controller.AbstractDnetController; import eu.dnetlib.organizations.importer.ImportExecution; import eu.dnetlib.organizations.importer.ImportExecutor; +import eu.dnetlib.organizations.model.PersistentOrganization; import eu.dnetlib.organizations.model.SystemConfiguration; import eu.dnetlib.organizations.model.utils.VocabularyTerm; +import eu.dnetlib.organizations.model.view.PersistentOrganizationView; import eu.dnetlib.organizations.model.view.UserView; +import eu.dnetlib.organizations.repository.PersistentOrganizationRepository; import eu.dnetlib.organizations.repository.SystemConfigurationRepository; import eu.dnetlib.organizations.repository.UserRepository; +import eu.dnetlib.organizations.repository.readonly.PersistentOrganizationViewRepository; import eu.dnetlib.organizations.repository.readonly.UserViewRepository; import eu.dnetlib.organizations.utils.DatabaseUtils; import eu.dnetlib.organizations.utils.MailDispatcher; @@ -38,6 +42,12 @@ public class AdminController extends AbstractDnetController { @Autowired private UserViewRepository userViewRepository; + @Autowired + private PersistentOrganizationRepository persistentOrganizationRepository; + + @Autowired + private PersistentOrganizationViewRepository persistentOrganizationViewRepository; + @Autowired private SystemConfigurationRepository systemConfigurationRepository; @@ -202,4 +212,22 @@ public class AdminController extends AbstractDnetController { throw new RuntimeException("User not authorized"); } } + + @GetMapping("/api/persistentOrgs") + public Iterable listPersistentOrgs() { + return persistentOrganizationViewRepository.findAll(); + } + + @PostMapping("/api/persistentOrgs") + public Iterable addPersistentOrgs(@RequestBody final List ids) { + ids.forEach(id -> persistentOrganizationRepository.save(new PersistentOrganization(id))); + return persistentOrganizationViewRepository.findAll(); + } + + @DeleteMapping("/api/persistentOrgs") + public Iterable deletePersistentOrgs(@RequestParam final String id) { + persistentOrganizationRepository.deleteById(id); + return persistentOrganizationViewRepository.findAll(); + } + } diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/PersistentOrganization.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/PersistentOrganization.java new file mode 100644 index 00000000..b01ea8a1 --- /dev/null +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/PersistentOrganization.java @@ -0,0 +1,34 @@ +package eu.dnetlib.organizations.model; + +import java.io.Serializable; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; + +@Entity +@Table(name = "persistent_orgs") +public class PersistentOrganization implements Serializable { + + private static final long serialVersionUID = 7684478315366015099L; + + @Id + @Column(name = "id") + private String id; + + public PersistentOrganization() {} + + public PersistentOrganization(final String id) { + this.id = id; + } + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + +} diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/PersistentOrganizationView.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/PersistentOrganizationView.java new file mode 100644 index 00000000..b9a40aac --- /dev/null +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/PersistentOrganizationView.java @@ -0,0 +1,72 @@ +package eu.dnetlib.organizations.model.view; + +import java.io.Serializable; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; + +@Entity +@Table(name = "persistent_orgs_view") +public class PersistentOrganizationView implements Serializable { + + private static final long serialVersionUID = -8906936709574708538L; + + @Id + @Column(name = "id") + private String id; + + @Column(name = "openaire_id") + private String openaireId; + + @Column(name = "name") + private String name; + + @Column(name = "city") + private String city; + + @Column(name = "country") + private String country; + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getOpenaireId() { + return openaireId; + } + + public void setOpenaireId(final String openaireId) { + this.openaireId = openaireId; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + public String getCity() { + return city; + } + + public void setCity(final String city) { + this.city = city; + } + + public String getCountry() { + return country; + } + + public void setCountry(final String country) { + this.country = country; + } + +} diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/PersistentOrganizationRepository.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/PersistentOrganizationRepository.java new file mode 100644 index 00000000..c83fbb5a --- /dev/null +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/PersistentOrganizationRepository.java @@ -0,0 +1,11 @@ +package eu.dnetlib.organizations.repository; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +import eu.dnetlib.organizations.model.PersistentOrganization; + +@Repository +public interface PersistentOrganizationRepository extends JpaRepository { + +} diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/readonly/PersistentOrganizationViewRepository.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/readonly/PersistentOrganizationViewRepository.java new file mode 100644 index 00000000..c53dd904 --- /dev/null +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/readonly/PersistentOrganizationViewRepository.java @@ -0,0 +1,10 @@ +package eu.dnetlib.organizations.repository.readonly; + +import org.springframework.stereotype.Repository; + +import eu.dnetlib.organizations.model.view.PersistentOrganizationView; + +@Repository +public interface PersistentOrganizationViewRepository extends ReadOnlyRepository { + +} diff --git a/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql b/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql index cb91a260..31ffbf5f 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql +++ b/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql @@ -5,6 +5,7 @@ DROP VIEW IF EXISTS users_view; DROP VIEW IF EXISTS conflict_groups_view; DROP VIEW IF EXISTS suggestions_info_by_country_view; DROP VIEW IF EXISTS duplicate_groups_view; +DROP VIEW IF EXISTS persistent_orgs_view; DROP TABLE IF EXISTS sysconf; DROP TABLE IF EXISTS other_ids; @@ -14,6 +15,8 @@ DROP TABLE IF EXISTS relationships; DROP TABLE IF EXISTS urls; DROP TABLE IF EXISTS oa_duplicates; DROP TABLE IF EXISTS oa_conflicts; +DROP TABLE IF EXISTS persistent_orgs; + DROP TABLE IF EXISTS organizations; DROP TABLE IF EXISTS org_types; @@ -374,6 +377,10 @@ CREATE TABLE organizations ( CREATE INDEX organizations_type_idx ON organizations(type); CREATE INDEX organizations_country_idx ON organizations(country); +CREATE TABLE persistent_orgs ( + id text PRIMARY KEY REFERENCES organizations(id) ON UPDATE CASCADE ON DELETE CASCADE +); + CREATE TABLE other_ids ( id text REFERENCES organizations(id) ON UPDATE CASCADE ON DELETE CASCADE, otherid text, @@ -674,6 +681,17 @@ WHERE GROUP BY o.id, o.name, o.city, o.country ORDER BY o.name; +CREATE VIEW persistent_orgs_view AS SELECT + po.id, + substr(po.id, 1, 14)||md5(substr(po.id,15)) as openaire_id, + o.name, + o.city, + o.country +FROM + persistent_orgs po + JOIN organizations o ON (po.id = o.id) +ORDER BY o.name; + CREATE TABLE org_index_search(id text PRIMARY KEY, txt tsvector); CREATE INDEX org_index_search_txt_gin_idx ON org_index_search USING gin(txt); diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/admin/persistentOrgs.html b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/admin/persistentOrgs.html new file mode 100644 index 00000000..406d155a --- /dev/null +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/admin/persistentOrgs.html @@ -0,0 +1,47 @@ +

Persistent Organizations

+ +

+ It is necessary to persist the identifiers of the organizations associated to an Institutional Dashboard +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IDOA Graph Node IDNamePlace
{{o.id}}{{o.openaireId}}{{o.name}} {{o.city || '-'}}, {{o.country}} + +
No persistent organizazions
+ + + +
\ No newline at end of file diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/js/organizations.js b/apps/dnet-orgs-database-application/src/main/resources/static/resources/js/organizations.js index e5ec7f74..ca08e30b 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/static/resources/js/organizations.js +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/js/organizations.js @@ -379,6 +379,7 @@ orgsModule.config(function($routeProvider) { .when('/sysconf', { templateUrl: 'resources/html/pages/admin/sysConf.html', controller: 'sysConfCtrl' }) .when('/utils', { templateUrl: 'resources/html/pages/admin/utils.html', controller: 'utilsCtrl' }) .when('/lastImport', { templateUrl: 'resources/html/pages/admin/lastImport.html', controller: 'lastImportCtrl' }) + .when('/persistentOrgs', { templateUrl: 'resources/html/pages/admin/persistentOrgs.html', controller: 'persistentOrgsCtrl' }) .otherwise({ redirectTo: '/search' }); }); @@ -806,6 +807,26 @@ orgsModule.controller('lastImportCtrl', function ($scope, $http) { $scope.refresh(); }); +orgsModule.controller('persistentOrgsCtrl', function ($scope, $http) { + $scope.orgs = {}; + + $scope.refresh = function() { + call_http_get($http, 'api/persistentOrgs', function(res) { $scope.orgs = res.data; }); + } + + $scope.addPersistentOrg = function(id) { + call_http_post($http, 'api/persistentOrgs', [ id ], function(res) { $scope.orgs = res.data; }); + } + + $scope.deletePersistentOrg = function(id) { + if (confirm("Are you sure ?")) { + call_http_delete($http, 'api/persistentOrgs?id=' + id, function(res) { $scope.orgs = res.data; }); + } + } + + $scope.refresh(); +}); + orgsModule.controller('utilsCtrl', function ($scope, $http) { $scope.fulltextIndexMessage = ''; diff --git a/apps/dnet-orgs-database-application/src/main/resources/templates/main.html b/apps/dnet-orgs-database-application/src/main/resources/templates/main.html index 461dab24..5c3e863a 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/templates/main.html +++ b/apps/dnet-orgs-database-application/src/main/resources/templates/main.html @@ -117,6 +117,8 @@ fieldset > legend { font-size : 1.2rem !important; } System utilities Last import of suggestions
+ Configure persistent organizations +
Manage users From 1ccb1c266fb9691ada7a75c4b86424a582fecbcf Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 26 Sep 2022 10:29:15 +0200 Subject: [PATCH 02/20] a control on the persistent orgs --- .../controller/AdminController.java | 19 ++++--------- .../organizations/utils/DatabaseUtils.java | 28 +++++++++++++++++++ 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/AdminController.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/AdminController.java index 33635269..943ceadf 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/AdminController.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/AdminController.java @@ -20,15 +20,12 @@ import org.springframework.web.bind.annotation.RestController; import eu.dnetlib.common.controller.AbstractDnetController; import eu.dnetlib.organizations.importer.ImportExecution; import eu.dnetlib.organizations.importer.ImportExecutor; -import eu.dnetlib.organizations.model.PersistentOrganization; import eu.dnetlib.organizations.model.SystemConfiguration; import eu.dnetlib.organizations.model.utils.VocabularyTerm; import eu.dnetlib.organizations.model.view.PersistentOrganizationView; import eu.dnetlib.organizations.model.view.UserView; -import eu.dnetlib.organizations.repository.PersistentOrganizationRepository; import eu.dnetlib.organizations.repository.SystemConfigurationRepository; import eu.dnetlib.organizations.repository.UserRepository; -import eu.dnetlib.organizations.repository.readonly.PersistentOrganizationViewRepository; import eu.dnetlib.organizations.repository.readonly.UserViewRepository; import eu.dnetlib.organizations.utils.DatabaseUtils; import eu.dnetlib.organizations.utils.MailDispatcher; @@ -42,12 +39,6 @@ public class AdminController extends AbstractDnetController { @Autowired private UserViewRepository userViewRepository; - @Autowired - private PersistentOrganizationRepository persistentOrganizationRepository; - - @Autowired - private PersistentOrganizationViewRepository persistentOrganizationViewRepository; - @Autowired private SystemConfigurationRepository systemConfigurationRepository; @@ -215,19 +206,19 @@ public class AdminController extends AbstractDnetController { @GetMapping("/api/persistentOrgs") public Iterable listPersistentOrgs() { - return persistentOrganizationViewRepository.findAll(); + return dbUtils.listPersistentOrgs(); } @PostMapping("/api/persistentOrgs") public Iterable addPersistentOrgs(@RequestBody final List ids) { - ids.forEach(id -> persistentOrganizationRepository.save(new PersistentOrganization(id))); - return persistentOrganizationViewRepository.findAll(); + ids.forEach(id -> dbUtils.addPersistentOrgs(id)); + return dbUtils.listPersistentOrgs(); } @DeleteMapping("/api/persistentOrgs") public Iterable deletePersistentOrgs(@RequestParam final String id) { - persistentOrganizationRepository.deleteById(id); - return persistentOrganizationViewRepository.findAll(); + dbUtils.deletePersistentOrgs(id); + return dbUtils.listPersistentOrgs(); } } diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java index 4e9c3c07..a5714847 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java @@ -37,6 +37,7 @@ import eu.dnetlib.organizations.model.OpenaireDuplicate; import eu.dnetlib.organizations.model.Organization; import eu.dnetlib.organizations.model.OtherIdentifier; import eu.dnetlib.organizations.model.OtherName; +import eu.dnetlib.organizations.model.PersistentOrganization; import eu.dnetlib.organizations.model.Relationship; import eu.dnetlib.organizations.model.Url; import eu.dnetlib.organizations.model.User; @@ -46,6 +47,7 @@ import eu.dnetlib.organizations.model.utils.OrganizationConflict; import eu.dnetlib.organizations.model.utils.TempBrowseEntry; import eu.dnetlib.organizations.model.utils.VocabularyTerm; import eu.dnetlib.organizations.model.view.OrganizationView; +import eu.dnetlib.organizations.model.view.PersistentOrganizationView; import eu.dnetlib.organizations.model.view.UserView; import eu.dnetlib.organizations.repository.AcronymRepository; import eu.dnetlib.organizations.repository.JournalEntryRepository; @@ -54,11 +56,13 @@ import eu.dnetlib.organizations.repository.OpenaireDuplicateRepository; import eu.dnetlib.organizations.repository.OrganizationRepository; import eu.dnetlib.organizations.repository.OtherIdentifierRepository; import eu.dnetlib.organizations.repository.OtherNameRepository; +import eu.dnetlib.organizations.repository.PersistentOrganizationRepository; import eu.dnetlib.organizations.repository.RelationshipRepository; import eu.dnetlib.organizations.repository.UrlRepository; import eu.dnetlib.organizations.repository.UserCountryRepository; import eu.dnetlib.organizations.repository.UserRepository; import eu.dnetlib.organizations.repository.readonly.OrganizationViewRepository; +import eu.dnetlib.organizations.repository.readonly.PersistentOrganizationViewRepository; @Component public class DatabaseUtils { @@ -87,6 +91,10 @@ public class DatabaseUtils { private OrganizationViewRepository organizationViewRepository; @Autowired private JournalEntryRepository journalEntryRepository; + @Autowired + private PersistentOrganizationRepository persistentOrganizationRepository; + @Autowired + private PersistentOrganizationViewRepository persistentOrganizationViewRepository; @Autowired private JdbcTemplate jdbcTemplate; @@ -585,4 +593,24 @@ public class DatabaseUtils { return jdbcTemplate.queryForList(sql, String.class); } + public Iterable listPersistentOrgs() { + return persistentOrganizationViewRepository.findAll(); + } + + public void addPersistentOrgs(final String id) { + final boolean valid = organizationRepository.findById(id) + .map(Organization::getStatus) + .filter(s -> s.equals(OrganizationStatus.approved.toString())) + .isPresent(); + if (valid) { + persistentOrganizationRepository.save(new PersistentOrganization(id)); + } else { + throw new RuntimeException("The ID does not refer to an approved Organization"); + } + } + + public void deletePersistentOrgs(final String id) { + persistentOrganizationRepository.deleteById(id); + } + } From 98b59cd0d5a4873ee4530470dcdf21dd7ec120ae Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 26 Sep 2022 14:16:22 +0200 Subject: [PATCH 03/20] ui --- .../controller/OrganizationController.java | 11 +---- .../model/utils/OpenaireGraphNode.java | 43 +++++++++++++------ .../model/view/OpenaireDuplicateView.java | 29 +------------ .../model/view/OrganizationInfoView.java | 29 +------------ .../view/PersistentOrganizationView.java | 15 ++----- .../src/main/resources/application.properties | 2 - .../src/main/resources/sql/schema.sql | 17 ++++++-- .../resources/html/pages/edit/edit.html | 2 +- .../resources/html/parts/org_duplicates.html | 2 +- 9 files changed, 53 insertions(+), 97 deletions(-) diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java index 9ecd3cca..eb706c5f 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/OrganizationController.java @@ -17,7 +17,6 @@ import javax.servlet.http.HttpServletResponse; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; import org.springframework.security.core.Authentication; @@ -83,8 +82,6 @@ public class OrganizationController extends AbstractDnetController { private JournalEntryRepository journalEntryRepository; @Autowired private DatabaseUtils databaseUtils; - @Value("${openaire.explore.organization.baseurl}") - private String oaBaseUrl; @PostMapping("/save") public List save(@RequestBody final OrganizationView org, final Authentication authentication) { @@ -107,9 +104,7 @@ public class OrganizationController extends AbstractDnetController { @GetMapping("/info") public OrganizationInfoView infoById(@RequestParam final String id, final Authentication authentication) { - final OrganizationInfoView info = organizationInfoViewRepository.findById(id).get(); - info.fillGraphNodeInfo(info.getId(), oaBaseUrl); - return info; + return organizationInfoViewRepository.findById(id).get(); } @GetMapping("/suggestionsInfo") @@ -160,9 +155,7 @@ public class OrganizationController extends AbstractDnetController { } private List listDuplicates(final String id) { - final List list = openaireDuplicateViewRepository.findByLocalId(id); - list.forEach(d -> d.fillGraphNodeInfo(d.getOaOriginalId(), oaBaseUrl)); - return list; + return openaireDuplicateViewRepository.findByLocalId(id); } @GetMapping("/conflicts/byCountry/{country}") diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/utils/OpenaireGraphNode.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/utils/OpenaireGraphNode.java index 6867942b..61982659 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/utils/OpenaireGraphNode.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/utils/OpenaireGraphNode.java @@ -1,23 +1,42 @@ package eu.dnetlib.organizations.model.utils; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.lang3.StringUtils; +import javax.persistence.Column; +import javax.persistence.MappedSuperclass; -public interface OpenaireGraphNode { +@MappedSuperclass +public abstract class OpenaireGraphNode { - String getOaGraphId(); + @Column(name = "openaire_id") + private String openaireId; - void setOaGraphId(String oaGraphId); + @Column(name = "openaire_url") + private String openaireUrl; - String getOaGraphUrl(); + @Column(name = "openaire_persistent") + private Boolean persistent = false; - void setOaGraphUrl(String oaGraphUrl); + public String getOpenaireId() { + return openaireId; + } - default void fillGraphNodeInfo(final String origId, final String baseUrl) { - final String oaGraphId = StringUtils.substringBefore(origId, "::") + "::" - + DigestUtils.md5Hex(StringUtils.substringAfter(origId, "::")); - setOaGraphId(oaGraphId); - setOaGraphUrl(baseUrl + oaGraphId); + public void setOpenaireId(final String openaireId) { + this.openaireId = openaireId; + } + + public String getOpenaireUrl() { + return openaireUrl; + } + + public void setOpenaireUrl(final String openaireUrl) { + this.openaireUrl = openaireUrl; + } + + public Boolean getPersistent() { + return persistent; + } + + public void setPersistent(final Boolean persistent) { + this.persistent = persistent; } } diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/OpenaireDuplicateView.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/OpenaireDuplicateView.java index 2a02ed35..9cfba1a3 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/OpenaireDuplicateView.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/OpenaireDuplicateView.java @@ -8,7 +8,6 @@ import javax.persistence.Entity; import javax.persistence.Id; import javax.persistence.IdClass; import javax.persistence.Table; -import javax.persistence.Transient; import org.hibernate.annotations.Type; @@ -18,7 +17,7 @@ import eu.dnetlib.organizations.model.utils.OpenaireGraphNode; @Entity @Table(name = "oa_duplicates_view") @IdClass(OpenaireDuplicatePK.class) -public class OpenaireDuplicateView implements Serializable, OpenaireGraphNode { +public class OpenaireDuplicateView extends OpenaireGraphNode implements Serializable { /** * @@ -88,12 +87,6 @@ public class OpenaireDuplicateView implements Serializable, OpenaireGraphNode { @Column(name = "ec_nutscode") private Boolean ecNutscode; - @Transient - private String oaGraphId; - - @Transient - private String oaGraphUrl; - public String getLocalId() { return localId; } @@ -254,24 +247,4 @@ public class OpenaireDuplicateView implements Serializable, OpenaireGraphNode { this.ecNutscode = ecNutscode; } - @Override - public String getOaGraphId() { - return oaGraphId; - } - - @Override - public void setOaGraphId(final String oaGraphId) { - this.oaGraphId = oaGraphId; - } - - @Override - public String getOaGraphUrl() { - return oaGraphUrl; - } - - @Override - public void setOaGraphUrl(final String oaGraphUrl) { - this.oaGraphUrl = oaGraphUrl; - } - } diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/OrganizationInfoView.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/OrganizationInfoView.java index 7109d8a1..3ca41df6 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/OrganizationInfoView.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/OrganizationInfoView.java @@ -7,13 +7,12 @@ import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Id; import javax.persistence.Table; -import javax.persistence.Transient; import eu.dnetlib.organizations.model.utils.OpenaireGraphNode; @Entity @Table(name = "organizations_info_view") -public class OrganizationInfoView implements Serializable, OpenaireGraphNode { +public class OrganizationInfoView extends OpenaireGraphNode implements Serializable { /** * @@ -48,12 +47,6 @@ public class OrganizationInfoView implements Serializable, OpenaireGraphNode { @Column(name = "note") private boolean note; - @Transient - private String oaGraphId; - - @Transient - private String oaGraphUrl; - public String getId() { return id; } @@ -126,24 +119,4 @@ public class OrganizationInfoView implements Serializable, OpenaireGraphNode { this.note = note; } - @Override - public String getOaGraphId() { - return oaGraphId; - } - - @Override - public void setOaGraphId(final String oaGraphId) { - this.oaGraphId = oaGraphId; - } - - @Override - public String getOaGraphUrl() { - return oaGraphUrl; - } - - @Override - public void setOaGraphUrl(final String oaGraphUrl) { - this.oaGraphUrl = oaGraphUrl; - } - } diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/PersistentOrganizationView.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/PersistentOrganizationView.java index b9a40aac..872fa06a 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/PersistentOrganizationView.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/PersistentOrganizationView.java @@ -7,9 +7,11 @@ import javax.persistence.Entity; import javax.persistence.Id; import javax.persistence.Table; +import eu.dnetlib.organizations.model.utils.OpenaireGraphNode; + @Entity @Table(name = "persistent_orgs_view") -public class PersistentOrganizationView implements Serializable { +public class PersistentOrganizationView extends OpenaireGraphNode implements Serializable { private static final long serialVersionUID = -8906936709574708538L; @@ -17,9 +19,6 @@ public class PersistentOrganizationView implements Serializable { @Column(name = "id") private String id; - @Column(name = "openaire_id") - private String openaireId; - @Column(name = "name") private String name; @@ -37,14 +36,6 @@ public class PersistentOrganizationView implements Serializable { this.id = id; } - public String getOpenaireId() { - return openaireId; - } - - public void setOpenaireId(final String openaireId) { - this.openaireId = openaireId; - } - public String getName() { return name; } diff --git a/apps/dnet-orgs-database-application/src/main/resources/application.properties b/apps/dnet-orgs-database-application/src/main/resources/application.properties index 6425e130..7f2e92a9 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/application.properties +++ b/apps/dnet-orgs-database-application/src/main/resources/application.properties @@ -50,5 +50,3 @@ openaire.api.https.proxy = 10.19.65.35 openorgs.support.pages = { "Ask a question": "https://www.openaire.eu/support/helpdesk?view=ticket&layout=open", "FAQ": "https://www.openaire.eu/faqs" } openaire.override.logout.url = - -openaire.explore.organization.baseurl = https://explore.openaire.eu/search/organization?organizationId= diff --git a/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql b/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql index 31ffbf5f..c3c0351e 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql +++ b/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql @@ -471,7 +471,10 @@ CREATE VIEW oa_duplicates_view AS o.ec_internationalorganization, o.ec_enterprise, o.ec_smevalidated, - o.ec_nutscode + o.ec_nutscode, + substr(d.oa_original_id, 1, 14)||md5(substr(d.oa_original_id, 15)) as openaire_id, + 'https://explore.openaire.eu/search/organization?organizationId='||substr(d.oa_original_id, 1, 14)||md5(substr(d.oa_original_id, 15)) as openaire_url, + false as openaire_persistent FROM oa_duplicates d LEFT OUTER JOIN organizations o ON (o.id = d.oa_original_id) @@ -571,13 +574,17 @@ CREATE VIEW organizations_info_view AS SELECT org.creation_date, org.modified_by, org.modification_date, + substr(org.id, 1, 14)||md5(substr(org.id, 15)) as openaire_id, + 'https://explore.openaire.eu/search/organization?organizationId='||substr(org.id, 1, 14)||md5(substr(org.id, 15)) as openaire_url, + count(po.id) > 0 as openaire_persistent, count(DISTINCT d.oa_original_id) as n_duplicates, count(DISTINCT c.id2) as n_conflicts, count(DISTINCT n.note) > 0 as note FROM organizations org - LEFT OUTER JOIN oa_duplicates d ON (org.id = d.local_id AND d.reltype = 'suggested') - LEFT OUTER JOIN oa_conflicts c ON (org.id = c.id1 AND c.reltype = 'suggested') - LEFT OUTER JOIN notes n ON (org.id = n.id) + LEFT OUTER JOIN oa_duplicates d ON (org.id = d.local_id AND d.reltype = 'suggested') + LEFT OUTER JOIN oa_conflicts c ON (org.id = c.id1 AND c.reltype = 'suggested') + LEFT OUTER JOIN notes n ON (org.id = n.id) + LEFT OUTER JOIN persistent_orgs po ON (org.id = po.id) GROUP BY org.id; CREATE VIEW organizations_simple_view AS SELECT @@ -684,6 +691,8 @@ ORDER BY o.name; CREATE VIEW persistent_orgs_view AS SELECT po.id, substr(po.id, 1, 14)||md5(substr(po.id,15)) as openaire_id, + 'https://explore.openaire.eu/search/organization?organizationId='||substr(po.id, 1, 14)||md5(substr(po.id, 15)) as openaire_url, + true as openaire_persistent, o.name, o.city, o.country diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/edit/edit.html b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/edit/edit.html index 5d7c1599..25143870 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/edit/edit.html +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/edit/edit.html @@ -6,7 +6,7 @@ ID: {{info.id}}
Created at {{info.creationDate | date:'MMMM d, y HH:mm:ss'}} by {{info.createdBy}}
Modified at {{info.modificationDate | date:'MMMM d, y HH:mm:ss'}} by {{info.modifiedBy}}
- OA Graph Node ID: {{info.oaGraphId}} [try on OA Explore]
+ OA Graph Node ID: {{info.openaireId}} [try on OA Explore]

diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/org_duplicates.html b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/org_duplicates.html index 48f79291..17b20312 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/org_duplicates.html +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/org_duplicates.html @@ -36,7 +36,7 @@ {{sr.oaCountry}} Original Id: {{sr.oaOriginalId}}
- OA Graph Node ID: {{sr.oaGraphId}} [try] + OA Graph Node ID: {{sr.openaireId}} [try]
Provenance: {{sr.oaCollectedFrom}}
Added by: {{sr.createdBy}} From 5523ab8657367be4d36fcb60dcc51f7ddd25c7bb Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 26 Sep 2022 14:50:23 +0200 Subject: [PATCH 04/20] updated views --- .../organizations/controller/HomeController.java | 9 +++++++++ .../organizations/model/utils/OpenaireGraphNode.java | 11 ----------- .../src/main/resources/application.properties | 2 ++ .../src/main/resources/sql/schema.sql | 3 --- .../static/resources/html/pages/edit/edit.html | 4 +++- .../static/resources/html/parts/org_duplicates.html | 2 +- 6 files changed, 15 insertions(+), 16 deletions(-) diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/HomeController.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/HomeController.java index db337ffd..7d76fb8d 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/HomeController.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/HomeController.java @@ -11,6 +11,7 @@ import org.springframework.security.core.Authentication; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.ModelAttribute; +import org.springframework.web.bind.annotation.PathVariable; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; @@ -30,6 +31,9 @@ public class HomeController extends AbstractDnetController { @Autowired private SystemConfigurationRepository systemConfigurationRepository; + @Value("${openaire.explore.organization.baseurl}") + private String openaireBaseUrl; + @Value("${openorgs.support.pages}") private String supportPagesJson; @@ -40,6 +44,11 @@ public class HomeController extends AbstractDnetController { return env.acceptsProfiles(Profiles.of("dev")) ? "redirect:main" : "home"; } + @GetMapping("/redirect/oa/{orgId}") + public String openaireUrl(@PathVariable final String orgId) { + return "redirect:" + String.format(openaireBaseUrl, orgId); + } + @GetMapping("/main") public String main() { return "main"; diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/utils/OpenaireGraphNode.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/utils/OpenaireGraphNode.java index 61982659..1b34ab01 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/utils/OpenaireGraphNode.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/utils/OpenaireGraphNode.java @@ -9,9 +9,6 @@ public abstract class OpenaireGraphNode { @Column(name = "openaire_id") private String openaireId; - @Column(name = "openaire_url") - private String openaireUrl; - @Column(name = "openaire_persistent") private Boolean persistent = false; @@ -23,14 +20,6 @@ public abstract class OpenaireGraphNode { this.openaireId = openaireId; } - public String getOpenaireUrl() { - return openaireUrl; - } - - public void setOpenaireUrl(final String openaireUrl) { - this.openaireUrl = openaireUrl; - } - public Boolean getPersistent() { return persistent; } diff --git a/apps/dnet-orgs-database-application/src/main/resources/application.properties b/apps/dnet-orgs-database-application/src/main/resources/application.properties index 7f2e92a9..83038eca 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/application.properties +++ b/apps/dnet-orgs-database-application/src/main/resources/application.properties @@ -50,3 +50,5 @@ openaire.api.https.proxy = 10.19.65.35 openorgs.support.pages = { "Ask a question": "https://www.openaire.eu/support/helpdesk?view=ticket&layout=open", "FAQ": "https://www.openaire.eu/faqs" } openaire.override.logout.url = + +openaire.explore.organization.baseurl = https://explore.openaire.eu/search/organization?organizationId=%s diff --git a/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql b/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql index c3c0351e..b0469ac0 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql +++ b/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql @@ -473,7 +473,6 @@ CREATE VIEW oa_duplicates_view AS o.ec_smevalidated, o.ec_nutscode, substr(d.oa_original_id, 1, 14)||md5(substr(d.oa_original_id, 15)) as openaire_id, - 'https://explore.openaire.eu/search/organization?organizationId='||substr(d.oa_original_id, 1, 14)||md5(substr(d.oa_original_id, 15)) as openaire_url, false as openaire_persistent FROM oa_duplicates d @@ -575,7 +574,6 @@ CREATE VIEW organizations_info_view AS SELECT org.modified_by, org.modification_date, substr(org.id, 1, 14)||md5(substr(org.id, 15)) as openaire_id, - 'https://explore.openaire.eu/search/organization?organizationId='||substr(org.id, 1, 14)||md5(substr(org.id, 15)) as openaire_url, count(po.id) > 0 as openaire_persistent, count(DISTINCT d.oa_original_id) as n_duplicates, count(DISTINCT c.id2) as n_conflicts, @@ -691,7 +689,6 @@ ORDER BY o.name; CREATE VIEW persistent_orgs_view AS SELECT po.id, substr(po.id, 1, 14)||md5(substr(po.id,15)) as openaire_id, - 'https://explore.openaire.eu/search/organization?organizationId='||substr(po.id, 1, 14)||md5(substr(po.id, 15)) as openaire_url, true as openaire_persistent, o.name, o.city, diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/edit/edit.html b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/edit/edit.html index 25143870..9cac6d11 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/edit/edit.html +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/edit/edit.html @@ -6,7 +6,9 @@ ID: {{info.id}}
Created at {{info.creationDate | date:'MMMM d, y HH:mm:ss'}} by {{info.createdBy}}
Modified at {{info.modificationDate | date:'MMMM d, y HH:mm:ss'}} by {{info.modifiedBy}}
- OA Graph Node ID: {{info.openaireId}} [try on OA Explore]
+ OA Graph Node ID: {{info.openaireId}} [try on OA Explore] + persistent +

diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/org_duplicates.html b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/org_duplicates.html index 17b20312..23c22e88 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/org_duplicates.html +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/org_duplicates.html @@ -36,7 +36,7 @@ {{sr.oaCountry}} Original Id: {{sr.oaOriginalId}}
- OA Graph Node ID: {{sr.openaireId}} [try] + OA Graph Node ID: {{sr.openaireId}} [try]
Provenance: {{sr.oaCollectedFrom}}
Added by: {{sr.createdBy}} From c990fc945bb7b36dfe39b6a832d9904caba0e806 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Tue, 27 Sep 2022 12:03:56 +0200 Subject: [PATCH 05/20] ui --- .../eu/dnetlib/organizations/model/view/OrganizationView.java | 4 +++- .../src/main/resources/sql/schema.sql | 3 +++ .../resources/static/resources/html/parts/org_details.html | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/OrganizationView.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/OrganizationView.java index 06d92519..cdf41c94 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/OrganizationView.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/model/view/OrganizationView.java @@ -15,13 +15,15 @@ import org.hibernate.annotations.TypeDefs; import com.vladmihalcea.hibernate.type.json.JsonBinaryType; import com.vladmihalcea.hibernate.type.json.JsonStringType; +import eu.dnetlib.organizations.model.utils.OpenaireGraphNode; + @Entity @Table(name = "organizations_view") @TypeDefs({ @TypeDef(name = "json", typeClass = JsonStringType.class), @TypeDef(name = "jsonb", typeClass = JsonBinaryType.class) }) -public class OrganizationView implements Serializable { +public class OrganizationView extends OpenaireGraphNode implements Serializable { /** * diff --git a/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql b/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql index b0469ac0..50c9e0f9 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql +++ b/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql @@ -533,6 +533,8 @@ CREATE VIEW organizations_view AS SELECT org.ec_enterprise, org.ec_smevalidated, org.ec_nutscode, + substr(org.id, 1, 14)||md5(substr(org.id, 15)) as openaire_id, + count(po.id) > 0 as openaire_persistent, COALESCE(jsonb_agg(DISTINCT jsonb_build_object('id', oid.otherid, 'type', oid.type)) FILTER (WHERE oid.otherid IS NOT NULL), '[]') AS other_ids, COALESCE(jsonb_agg(DISTINCT jsonb_build_object('name', n.name, 'lang', n.lang)) FILTER (WHERE n.name IS NOT NULL), '[]') AS other_names, COALESCE(jsonb_agg(DISTINCT a.acronym) FILTER (WHERE a.acronym IS NOT NULL), '[]') AS acronyms, @@ -546,6 +548,7 @@ FROM LEFT OUTER JOIN urls u ON (org.id = u.id) LEFT OUTER JOIN relationships r ON (org.id = r.id1) LEFT OUTER JOIN organizations relorg ON (relorg.id = r.id2) + LEFT OUTER JOIN persistent_orgs po ON (org.id = po.id) GROUP BY org.id, org.name, diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/org_details.html b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/org_details.html index 7452777d..c5751c6e 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/org_details.html +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/parts/org_details.html @@ -4,7 +4,7 @@ 'text-white bg-success' : show == 'success', 'text-white bg-info' : show == 'info', 'bg-secondary' : show == 'secondary', - }">{{orgTitle}}
+ }">{{orgTitle}} persistent ID
From 7a2ce6b2ad9ecedda59749c186f5dfe2fda11387 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 10 Oct 2022 11:13:52 +0200 Subject: [PATCH 06/20] multisearch for eoscdatatasourcetypes --- .../openaire/dsm/dao/DatasourceSpecs.java | 28 +++++++++++++++++++ .../openaire/dsm/domain/FilterName.java | 3 +- .../openaire/dsm/domain/FilterType.java | 5 +++- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceSpecs.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceSpecs.java index 7fc5958d..dcbb9dba 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceSpecs.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceSpecs.java @@ -1,5 +1,6 @@ package eu.dnetlib.openaire.dsm.dao; +import java.util.Arrays; import java.util.List; import java.util.Map.Entry; @@ -55,6 +56,11 @@ public class DatasourceSpecs { case search: expressions.add(likeSearch(ds, cb, e)); + break; + + case multiSearch: + expressions.add(likeSearchList(ds, cb, e)); + break; case searchOrgs: // search by case insensitive organization's country @@ -100,6 +106,11 @@ public class DatasourceSpecs { expressions.add(likeSearch(api, cb, e)); break; + + case multiSearch: + expressions.add(likeSearchList(api, cb, e)); + + break; case searchOrgs: throw new DsmRuntimeException("not implemented"); } @@ -117,6 +128,23 @@ public class DatasourceSpecs { return cb.like(cb.lower(r.get(e.getKey().name())), WILDCARD + getValue(e) + WILDCARD); } + // substring, case insensitive, like based search + private static Predicate likeSearchList(final Root r, final CriteriaBuilder cb, final Entry e) { + final String key = e.getKey().name(); + + log.info("key : " + key); + log.info("val : " + e.getValue()); + + final Predicate[] arr = + Arrays.stream(e.getValue().toString().split(",")) + .map(String::toLowerCase) + .map(s -> cb.like(cb.lower(r.get(key)), WILDCARD + s + WILDCARD)) + .toArray(size -> new Predicate[size]); + + return cb.or(arr); + + } + // search by ID, managed. exact match private static Predicate exactSearch(final Root r, final CriteriaBuilder cb, final Entry e) { return cb.equal(r.get(e.getKey().name()), getValue(e)); diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterName.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterName.java index c929d920..2dbd67eb 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterName.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterName.java @@ -33,9 +33,10 @@ public enum FilterName { case contactemail: case registeredby: case typology: - case eoscDatasourceType: case platform: return FilterType.search; + case eoscDatasourceType: + return FilterType.multiSearch; case country: return FilterType.searchOrgs; default: diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterType.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterType.java index 5d15adfd..a028e350 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterType.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/FilterType.java @@ -1,5 +1,8 @@ package eu.dnetlib.openaire.dsm.domain; public enum FilterType { - exact, search, searchOrgs + exact, + search, + searchOrgs, + multiSearch } From bf117211594ba3c449d3b1e83b85028ee77add8b Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 10 Oct 2022 11:20:27 +0200 Subject: [PATCH 07/20] some filter conditions --- .../java/eu/dnetlib/openaire/dsm/dao/DatasourceSpecs.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceSpecs.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceSpecs.java index dcbb9dba..ba110c16 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceSpecs.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/DatasourceSpecs.java @@ -132,17 +132,15 @@ public class DatasourceSpecs { private static Predicate likeSearchList(final Root r, final CriteriaBuilder cb, final Entry e) { final String key = e.getKey().name(); - log.info("key : " + key); - log.info("val : " + e.getValue()); - final Predicate[] arr = Arrays.stream(e.getValue().toString().split(",")) + .map(String::trim) .map(String::toLowerCase) + .filter(StringUtils::isNotBlank) .map(s -> cb.like(cb.lower(r.get(key)), WILDCARD + s + WILDCARD)) .toArray(size -> new Predicate[size]); return cb.or(arr); - } // search by ID, managed. exact match From c4c71790097677c905e3542642d2a2d34dfd51ee Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Tue, 11 Oct 2022 10:40:31 +0200 Subject: [PATCH 08/20] conflicts with a persistent org --- .../organizations/utils/DatabaseUtils.java | 121 +++++++++++++----- .../utils/JournalOperations.java | 1 + 2 files changed, 90 insertions(+), 32 deletions(-) diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java index a5714847..27517518 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java @@ -12,10 +12,12 @@ import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Collectors; import javax.transaction.Transactional; +import org.apache.commons.beanutils.BeanUtils; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; @@ -476,45 +478,94 @@ public class DatabaseUtils { @Transactional public String fixConflictSimilars(final List similarIds, final String user) { - final OffsetDateTime now = OffsetDateTime.now(); - final List views = similarIds.stream().map(organizationViewRepository::findById).filter(Optional::isPresent).map(Optional::get).collect(Collectors.toList()); - // I create a new org - final OrganizationView newOrg = new OrganizationView(); - newOrg.setId(null); - newOrg.setStatus(null); - newOrg.setName(findFirstString(views, OrganizationView::getName)); - newOrg.setType(findFirstString(views, OrganizationView::getType)); - newOrg.setLat(findFirstNumber(views, OrganizationView::getLat)); - newOrg.setLng(findFirstNumber(views, OrganizationView::getLng)); - newOrg.setCity(findFirstString(views, OrganizationView::getCity)); - newOrg.setCountry(findFirstString(views, OrganizationView::getCountry)); - newOrg.setOtherIdentifiers(findAll(views, OrganizationView::getOtherIdentifiers)); - newOrg.setOtherNames(findAll(views, OrganizationView::getOtherNames)); - newOrg.setAcronyms(findAll(views, OrganizationView::getAcronyms)); - newOrg.setUrls(findAll(views, OrganizationView::getUrls)); - newOrg.setRelations(findAll(views, OrganizationView::getRelations)); + final List persistents = views.stream().filter(v -> v.getPersistent()).collect(Collectors.toList()); - newOrg.getOtherNames() - .addAll(views.stream() + if (persistents.size() > 1) { + throw new RuntimeException("Too many persintent organizations"); + } else if (persistents.size() > 1) { + backupOrg(persistents.get(0), user); + return fixConflicts(persistents.get(0), views, user); + } else { + // I create a new org + final OrganizationView masterOrg = new OrganizationView(); + masterOrg.setId(null); + masterOrg.setStatus(null); + return fixConflicts(masterOrg, views, user); + } + } + + private String backupOrg(final OrganizationView org, final String user) { + final String origId = org.getId(); + final String backupId = origId + "::" + OffsetDateTime.now().toEpochSecond(); + + try { + + final OrganizationView backupOrg = (OrganizationView) BeanUtils.cloneBean(org); + backupOrg.setId(backupId); + backupOrg.setStatus(OrganizationStatus.hidden.toString()); + + insertOrUpdateOrganization(backupOrg, user, false); + + journalEntryRepository + .save(new JournalEntry(origId, JournalOperations.BACKUP_ORG, "Saved a backup copy: " + backupId, user)); + journalEntryRepository + .save(new JournalEntry(backupId, JournalOperations.BACKUP_ORG, "Saved a backup copy of " + origId, user)); + + return backupId; + } catch (final Exception e) { + log.error("Error performing the backup of " + origId, e); + throw new RuntimeException("Error performing the backup of " + origId, e); + } + } + + private String fixConflicts(final OrganizationView masterOrg, final List orgs, final String user) { + + final OffsetDateTime now = OffsetDateTime.now(); + + final String finalMessage = (masterOrg.getId() == null ? "New org created merging: " : "Merging in persistent org: ") + + orgs.stream() + .map(OrganizationView::getId) + .collect(Collectors.joining(", ")); + + masterOrg.setName(findFirstString(orgs, OrganizationView::getName)); + masterOrg.setType(findFirstString(orgs, OrganizationView::getType)); + masterOrg.setLat(findFirstNumber(orgs, OrganizationView::getLat)); + masterOrg.setLng(findFirstNumber(orgs, OrganizationView::getLng)); + masterOrg.setCity(findFirstString(orgs, OrganizationView::getCity)); + masterOrg.setCountry(findFirstString(orgs, OrganizationView::getCountry)); + masterOrg.setOtherIdentifiers(findAll(orgs, OrganizationView::getOtherIdentifiers)); + masterOrg.setOtherNames(findAll(orgs, OrganizationView::getOtherNames)); + masterOrg.setAcronyms(findAll(orgs, OrganizationView::getAcronyms)); + masterOrg.setUrls(findAll(orgs, OrganizationView::getUrls)); + masterOrg.setRelations(findAll(orgs, OrganizationView::getRelations, r -> !r.getType().equals(RelationType.Merged_In.toString()) + && !r.getType().equals(RelationType.Merged_In.toString()))); + + masterOrg.getOtherNames() + .addAll(orgs.stream() .map(OrganizationView::getName) .filter(StringUtils::isNotBlank) - .filter(s -> StringUtils.equalsIgnoreCase(s, newOrg.getName())) + .filter(s -> StringUtils.equalsIgnoreCase(s, masterOrg.getName())) .map(s -> new eu.dnetlib.organizations.model.view.OtherName(s, "UNKNOWN")) .collect(Collectors.toList())); - final String masterId = insertOrUpdateOrganization(newOrg, user, false); + final String masterId = insertOrUpdateOrganization(masterOrg, user, false); // I hide the merged organizations - similarIds.forEach(id -> { - hideConflictOrgs(masterId, id); - journalEntryRepository.save(new JournalEntry(masterId, JournalOperations.FIX_CONFLICT, "The org has been hidded and merged in " + masterId, user)); - }); + orgs.stream() + .map(OrganizationView::getId) + .filter(id -> !id.equals(masterId)) + .forEach(id -> { + hideConflictOrgs(masterId, id); + journalEntryRepository + .save(new JournalEntry(masterId, JournalOperations.FIX_CONFLICT, "The org has been hidded and merged in " + masterId, user)); + }); // I reassign the duplicates to the new org - final List newDuplicates = similarIds.stream() + final List newDuplicates = orgs.stream() + .map(OrganizationView::getId) .map(openaireDuplicateRepository::findByLocalId) .flatMap(l -> l.stream()) .map(d -> new OpenaireDuplicate(masterId, d.getOaOriginalId(), d.getRelType(), d.getOaCollectedFrom())) @@ -526,18 +577,20 @@ public class DatabaseUtils { openaireDuplicateRepository.updateModificationDate(d.getLocalId(), d.getOaOriginalId(), user, now); }); - for (final String similarId : similarIds) { + orgs.forEach(org -> { + final String similarId = org.getId(); openaireConflictRepository.updateMultipleStatusAndResetGroup(similarId, SimilarityType.is_different.toString(), user, now); - } + }); - for (int i = 0; i < similarIds.size(); i++) { - for (int j = i + 1; j < similarIds.size(); j++) { - openaireConflictRepository.updateStatusAndResetGroup(similarIds.get(i), similarIds.get(j), SimilarityType.is_similar.toString(), user, now); + for (int i = 0; i < orgs.size(); i++) { + for (int j = i + 1; j < orgs.size(); j++) { + openaireConflictRepository + .updateStatusAndResetGroup(orgs.get(i).getId(), orgs.get(j).getId(), SimilarityType.is_similar.toString(), user, now); } } journalEntryRepository - .save(new JournalEntry(masterId, JournalOperations.FIX_CONFLICT, "New org created merging: " + StringUtils.join(similarIds, ", "), user)); + .save(new JournalEntry(masterId, JournalOperations.FIX_CONFLICT, finalMessage, user)); return masterId; } @@ -580,6 +633,10 @@ public class DatabaseUtils { return views.stream().map(mapper).flatMap(s -> s.stream()).collect(Collectors.toCollection(LinkedHashSet::new)); } + private Set findAll(final List views, final Function> mapper, final Predicate filter) { + return views.stream().map(mapper).flatMap(s -> s.stream()).filter(filter).collect(Collectors.toCollection(LinkedHashSet::new)); + } + private List hideConflictOrgs(final String masterId, final String otherId) { organizationRepository.updateStatus(otherId, OrganizationStatus.hidden.toString()); openaireConflictRepository.findById(new OpenaireConflictPK(masterId, otherId)).ifPresent(openaireConflictRepository::delete); diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/JournalOperations.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/JournalOperations.java index d2db9715..7bcb1ec7 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/JournalOperations.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/JournalOperations.java @@ -9,5 +9,6 @@ public enum JournalOperations { DUPLICATES, FIX_CONFLICT, NO_CONFLICT, + BACKUP_ORG, UNKNOWN } From 7716f199ad4ed273ec71da15ed029feec11223b2 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Tue, 11 Oct 2022 11:33:45 +0200 Subject: [PATCH 09/20] fixed an if condition --- .../main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java index 27517518..8716cce0 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java @@ -485,7 +485,7 @@ public class DatabaseUtils { if (persistents.size() > 1) { throw new RuntimeException("Too many persintent organizations"); - } else if (persistents.size() > 1) { + } else if (persistents.size() == 1) { backupOrg(persistents.get(0), user); return fixConflicts(persistents.get(0), views, user); } else { From 96ec14722c04c1d37c61200892ccb49d5bf8c1c4 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Tue, 11 Oct 2022 12:36:02 +0200 Subject: [PATCH 10/20] some fixes --- .../repository/OrganizationRepository.java | 3 ++- .../organizations/utils/DatabaseUtils.java | 17 ++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/OrganizationRepository.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/OrganizationRepository.java index 6071b0d4..30301239 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/OrganizationRepository.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/OrganizationRepository.java @@ -22,9 +22,10 @@ public interface OrganizationRepository extends JpaRepository persistents = views.stream().filter(v -> v.getPersistent()).collect(Collectors.toList()); + final OrganizationView masterOrg = new OrganizationView(); + if (persistents.size() > 1) { throw new RuntimeException("Too many persintent organizations"); } else if (persistents.size() == 1) { backupOrg(persistents.get(0), user); - return fixConflicts(persistents.get(0), views, user); + masterOrg.setId(persistents.get(0).getId()); + masterOrg.setStatus(OrganizationStatus.approved.toString()); } else { - // I create a new org - final OrganizationView masterOrg = new OrganizationView(); masterOrg.setId(null); masterOrg.setStatus(null); - return fixConflicts(masterOrg, views, user); } + return fixConflicts(masterOrg, views, user); } private String backupOrg(final OrganizationView org, final String user) { final String origId = org.getId(); final String backupId = origId + "::" + OffsetDateTime.now().toEpochSecond(); + organizationRepository.prepareOrgWithId(backupId); try { final OrganizationView backupOrg = (OrganizationView) BeanUtils.cloneBean(org); backupOrg.setId(backupId); - backupOrg.setStatus(OrganizationStatus.hidden.toString()); insertOrUpdateOrganization(backupOrg, user, false); + organizationRepository.updateStatus(backupId, OrganizationStatus.hidden.toString()); + journalEntryRepository .save(new JournalEntry(origId, JournalOperations.BACKUP_ORG, "Saved a backup copy: " + backupId, user)); journalEntryRepository @@ -560,7 +563,7 @@ public class DatabaseUtils { .forEach(id -> { hideConflictOrgs(masterId, id); journalEntryRepository - .save(new JournalEntry(masterId, JournalOperations.FIX_CONFLICT, "The org has been hidded and merged in " + masterId, user)); + .save(new JournalEntry(id, JournalOperations.FIX_CONFLICT, "The org has been hidded and merged in " + masterId, user)); }); // I reassign the duplicates to the new org From 4573b41b302346976024002235ac6192beb8398f Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Tue, 11 Oct 2022 14:37:02 +0200 Subject: [PATCH 11/20] ui --- .../static/resources/js/organizations.js | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/js/organizations.js b/apps/dnet-orgs-database-application/src/main/resources/static/resources/js/organizations.js index ca08e30b..b7fb83a7 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/static/resources/js/organizations.js +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/js/organizations.js @@ -119,7 +119,12 @@ orgsModule.directive('resolveConflictsModal', function($http, $route, $window) { call_http_post($http, 'api/organizations/conflicts/fix/similar', ids, function(res) { $('#' + scope.modalId).modal('hide'); if (scope.openNewOrg == '1') { - $('#' + scope.modalId).on('hidden.bs.modal', function (e) { $window.location.assign('#!/edit/0/' + res.data[0]); }); + $('#' + scope.modalId).on('hidden.bs.modal', function (e) { + var oldLoc = $window.location; + $window.location.assign('#!/edit/0/' + res.data[0]); + var newLoc = $window.location; + if (oldLoc == newLoc) $route.reload(); + }); } else { $('#' + scope.modalId).on('hidden.bs.modal', function (e) { $route.reload(); }); } @@ -323,7 +328,13 @@ orgsModule.directive('orgConflicts', function($http, $window, $location, $route, angular.forEach(scope.conflicts, function(o, pos) { ids.push(o.id); }); if (merge) { - call_http_post($http, "api/organizations/conflicts/fix/similar", ids, function(res) { $window.location.assign('#!/edit/0/' + res.data[0]); }); + call_http_post($http, "api/organizations/conflicts/fix/similar", ids, function(res) { + if (res.data[0] == scope.org.id) { + $route.reload(); + } else { + $window.location.assign('#!/edit/0/' + res.data[0]); + } + }); } else { call_http_post($http, "api/organizations/conflicts/fix/different", ids, function(res) { $route.reload(); }); } From 5f4a8cd2912bf173961971417e7130ff3eb6948d Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Wed, 19 Oct 2022 10:17:58 +0200 Subject: [PATCH 12/20] label --- .../main/resources/static/resources/html/pages/admin/users.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/admin/users.html b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/admin/users.html index dfef8343..dffb9b41 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/admin/users.html +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/admin/users.html @@ -86,7 +86,7 @@ - + From 047ee7b89a4600a10754d62660d7dec717a6077d Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Wed, 19 Oct 2022 14:10:25 +0200 Subject: [PATCH 13/20] invalid symbol --- .../main/resources/static/resources/html/pages/admin/users.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/admin/users.html b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/admin/users.html index dffb9b41..e42d95f6 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/admin/users.html +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/admin/users.html @@ -82,7 +82,7 @@ - + From 7542dcc511630c2874fa2d94d78970911d614223 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Fri, 21 Oct 2022 09:40:37 +0200 Subject: [PATCH 14/20] [maven-release-plugin] prepare release dnet-applications-3.3.2 --- apps/bioschemas-api/pom.xml | 2 +- apps/dhp-broker-application/pom.xml | 2 +- apps/dhp-broker-public-application/pom.xml | 2 +- apps/dhp-mdstore-manager/pom.xml | 2 +- apps/dnet-exporter-api/pom.xml | 2 +- apps/dnet-orgs-database-application/pom.xml | 2 +- apps/pom.xml | 2 +- apps/scholexplorer-api/pom.xml | 2 +- cmd-line-apps/dhp-broker-client/pom.xml | 2 +- cmd-line-apps/pom.xml | 2 +- libs/dnet-apps-common/pom.xml | 2 +- libs/dnet-broker-apps-common/pom.xml | 2 +- libs/dnet-openaire-broker-common/pom.xml | 2 +- libs/pom.xml | 2 +- pom.xml | 4 ++-- 15 files changed, 16 insertions(+), 16 deletions(-) diff --git a/apps/bioschemas-api/pom.xml b/apps/bioschemas-api/pom.xml index 3b4086da..6307d5df 100644 --- a/apps/bioschemas-api/pom.xml +++ b/apps/bioschemas-api/pom.xml @@ -3,7 +3,7 @@ eu.dnetlib.dhp apps - 3.3.2-SNAPSHOT + 3.3.2 ../pom.xml diff --git a/apps/dhp-broker-application/pom.xml b/apps/dhp-broker-application/pom.xml index c0f7c74a..2551344d 100644 --- a/apps/dhp-broker-application/pom.xml +++ b/apps/dhp-broker-application/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp apps - 3.3.2-SNAPSHOT + 3.3.2 ../pom.xml diff --git a/apps/dhp-broker-public-application/pom.xml b/apps/dhp-broker-public-application/pom.xml index 652d150e..a03940dc 100644 --- a/apps/dhp-broker-public-application/pom.xml +++ b/apps/dhp-broker-public-application/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp apps - 3.3.2-SNAPSHOT + 3.3.2 ../pom.xml diff --git a/apps/dhp-mdstore-manager/pom.xml b/apps/dhp-mdstore-manager/pom.xml index 1c332429..a1819d81 100644 --- a/apps/dhp-mdstore-manager/pom.xml +++ b/apps/dhp-mdstore-manager/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp apps - 3.3.2-SNAPSHOT + 3.3.2 ../pom.xml diff --git a/apps/dnet-exporter-api/pom.xml b/apps/dnet-exporter-api/pom.xml index 4478e15c..032f88c5 100644 --- a/apps/dnet-exporter-api/pom.xml +++ b/apps/dnet-exporter-api/pom.xml @@ -3,7 +3,7 @@ eu.dnetlib.dhp apps - 3.3.2-SNAPSHOT + 3.3.2 ../ diff --git a/apps/dnet-orgs-database-application/pom.xml b/apps/dnet-orgs-database-application/pom.xml index 996c63a2..89c235e5 100644 --- a/apps/dnet-orgs-database-application/pom.xml +++ b/apps/dnet-orgs-database-application/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp apps - 3.3.2-SNAPSHOT + 3.3.2 ../ diff --git a/apps/pom.xml b/apps/pom.xml index 9aa3da84..0ad0f1b9 100644 --- a/apps/pom.xml +++ b/apps/pom.xml @@ -3,7 +3,7 @@ eu.dnetlib.dhp dnet-applications - 3.3.2-SNAPSHOT + 3.3.2 ../pom.xml diff --git a/apps/scholexplorer-api/pom.xml b/apps/scholexplorer-api/pom.xml index 56b8f577..5f9c3ddc 100644 --- a/apps/scholexplorer-api/pom.xml +++ b/apps/scholexplorer-api/pom.xml @@ -3,7 +3,7 @@ eu.dnetlib.dhp apps - 3.3.2-SNAPSHOT + 3.3.2 ../pom.xml diff --git a/cmd-line-apps/dhp-broker-client/pom.xml b/cmd-line-apps/dhp-broker-client/pom.xml index 91841aa6..666a7af7 100644 --- a/cmd-line-apps/dhp-broker-client/pom.xml +++ b/cmd-line-apps/dhp-broker-client/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp cmd-line-apps - 3.3.2-SNAPSHOT + 3.3.2 ../ diff --git a/cmd-line-apps/pom.xml b/cmd-line-apps/pom.xml index 0e5b954b..843ca22a 100644 --- a/cmd-line-apps/pom.xml +++ b/cmd-line-apps/pom.xml @@ -3,7 +3,7 @@ eu.dnetlib.dhp dnet-applications - 3.3.2-SNAPSHOT + 3.3.2 ../ diff --git a/libs/dnet-apps-common/pom.xml b/libs/dnet-apps-common/pom.xml index 16f99e18..df66798a 100644 --- a/libs/dnet-apps-common/pom.xml +++ b/libs/dnet-apps-common/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp libs - 3.3.2-SNAPSHOT + 3.3.2 ../ diff --git a/libs/dnet-broker-apps-common/pom.xml b/libs/dnet-broker-apps-common/pom.xml index 44d7ad8e..07f24041 100644 --- a/libs/dnet-broker-apps-common/pom.xml +++ b/libs/dnet-broker-apps-common/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp libs - 3.3.2-SNAPSHOT + 3.3.2 ../ diff --git a/libs/dnet-openaire-broker-common/pom.xml b/libs/dnet-openaire-broker-common/pom.xml index b67978b7..019492ae 100644 --- a/libs/dnet-openaire-broker-common/pom.xml +++ b/libs/dnet-openaire-broker-common/pom.xml @@ -3,7 +3,7 @@ eu.dnetlib.dhp libs - 3.3.2-SNAPSHOT + 3.3.2 ../ diff --git a/libs/pom.xml b/libs/pom.xml index fe9f80a5..eb94c342 100644 --- a/libs/pom.xml +++ b/libs/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dnet-applications - 3.3.2-SNAPSHOT + 3.3.2 ../ diff --git a/pom.xml b/pom.xml index bceeded4..558043cd 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ 4.0.0 eu.dnetlib.dhp dnet-applications - 3.3.2-SNAPSHOT + 3.3.2 pom @@ -44,7 +44,7 @@ scm:git:gitea@code-repo.d4science.org:D-Net/dnet-applications.git scm:git:gitea@code-repo.d4science.org:D-Net/dnet-applications.git https://code-repo.d4science.org/D-Net/dnet-applications/ - HEAD + dnet-applications-3.3.2 This module is the root descriptor for the dnet-applications project From 92bd92f1b973f3c884f2fe4753ef92c0c8745715 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Fri, 21 Oct 2022 09:40:40 +0200 Subject: [PATCH 15/20] [maven-release-plugin] prepare for next development iteration --- apps/bioschemas-api/pom.xml | 2 +- apps/dhp-broker-application/pom.xml | 2 +- apps/dhp-broker-public-application/pom.xml | 2 +- apps/dhp-mdstore-manager/pom.xml | 2 +- apps/dnet-exporter-api/pom.xml | 2 +- apps/dnet-orgs-database-application/pom.xml | 2 +- apps/pom.xml | 2 +- apps/scholexplorer-api/pom.xml | 2 +- cmd-line-apps/dhp-broker-client/pom.xml | 2 +- cmd-line-apps/pom.xml | 2 +- libs/dnet-apps-common/pom.xml | 2 +- libs/dnet-broker-apps-common/pom.xml | 2 +- libs/dnet-openaire-broker-common/pom.xml | 2 +- libs/pom.xml | 2 +- pom.xml | 4 ++-- 15 files changed, 16 insertions(+), 16 deletions(-) diff --git a/apps/bioschemas-api/pom.xml b/apps/bioschemas-api/pom.xml index 6307d5df..8310774b 100644 --- a/apps/bioschemas-api/pom.xml +++ b/apps/bioschemas-api/pom.xml @@ -3,7 +3,7 @@ eu.dnetlib.dhp apps - 3.3.2 + 3.3.3-SNAPSHOT ../pom.xml diff --git a/apps/dhp-broker-application/pom.xml b/apps/dhp-broker-application/pom.xml index 2551344d..931b0c70 100644 --- a/apps/dhp-broker-application/pom.xml +++ b/apps/dhp-broker-application/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp apps - 3.3.2 + 3.3.3-SNAPSHOT ../pom.xml diff --git a/apps/dhp-broker-public-application/pom.xml b/apps/dhp-broker-public-application/pom.xml index a03940dc..3420d26b 100644 --- a/apps/dhp-broker-public-application/pom.xml +++ b/apps/dhp-broker-public-application/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp apps - 3.3.2 + 3.3.3-SNAPSHOT ../pom.xml diff --git a/apps/dhp-mdstore-manager/pom.xml b/apps/dhp-mdstore-manager/pom.xml index a1819d81..cc28e3a5 100644 --- a/apps/dhp-mdstore-manager/pom.xml +++ b/apps/dhp-mdstore-manager/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp apps - 3.3.2 + 3.3.3-SNAPSHOT ../pom.xml diff --git a/apps/dnet-exporter-api/pom.xml b/apps/dnet-exporter-api/pom.xml index 032f88c5..f80aa74a 100644 --- a/apps/dnet-exporter-api/pom.xml +++ b/apps/dnet-exporter-api/pom.xml @@ -3,7 +3,7 @@ eu.dnetlib.dhp apps - 3.3.2 + 3.3.3-SNAPSHOT ../ diff --git a/apps/dnet-orgs-database-application/pom.xml b/apps/dnet-orgs-database-application/pom.xml index 89c235e5..026fa11d 100644 --- a/apps/dnet-orgs-database-application/pom.xml +++ b/apps/dnet-orgs-database-application/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp apps - 3.3.2 + 3.3.3-SNAPSHOT ../ diff --git a/apps/pom.xml b/apps/pom.xml index 0ad0f1b9..2a316999 100644 --- a/apps/pom.xml +++ b/apps/pom.xml @@ -3,7 +3,7 @@ eu.dnetlib.dhp dnet-applications - 3.3.2 + 3.3.3-SNAPSHOT ../pom.xml diff --git a/apps/scholexplorer-api/pom.xml b/apps/scholexplorer-api/pom.xml index 5f9c3ddc..682cfb91 100644 --- a/apps/scholexplorer-api/pom.xml +++ b/apps/scholexplorer-api/pom.xml @@ -3,7 +3,7 @@ eu.dnetlib.dhp apps - 3.3.2 + 3.3.3-SNAPSHOT ../pom.xml diff --git a/cmd-line-apps/dhp-broker-client/pom.xml b/cmd-line-apps/dhp-broker-client/pom.xml index 666a7af7..50f5df8c 100644 --- a/cmd-line-apps/dhp-broker-client/pom.xml +++ b/cmd-line-apps/dhp-broker-client/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp cmd-line-apps - 3.3.2 + 3.3.3-SNAPSHOT ../ diff --git a/cmd-line-apps/pom.xml b/cmd-line-apps/pom.xml index 843ca22a..c766a841 100644 --- a/cmd-line-apps/pom.xml +++ b/cmd-line-apps/pom.xml @@ -3,7 +3,7 @@ eu.dnetlib.dhp dnet-applications - 3.3.2 + 3.3.3-SNAPSHOT ../ diff --git a/libs/dnet-apps-common/pom.xml b/libs/dnet-apps-common/pom.xml index df66798a..6c80155e 100644 --- a/libs/dnet-apps-common/pom.xml +++ b/libs/dnet-apps-common/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp libs - 3.3.2 + 3.3.3-SNAPSHOT ../ diff --git a/libs/dnet-broker-apps-common/pom.xml b/libs/dnet-broker-apps-common/pom.xml index 07f24041..a1ef64b4 100644 --- a/libs/dnet-broker-apps-common/pom.xml +++ b/libs/dnet-broker-apps-common/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp libs - 3.3.2 + 3.3.3-SNAPSHOT ../ diff --git a/libs/dnet-openaire-broker-common/pom.xml b/libs/dnet-openaire-broker-common/pom.xml index 019492ae..56c420f3 100644 --- a/libs/dnet-openaire-broker-common/pom.xml +++ b/libs/dnet-openaire-broker-common/pom.xml @@ -3,7 +3,7 @@ eu.dnetlib.dhp libs - 3.3.2 + 3.3.3-SNAPSHOT ../ diff --git a/libs/pom.xml b/libs/pom.xml index eb94c342..067c9d35 100644 --- a/libs/pom.xml +++ b/libs/pom.xml @@ -4,7 +4,7 @@ eu.dnetlib.dhp dnet-applications - 3.3.2 + 3.3.3-SNAPSHOT ../ diff --git a/pom.xml b/pom.xml index 558043cd..b7839b62 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ 4.0.0 eu.dnetlib.dhp dnet-applications - 3.3.2 + 3.3.3-SNAPSHOT pom @@ -44,7 +44,7 @@ scm:git:gitea@code-repo.d4science.org:D-Net/dnet-applications.git scm:git:gitea@code-repo.d4science.org:D-Net/dnet-applications.git https://code-repo.d4science.org/D-Net/dnet-applications/ - dnet-applications-3.3.2 + HEAD This module is the root descriptor for the dnet-applications project From 13e9821940dc6c9ddc7bfd56efdc5df2d1792cd0 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Fri, 21 Oct 2022 15:30:59 +0200 Subject: [PATCH 16/20] fixed a bug counting datasources --- .../sql/recent_registered_datasources_fromDate.st.sql | 4 +--- .../recent_registered_datasources_fromDate_typology.st.sql | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate.st.sql b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate.st.sql index e014461c..6d8bbf02 100644 --- a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate.st.sql +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate.st.sql @@ -1,9 +1,7 @@ -select count(d.id) as count +select count(DISTINCT d.id) as count from dsm_services d left outer join dsm_api a on (d.id = a.service) - left outer join dsm_service_organization dso on (d.id = dso.service) - left outer join dsm_organizations o on (o.id = dso.organization) where d.registrationdate >= cast(? as date) and d.registrationdate < a.last_collection_date diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql index 7abca790..9e8a9f0b 100644 --- a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_fromDate_typology.st.sql @@ -1,9 +1,7 @@ -select count(d.id) as count +select count(DISTINCT d.id) as count from dsm_services d left outer join dsm_api a on (d.id = a.service) - left outer join dsm_service_organization dso on (d.id = dso.service) - left outer join dsm_organizations o on (o.id = dso.organization) where d.registrationdate >= cast(? as date) and d._typology_to_remove_ like ? From 362ab547cbcf08f8e5fd13a8c1549b47e9cbb1fc Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Thu, 27 Oct 2022 09:36:45 +0200 Subject: [PATCH 17/20] new dsm api for first collection date and registration date --- .../openaire/dsm/DsmApiController.java | 63 ++++++++- .../java/eu/dnetlib/openaire/dsm/DsmCore.java | 123 ++++++++++++++++- .../openaire/dsm/dao/ResponseUtils.java | 4 +- .../dsm/domain/SimpleDatasourceInfo.java | 129 ++++++++++++++++++ ...irst_collected_datasources_fromDate.st.sql | 7 + ...ected_datasources_fromDate_typology.st.sql | 8 ++ ...irst_collected_datasources_fromDate.st.sql | 29 ++++ ...ected_datasources_fromDate_typology.st.sql | 31 +++++ .../recent_registered_datasources_v2.sql.st | 38 ++++++ 9 files changed, 427 insertions(+), 5 deletions(-) create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/SimpleDatasourceInfo.java create mode 100644 apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate.st.sql create mode 100644 apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate_typology.st.sql create mode 100644 apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate.st.sql create mode 100644 apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate_typology.st.sql create mode 100644 apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_v2.sql.st diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java index 329923ef..99450dc5 100755 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java @@ -28,6 +28,7 @@ import eu.dnetlib.enabling.datasources.common.DsmForbiddenException; import eu.dnetlib.enabling.datasources.common.DsmNotFoundException; import eu.dnetlib.openaire.common.AbstractExporterController; import eu.dnetlib.openaire.common.OperationManager; +import eu.dnetlib.openaire.dsm.dao.ResponseUtils; import eu.dnetlib.openaire.dsm.domain.AggregationHistoryResponse; import eu.dnetlib.openaire.dsm.domain.ApiDetails; import eu.dnetlib.openaire.dsm.domain.ApiDetailsResponse; @@ -36,10 +37,12 @@ import eu.dnetlib.openaire.dsm.domain.DatasourceDetails; import eu.dnetlib.openaire.dsm.domain.DatasourceDetailsUpdate; import eu.dnetlib.openaire.dsm.domain.DatasourceDetailsWithApis; import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetResponse; +import eu.dnetlib.openaire.dsm.domain.RegisteredDatasourceInfo; import eu.dnetlib.openaire.dsm.domain.RequestFilter; import eu.dnetlib.openaire.dsm.domain.RequestSort; import eu.dnetlib.openaire.dsm.domain.RequestSortOrder; import eu.dnetlib.openaire.dsm.domain.Response; +import eu.dnetlib.openaire.dsm.domain.SimpleDatasourceInfo; import eu.dnetlib.openaire.dsm.domain.SimpleResponse; import eu.dnetlib.openaire.vocabularies.Country; import io.swagger.v3.oas.annotations.Operation; @@ -163,9 +166,9 @@ public class DsmApiController extends AbstractExporterController { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) - public SimpleResponse recentRegistered(@PathVariable final int size) throws Throwable { + public SimpleResponse recentRegistered(@PathVariable final int size) throws Throwable { final StopWatch stop = StopWatch.createStarted(); - final SimpleResponse rsp = dsmCore.searchRecentRegistered(size); + final SimpleResponse rsp = dsmCore.searchRecentRegistered(size); return prepareResponse(1, size, stop, rsp); } @@ -422,4 +425,60 @@ public class DsmApiController extends AbstractExporterController { .setSize(size); return rsp; } + + // ------------------------------ + + @RequestMapping(value = "/ds/recentregistered/v2/{size}", produces = { + "application/json" + }, method = RequestMethod.GET) + @Operation(summary = "return the latest datasources that were registered through Provide (v2)", description = "Returns list of Datasource basic info.", tags = { + DS, + R + }) + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "500", description = "unexpected error") + }) + public SimpleResponse recentRegisteredV2(@PathVariable final int size) throws Throwable { + final StopWatch stop = StopWatch.createStarted(); + final SimpleResponse rsp = dsmCore.searchRecentRegisteredV2(size); + return prepareResponse(1, size, stop, rsp); + } + + @RequestMapping(value = "/ds/countfirstcollect", produces = { + "application/json" + }, method = RequestMethod.GET) + @Operation(summary = "return the number of datasources registered after the given date", description = "Returns a number.", tags = { + DS, + R + }) + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "500", description = "unexpected error") + }) + public Long countFirstCollectAfter(@RequestParam final String fromDate, + @RequestParam(required = false) final String typologyFilter) throws Throwable { + return dsmCore.countFirstCollect(fromDate, typologyFilter); + } + + @RequestMapping(value = "/ds/firstCollected", produces = { + "application/json" + }, method = RequestMethod.GET) + @Operation(summary = "return the datasources that were collected for the first time after the specified date", description = "Returns list of Datasource basic info.", tags = { + DS, + R + }) + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "500", description = "unexpected error") + }) + public SimpleResponse firstCollectedAfter(@RequestParam final String fromDate, + @RequestParam(required = false) final String typologyFilter) throws Throwable { + final StopWatch stop = StopWatch.createStarted(); + final List list = dsmCore.getFirstCollectedAfter(fromDate, typologyFilter); + final SimpleResponse rsp = ResponseUtils.simpleResponse(list); + + return prepareResponse(1, list.size(), stop, rsp); + } + } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java index 13782b8c..42911d5b 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmCore.java @@ -26,6 +26,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.data.domain.Page; import org.springframework.jdbc.core.BeanPropertyRowMapper; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; import org.springframework.stereotype.Component; import eu.dnetlib.enabling.datasources.common.AggregationInfo; @@ -51,6 +52,7 @@ import eu.dnetlib.openaire.dsm.domain.RegisteredDatasourceInfo; import eu.dnetlib.openaire.dsm.domain.RequestFilter; import eu.dnetlib.openaire.dsm.domain.RequestSort; import eu.dnetlib.openaire.dsm.domain.RequestSortOrder; +import eu.dnetlib.openaire.dsm.domain.SimpleDatasourceInfo; import eu.dnetlib.openaire.dsm.domain.SimpleResponse; import eu.dnetlib.openaire.dsm.domain.db.ApiDbEntry; import eu.dnetlib.openaire.dsm.domain.db.DatasourceDbEntry; @@ -275,7 +277,7 @@ public class DsmCore { // HELPERS ////////////// - public SimpleResponse searchRecentRegistered(final int size) throws Throwable { + public SimpleResponse searchRecentRegistered(final int size) throws Throwable { try { final String sql = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources.sql.st"), Charset.defaultCharset()); @@ -318,4 +320,123 @@ public class DsmCore { return rsp; } + + public SimpleResponse searchRecentRegisteredV2(final int size) throws Throwable { + try { + final String sql = + IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/recent_registered_datasources_v2.sql.st"), Charset.defaultCharset()); + + final List list = + jdbcTemplate.query(sql, rowMapperForSimpleDatasourceInfo(), size); + + return ResponseUtils.simpleResponse(list); + } catch (final Throwable e) { + log.error("error searching recent datasources", e); + throw e; + } + } + + public Long countFirstCollect(final String fromDate, final String typeFilter) throws Throwable { + try { + if (StringUtils.isNotBlank(typeFilter)) { + final String sql = + IOUtils + .toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate_typology.st.sql"), Charset + .defaultCharset()); + + return jdbcTemplate.queryForObject(sql, Long.class, typeFilter + "%", fromDate); + } else { + final String sql = + IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate.st.sql"), Charset + .defaultCharset()); + + return jdbcTemplate.queryForObject(sql, Long.class, fromDate); + } + + } catch (final Throwable e) { + log.error("error searching datasources using the first collection date", e); + throw e; + } + } + + public List getFirstCollectedAfter(final String fromDate, final String typeFilter) throws Throwable { + try { + if (StringUtils.isNotBlank(typeFilter)) { + final String sql = + IOUtils + .toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate_typology.st.sql"), Charset + .defaultCharset()); + + return jdbcTemplate.query(sql, rowMapperForSimpleDatasourceInfo(), typeFilter + "%", fromDate); + + } else { + final String sql = + IOUtils + .toString(getClass().getResourceAsStream("/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate.st.sql"), Charset + .defaultCharset()); + + return jdbcTemplate.query(sql, rowMapperForSimpleDatasourceInfo(), fromDate); + + } + } catch (final Throwable e) { + log.error("error searching datasources using the first collection date", e); + throw e; + } + } + + private RowMapper rowMapperForSimpleDatasourceInfo() { + + return (rs, rowNum) -> { + final SimpleDatasourceInfo info = new SimpleDatasourceInfo(); + + info.setId(rs.getString("id")); + info.setOfficialName(rs.getString("officialName")); + info.setEnglishName(rs.getString("englishName")); + info.setTypology(rs.getString("typology")); + info.setEoscType(rs.getString("eoscType")); + info.setEoscDatasourceType(rs.getString("eoscDatasourceType")); + info.setRegisteredBy(rs.getString("registeredBy")); + info.setRegistrationDate(rs.getString("registrationDate")); + info.setFirstCollectionDate(rs.getString("firstCollectionDate")); + info.setLastCollectionDate(rs.getString("lastCollectionDate")); + info.setLastCollectionTotal(rs.getLong("lastCollectionTotal")); + + final Set compatibilities = new HashSet<>(); + for (final String s : (String[]) rs.getArray("compatibilities").getArray()) { + compatibilities.add(s); + } + + // The order of the condition is important + if (compatibilities.contains("openaire-cris_1.1")) { + info.setCompatibility("openaire-cris_1.1"); + } else if (compatibilities.contains("openaire4.0")) { + info.setCompatibility("openaire4.0"); + } else if (compatibilities.contains("driver") && compatibilities.contains("openaire2.0")) { + info.setCompatibility("driver-openaire2.0"); + } else if (compatibilities.contains("driver")) { + info.setCompatibility("driver"); + } else if (compatibilities.contains("openaire2.0")) { + info.setCompatibility("openaire2.0"); + } else if (compatibilities.contains("openaire3.0")) { + info.setCompatibility("openaire3.0"); + } else if (compatibilities.contains("openaire2.0_data")) { + info.setCompatibility("openaire2.0_data"); + } else if (compatibilities.contains("native")) { + info.setCompatibility("native"); + } else if (compatibilities.contains("hostedBy")) { + info.setCompatibility("hostedBy"); + } else if (compatibilities.contains("notCompatible")) { + info.setCompatibility("notCompatible"); + } else { + info.setCompatibility("UNKNOWN"); + } + + for (final String s : (String[]) rs.getArray("organizations").getArray()) { + info.getOrganizations().put(StringUtils.substringBefore(s, "@@@").trim(), StringUtils.substringAfter(s, "@@@").trim()); + } + + return info; + }; + } + } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ResponseUtils.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ResponseUtils.java index a2d422b8..cdc4c581 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ResponseUtils.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/dao/ResponseUtils.java @@ -52,8 +52,8 @@ public class ResponseUtils { return header(Lists.newLinkedList(), total); } - public static SimpleResponse simpleResponse(final List list) { - final SimpleResponse rsp = new SimpleResponse().setResponse(list);; + public static SimpleResponse simpleResponse(final List list) { + final SimpleResponse rsp = new SimpleResponse().setResponse(list);; rsp.setHeader(header(Lists.newLinkedList(), list.size())); return rsp; } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/SimpleDatasourceInfo.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/SimpleDatasourceInfo.java new file mode 100644 index 00000000..997f3894 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/domain/SimpleDatasourceInfo.java @@ -0,0 +1,129 @@ +package eu.dnetlib.openaire.dsm.domain; + +import java.util.LinkedHashMap; +import java.util.Map; + +public class SimpleDatasourceInfo { + + private String id; + private String officialName; + private String englishName; + private Map organizations = new LinkedHashMap<>(); + @Deprecated + private String typology; + private String eoscType; + private String eoscDatasourceType; + private String registeredBy; + private String registrationDate; + private String compatibility; + private String firstCollectionDate; + private String lastCollectionDate; + private long lastCollectionTotal; + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getOfficialName() { + return officialName; + } + + public void setOfficialName(final String officialName) { + this.officialName = officialName; + } + + public String getEnglishName() { + return englishName; + } + + public void setEnglishName(final String englishName) { + this.englishName = englishName; + } + + public Map getOrganizations() { + return organizations; + } + + public void setOrganizations(final Map organizations) { + this.organizations = organizations; + } + + @Deprecated + public String getTypology() { + return typology; + } + + @Deprecated + public void setTypology(final String typology) { + this.typology = typology; + } + + public String getEoscType() { + return eoscType; + } + + public void setEoscType(final String eoscType) { + this.eoscType = eoscType; + } + + public String getEoscDatasourceType() { + return eoscDatasourceType; + } + + public void setEoscDatasourceType(final String eoscDatasourceType) { + this.eoscDatasourceType = eoscDatasourceType; + } + + public String getRegisteredBy() { + return registeredBy; + } + + public void setRegisteredBy(final String registeredBy) { + this.registeredBy = registeredBy; + } + + public String getRegistrationDate() { + return registrationDate; + } + + public void setRegistrationDate(final String registrationDate) { + this.registrationDate = registrationDate; + } + + public String getCompatibility() { + return compatibility; + } + + public void setCompatibility(final String compatibility) { + this.compatibility = compatibility; + } + + public String getFirstCollectionDate() { + return firstCollectionDate; + } + + public void setFirstCollectionDate(final String firstCollectionDate) { + this.firstCollectionDate = firstCollectionDate; + } + + public String getLastCollectionDate() { + return lastCollectionDate; + } + + public void setLastCollectionDate(final String lastCollectionDate) { + this.lastCollectionDate = lastCollectionDate; + } + + public long getLastCollectionTotal() { + return lastCollectionTotal; + } + + public void setLastCollectionTotal(final long lastCollectionTotal) { + this.lastCollectionTotal = lastCollectionTotal; + } + +} diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate.st.sql b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate.st.sql new file mode 100644 index 00000000..1cf5f393 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate.st.sql @@ -0,0 +1,7 @@ +select count(*) as count +from ( + select d.id + from dsm_services d left outer join dsm_api a on (d.id = a.service) + group by d.id + having min(a.first_collection_date) >= cast(? as date) +) as t diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate_typology.st.sql b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate_typology.st.sql new file mode 100644 index 00000000..b0a79a29 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/count_first_collected_datasources_fromDate_typology.st.sql @@ -0,0 +1,8 @@ +select count(*) as count +from ( + select d.id + from dsm_services d left outer join dsm_api a on (d.id = a.service) + where d._typology_to_remove_ like ? + group by d.id + having min(a.first_collection_date) >= cast(? as date) +) as t diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate.st.sql b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate.st.sql new file mode 100644 index 00000000..f228e03b --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate.st.sql @@ -0,0 +1,29 @@ +SELECT + s.id AS "id", + s.officialname AS "officialName", + s.englishname AS "englishName", + s._typology_to_remove_ AS "typology", + s.eosc_type AS "eoscType", + s.eosc_datasource_type AS "eoscDatasourceType", + s.registeredby AS "registeredBy", + s.registrationdate::text AS "registrationDate", + MIN(a.first_collection_date) AS "firstCollectionDate", + MAX(a.last_collection_date) AS "lastCollectionDate", + (array_remove(array_agg(a.last_collection_total order by a.last_collection_date desc), NULL))[1] AS "lastCollectionTotal", + array_remove(array_agg(DISTINCT coalesce(a.compatibility_override, a.compatibility)), NULL) AS "compatibilities", + array_remove(array_agg(DISTINCT o.id||' @@@ '||o.legalname), NULL) AS "organizations" +FROM + dsm_services s + left outer join dsm_api a on (s.id = a.service) + left outer join dsm_service_organization dso on (s.id = dso.service) + left outer join dsm_organizations o on (o.id = dso.organization) +GROUP BY + s.id, + s.officialname, + s.englishname, + s._typology_to_remove_, + s.eosc_type, + s.eosc_datasource_type, + s.registeredby, + s.registrationdate +HAVING MIN(a.first_collection_date) >= cast(? as date) diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate_typology.st.sql b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate_typology.st.sql new file mode 100644 index 00000000..13d64cb4 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/first_collected_datasources_fromDate_typology.st.sql @@ -0,0 +1,31 @@ +SELECT + s.id AS "id", + s.officialname AS "officialName", + s.englishname AS "englishName", + s._typology_to_remove_ AS "typology", + s.eosc_type AS "eoscType", + s.eosc_datasource_type AS "eoscDatasourceType", + s.registeredby AS "registeredBy", + s.registrationdate::text AS "registrationDate", + MIN(a.first_collection_date) AS "firstCollectionDate", + MAX(a.last_collection_date) AS "lastCollectionDate", + (array_remove(array_agg(a.last_collection_total order by a.last_collection_date desc), NULL))[1] AS "lastCollectionTotal", + array_remove(array_agg(DISTINCT coalesce(a.compatibility_override, a.compatibility)), NULL) AS "compatibilities", + array_remove(array_agg(DISTINCT o.id||' @@@ '||o.legalname), NULL) AS "organizations" +FROM + dsm_services s + left outer join dsm_api a on (s.id = a.service) + left outer join dsm_service_organization dso on (s.id = dso.service) + left outer join dsm_organizations o on (o.id = dso.organization) +WHERE + s._typology_to_remove_ like ? +GROUP BY + s.id, + s.officialname, + s.englishname, + s._typology_to_remove_, + s.eosc_type, + s.eosc_datasource_type, + s.registeredby, + s.registrationdate +HAVING MIN(a.first_collection_date) >= cast(? as date) diff --git a/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_v2.sql.st b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_v2.sql.st new file mode 100644 index 00000000..b4002ce4 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/resources/eu/dnetlib/openaire/sql/recent_registered_datasources_v2.sql.st @@ -0,0 +1,38 @@ +SELECT + s.id AS "id", + s.officialname AS "officialName", + s.englishname AS "englishName", + s._typology_to_remove_ AS "typology", + s.eosc_type AS "eoscType", + s.eosc_datasource_type AS "eoscDatasourceType", + s.registeredby AS "registeredBy", + s.registrationdate::text AS "registrationDate", + MIN(a.first_collection_date) AS "firstCollectionDate", + MAX(a.last_collection_date) AS "lastCollectionDate", + (array_remove(array_agg(a.last_collection_total order by a.last_collection_date desc), NULL))[1] AS "lastCollectionTotal", + array_remove(array_agg(DISTINCT coalesce(a.compatibility_override, a.compatibility)), NULL) AS "compatibilities", + array_remove(array_agg(DISTINCT o.id||' @@@ '||o.legalname), NULL) AS "organizations" +FROM + dsm_services s + left outer join dsm_api a on (s.id = a.service) + left outer join dsm_service_organization dso on (s.id = dso.service) + left outer join dsm_organizations o on (o.id = dso.organization) +WHERE + s.registrationdate is not null + and s.registeredby is not null + and s.managed = true +GROUP BY + s.id, + s.officialname, + s.englishname, + s._typology_to_remove_, + s.eosc_type, + s.eosc_datasource_type, + s.registeredby, + s.registrationdate +HAVING + s.registrationdate < max(a.last_collection_date) + and sum(a.last_collection_total) > 0 +ORDER BY s.registrationdate desc +LIMIT ? + From f4a803eab18810bc134e8490aabd05649edc40cd Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Thu, 27 Oct 2022 11:19:57 +0200 Subject: [PATCH 18/20] dsm api 2.0 --- .../common/AbstractExporterController.java | 44 ++++-- .../openaire/dsm/DsmApiController.java | 134 +++++------------- .../openaire/dsm/DsmApiControllerV2.java | 86 +++++++++++ 3 files changed, 151 insertions(+), 113 deletions(-) create mode 100644 apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiControllerV2.java diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/AbstractExporterController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/AbstractExporterController.java index c11e15f0..711b7eb2 100644 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/AbstractExporterController.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/common/AbstractExporterController.java @@ -3,12 +3,9 @@ package eu.dnetlib.openaire.common; import java.util.List; import java.util.stream.Collectors; -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import eu.dnetlib.enabling.datasources.common.DsmException; -import eu.dnetlib.enabling.datasources.common.DsmForbiddenException; -import eu.dnetlib.enabling.datasources.common.DsmNotFoundException; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.commons.lang3.time.StopWatch; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.springframework.http.HttpStatus; @@ -17,6 +14,13 @@ import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; +import com.fasterxml.jackson.annotation.JsonAutoDetect; + +import eu.dnetlib.enabling.datasources.common.DsmException; +import eu.dnetlib.enabling.datasources.common.DsmForbiddenException; +import eu.dnetlib.enabling.datasources.common.DsmNotFoundException; +import eu.dnetlib.openaire.dsm.domain.Response; + /** * Created by claudio on 18/07/2017. */ @@ -25,7 +29,9 @@ public abstract class AbstractExporterController { private static final Log log = LogFactory.getLog(AbstractExporterController.class); // NOPMD by marko on 11/24/08 5:02 PM @ResponseBody - @ExceptionHandler({DsmException.class}) + @ExceptionHandler({ + DsmException.class + }) @ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR) public ErrorMessage handleDSMException(final Exception e) { return _handleError(e); @@ -39,7 +45,9 @@ public abstract class AbstractExporterController { } @ResponseBody - @ExceptionHandler({DsmNotFoundException.class}) + @ExceptionHandler({ + DsmNotFoundException.class + }) @ResponseStatus(value = HttpStatus.NOT_FOUND) public ErrorMessage handleNotFoundException(final Exception e) { return _handleError(e); @@ -50,23 +58,33 @@ public abstract class AbstractExporterController { @ResponseStatus(HttpStatus.BAD_REQUEST) public List processValidationError(final MethodArgumentNotValidException e) { return e.getBindingResult() - .getFieldErrors().stream() - .map(fe -> new ErrorMessage( - String.format("field '%s'", fe.getField()), - String.format("rejected value '%s'", fe.getRejectedValue()), - fe.getDefaultMessage())) - .collect(Collectors.toList()); + .getFieldErrors() + .stream() + .map(fe -> new ErrorMessage( + String.format("field '%s'", fe.getField()), + String.format("rejected value '%s'", fe.getRejectedValue()), + fe.getDefaultMessage())) + .collect(Collectors.toList()); } private ErrorMessage _handleError(final Exception e) { log.error(e); if (StringUtils.containsIgnoreCase(ExceptionUtils.getRootCauseMessage(e), "Broken pipe")) { - return null; //socket is closed, cannot return any response + return null; // socket is closed, cannot return any response } else { return new ErrorMessage(e); } } + // HELPERS + protected T prepareResponse(final int page, final int size, final StopWatch stopWatch, final T rsp) { + rsp.getHeader() + .setTime(stopWatch.getTime()) + .setPage(page) + .setSize(size); + return rsp; + } + @JsonAutoDetect public class ErrorMessage { diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java index 99450dc5..180cf716 100755 --- a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiController.java @@ -16,10 +16,11 @@ import org.apache.http.HttpStatus; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.web.bind.annotation.CrossOrigin; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; @@ -28,7 +29,6 @@ import eu.dnetlib.enabling.datasources.common.DsmForbiddenException; import eu.dnetlib.enabling.datasources.common.DsmNotFoundException; import eu.dnetlib.openaire.common.AbstractExporterController; import eu.dnetlib.openaire.common.OperationManager; -import eu.dnetlib.openaire.dsm.dao.ResponseUtils; import eu.dnetlib.openaire.dsm.domain.AggregationHistoryResponse; import eu.dnetlib.openaire.dsm.domain.ApiDetails; import eu.dnetlib.openaire.dsm.domain.ApiDetailsResponse; @@ -41,8 +41,6 @@ import eu.dnetlib.openaire.dsm.domain.RegisteredDatasourceInfo; import eu.dnetlib.openaire.dsm.domain.RequestFilter; import eu.dnetlib.openaire.dsm.domain.RequestSort; import eu.dnetlib.openaire.dsm.domain.RequestSortOrder; -import eu.dnetlib.openaire.dsm.domain.Response; -import eu.dnetlib.openaire.dsm.domain.SimpleDatasourceInfo; import eu.dnetlib.openaire.dsm.domain.SimpleResponse; import eu.dnetlib.openaire.vocabularies.Country; import io.swagger.v3.oas.annotations.Operation; @@ -61,9 +59,9 @@ public class DsmApiController extends AbstractExporterController { @Autowired private DsmCore dsmCore; - @RequestMapping(value = "/ds/countries", produces = { + @GetMapping(value = "/ds/countries", produces = { "application/json" - }, method = RequestMethod.GET) + }) @Operation(summary = "list the datasource countries", description = "list the datasource countries", tags = { DS, R }) @@ -75,9 +73,9 @@ public class DsmApiController extends AbstractExporterController { return dsmCore.listCountries(); } - @RequestMapping(value = "/ds/searchdetails/{page}/{size}", produces = { + @PostMapping(value = "/ds/searchdetails/{page}/{size}", produces = { "application/json" - }, method = RequestMethod.POST) + }) @Operation(summary = "search datasources", description = "Returns list of Datasource details.", tags = { DS, R }) @@ -96,9 +94,9 @@ public class DsmApiController extends AbstractExporterController { return prepareResponse(page, size, stop, rsp); } - @RequestMapping(value = "/ds/aggregationhistory/{dsId}", produces = { + @GetMapping(value = "/ds/aggregationhistory/{dsId}", produces = { "application/json" - }, method = RequestMethod.GET) + }) @Operation(summary = "search datasources", description = "Returns list of Datasource details.", tags = { DS, R }) @@ -112,9 +110,9 @@ public class DsmApiController extends AbstractExporterController { return prepareResponse(0, rsp.getAggregationInfo().size(), stop, rsp); } - @RequestMapping(value = "/ds/searchsnippet/{page}/{size}", produces = { + @PostMapping(value = "/ds/searchsnippet/{page}/{size}", produces = { "application/json" - }, method = RequestMethod.POST) + }) @Operation(summary = "search datasources", description = "Returns list of Datasource basic info.", tags = { DS, R }) @@ -133,9 +131,9 @@ public class DsmApiController extends AbstractExporterController { return prepareResponse(page, size, stop, rsp); } - @RequestMapping(value = "/ds/searchregistered/{page}/{size}", produces = { + @PostMapping(value = "/ds/searchregistered/{page}/{size}", produces = { "application/json" - }, method = RequestMethod.POST) + }) @Operation(summary = "search among registered datasources", description = "Returns list of Datasource basic info.", tags = { DS, R @@ -155,9 +153,9 @@ public class DsmApiController extends AbstractExporterController { return prepareResponse(page, size, stop, rsp); } - @RequestMapping(value = "/ds/recentregistered/{size}", produces = { + @GetMapping(value = "/ds/recentregistered/{size}", produces = { "application/json" - }, method = RequestMethod.GET) + }) @Operation(summary = "return the latest datasources that were registered through Provide", description = "Returns list of Datasource basic info.", tags = { DS, R @@ -172,9 +170,9 @@ public class DsmApiController extends AbstractExporterController { return prepareResponse(1, size, stop, rsp); } - @RequestMapping(value = "/ds/countregistered", produces = { + @GetMapping(value = "/ds/countregistered", produces = { "application/json" - }, method = RequestMethod.GET) + }) @Operation(summary = "return the number of datasources registered after the given date", description = "Returns a number.", tags = { DS, R @@ -188,9 +186,9 @@ public class DsmApiController extends AbstractExporterController { return dsmCore.countRegisteredAfter(fromDate, typologyFilter); } - @RequestMapping(value = "/ds/api/{dsId}", produces = { + @GetMapping(value = "/ds/api/{dsId}", produces = { "application/json" - }, method = RequestMethod.GET) + }) @Operation(summary = "get the list of API for a given datasource", description = "Returns the list of API for a given datasource.", tags = { API, R @@ -207,9 +205,9 @@ public class DsmApiController extends AbstractExporterController { return prepareResponse(0, rsp.getApi().size(), stop, rsp); } - @RequestMapping(value = "/api/baseurl/{page}/{size}", produces = { + @PostMapping(value = "/api/baseurl/{page}/{size}", produces = { "application/json" - }, method = RequestMethod.POST) + }) @Operation(summary = "search for the list of base URLs of Datasource APIs managed by a user", description = "Returns the list of base URLs of Datasource APIs managed by a user", tags = { DS, API, R }) @@ -225,7 +223,7 @@ public class DsmApiController extends AbstractExporterController { return dsmCore.findBaseURLs(requestFilter, page, size); } - @RequestMapping(value = "/ds/api/{apiId}", method = RequestMethod.DELETE) + @DeleteMapping(value = "/ds/api/{apiId}") @Operation(summary = "delete an API", description = "delete an API, if removable", tags = { API, W }) @@ -239,7 +237,7 @@ public class DsmApiController extends AbstractExporterController { dsmCore.deleteApi(apiId); } - @RequestMapping(value = "/ds/manage", method = RequestMethod.POST) + @PostMapping(value = "/ds/manage") @Operation(summary = "set the managed status for a given datasource", description = "set the managed status for a given datasource", tags = { DS, W }) @@ -254,7 +252,7 @@ public class DsmApiController extends AbstractExporterController { dsmCore.setManaged(id, managed); } - @RequestMapping(value = "/ds/managed/{id}", method = RequestMethod.GET) + @GetMapping(value = "/ds/managed/{id}") @Operation(summary = "get the datasource managed status", description = "get the datasource managed status", tags = { DS, R }) @@ -266,7 +264,7 @@ public class DsmApiController extends AbstractExporterController { return dsmCore.isManaged(id); } - @RequestMapping(value = "/ds/add", method = RequestMethod.POST) + @PostMapping(value = "/ds/add") @Operation(summary = "add a new Datasource", description = "add a new Datasource", tags = { DS, W }) @@ -283,7 +281,7 @@ public class DsmApiController extends AbstractExporterController { dsmCore.save(datasource); } - @RequestMapping(value = "/ds/addWithApis", method = RequestMethod.POST) + @PostMapping(value = "/ds/addWithApis") @Operation(summary = "add a new Datasource and its apis", description = "add a new Datasource and its apis", tags = { DS, W }) @@ -300,7 +298,7 @@ public class DsmApiController extends AbstractExporterController { dsmCore.save(d); } - @RequestMapping(value = "/ds/update", method = RequestMethod.POST) + @PostMapping(value = "/ds/update") @Operation(summary = "update Datasource details", description = "update Datasource details", tags = { DS, W }) @@ -314,7 +312,7 @@ public class DsmApiController extends AbstractExporterController { dsmCore.updateDatasource(ds); } - @RequestMapping(value = "/ds/api/baseurl", method = RequestMethod.POST) + @PostMapping(value = "/ds/api/baseurl") @Operation(summary = "update the base URL of a datasource interface", description = "update the base URL of a datasource interface", tags = { API, W }) @@ -330,7 +328,7 @@ public class DsmApiController extends AbstractExporterController { dsmCore.updateApiBaseurl(dsId, apiId, baseUrl); } - @RequestMapping(value = "/ds/api/compliance", method = RequestMethod.POST) + @PostMapping(value = "/ds/api/compliance") @Operation(summary = "update the compatibility of a datasource interface", description = "update the compatibility of a datasource interface", tags = { API, W }) @@ -347,7 +345,7 @@ public class DsmApiController extends AbstractExporterController { dsmCore.updateApiCompatibility(dsId, apiId, compliance, override); } - @RequestMapping(value = "/ds/api/oaiset", method = RequestMethod.POST) + @PostMapping(value = "/ds/api/oaiset") @Operation(summary = "update the OAI set of a datasource interface", description = "update the OAI set of a datasource interface", tags = { API, W }) @@ -363,7 +361,7 @@ public class DsmApiController extends AbstractExporterController { dsmCore.updateApiOaiSet(dsId, apiId, oaiSet); } - @RequestMapping(value = "/ds/api/add", method = RequestMethod.POST) + @PostMapping(value = "/ds/api/add") @Operation(summary = "adds a new Interface to one Datasource", description = "adds an Interface to one Datasource", tags = { API, W }) @@ -381,7 +379,7 @@ public class DsmApiController extends AbstractExporterController { @Autowired private OperationManager operationManager; - @RequestMapping(value = "/dsm/ops", method = RequestMethod.GET) + @GetMapping(value = "/dsm/ops") @Operation(summary = "get the number of pending operations", description = "get the number of pending operations", tags = { R, M }) @@ -393,7 +391,7 @@ public class DsmApiController extends AbstractExporterController { return operationManager.getOpSize(); } - @RequestMapping(value = "/dsm/killops", method = RequestMethod.POST) + @PostMapping(value = "/dsm/killops") @Operation(summary = "interrupts the pending operations", description = "return the number of interrupted operations", tags = { W, M }) @@ -405,7 +403,7 @@ public class DsmApiController extends AbstractExporterController { return operationManager.dropAll(); } - @RequestMapping(value = "/dsm/dropcache", method = RequestMethod.POST) + @PostMapping(value = "/dsm/dropcache") @Operation(summary = "drop the caches", description = "drop the internal caches", tags = { W, M }) @@ -417,68 +415,4 @@ public class DsmApiController extends AbstractExporterController { dsmCore.dropCaches(); } - // HELPERS - private T prepareResponse(final int page, final int size, final StopWatch stopWatch, final T rsp) { - rsp.getHeader() - .setTime(stopWatch.getTime()) - .setPage(page) - .setSize(size); - return rsp; - } - - // ------------------------------ - - @RequestMapping(value = "/ds/recentregistered/v2/{size}", produces = { - "application/json" - }, method = RequestMethod.GET) - @Operation(summary = "return the latest datasources that were registered through Provide (v2)", description = "Returns list of Datasource basic info.", tags = { - DS, - R - }) - @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "OK"), - @ApiResponse(responseCode = "500", description = "unexpected error") - }) - public SimpleResponse recentRegisteredV2(@PathVariable final int size) throws Throwable { - final StopWatch stop = StopWatch.createStarted(); - final SimpleResponse rsp = dsmCore.searchRecentRegisteredV2(size); - return prepareResponse(1, size, stop, rsp); - } - - @RequestMapping(value = "/ds/countfirstcollect", produces = { - "application/json" - }, method = RequestMethod.GET) - @Operation(summary = "return the number of datasources registered after the given date", description = "Returns a number.", tags = { - DS, - R - }) - @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "OK"), - @ApiResponse(responseCode = "500", description = "unexpected error") - }) - public Long countFirstCollectAfter(@RequestParam final String fromDate, - @RequestParam(required = false) final String typologyFilter) throws Throwable { - return dsmCore.countFirstCollect(fromDate, typologyFilter); - } - - @RequestMapping(value = "/ds/firstCollected", produces = { - "application/json" - }, method = RequestMethod.GET) - @Operation(summary = "return the datasources that were collected for the first time after the specified date", description = "Returns list of Datasource basic info.", tags = { - DS, - R - }) - @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "OK"), - @ApiResponse(responseCode = "500", description = "unexpected error") - }) - public SimpleResponse firstCollectedAfter(@RequestParam final String fromDate, - @RequestParam(required = false) final String typologyFilter) throws Throwable { - final StopWatch stop = StopWatch.createStarted(); - final List list = dsmCore.getFirstCollectedAfter(fromDate, typologyFilter); - final SimpleResponse rsp = ResponseUtils.simpleResponse(list); - - return prepareResponse(1, list.size(), stop, rsp); - } - } diff --git a/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiControllerV2.java b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiControllerV2.java new file mode 100644 index 00000000..356494e5 --- /dev/null +++ b/apps/dnet-exporter-api/src/main/java/eu/dnetlib/openaire/dsm/DsmApiControllerV2.java @@ -0,0 +1,86 @@ +package eu.dnetlib.openaire.dsm; + +import static eu.dnetlib.openaire.common.ExporterConstants.DS; +import static eu.dnetlib.openaire.common.ExporterConstants.R; + +import java.util.List; + +import org.apache.commons.lang3.time.StopWatch; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.web.bind.annotation.CrossOrigin; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import eu.dnetlib.openaire.common.AbstractExporterController; +import eu.dnetlib.openaire.dsm.dao.ResponseUtils; +import eu.dnetlib.openaire.dsm.domain.SimpleDatasourceInfo; +import eu.dnetlib.openaire.dsm.domain.SimpleResponse; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import io.swagger.v3.oas.annotations.tags.Tag; + +@RestController +@CrossOrigin(origins = { + "*" +}) +@ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true") +@Tag(name = "OpenAIRE DSM API (version 2.0)", description = "the OpenAIRE Datasource Manager API 2.0") +@RequestMapping("/dsm/2.0") +public class DsmApiControllerV2 extends AbstractExporterController { + + @Autowired + private DsmCore dsmCore; + + @GetMapping("/recentregistered/{size}") + @Operation(summary = "return the latest datasources that were registered through Provide (v2)", description = "Returns list of Datasource basic info.", tags = { + DS, + R + }) + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "500", description = "unexpected error") + }) + public SimpleResponse recentRegisteredV2(@PathVariable final int size) throws Throwable { + final StopWatch stop = StopWatch.createStarted(); + final SimpleResponse rsp = dsmCore.searchRecentRegisteredV2(size); + return prepareResponse(1, size, stop, rsp); + } + + @GetMapping("/countfirstcollect") + @Operation(summary = "return the number of datasources registered after the given date", description = "Returns a number.", tags = { + DS, + R + }) + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "500", description = "unexpected error") + }) + public Long countFirstCollectAfter(@RequestParam final String fromDate, + @RequestParam(required = false) final String typologyFilter) throws Throwable { + return dsmCore.countFirstCollect(fromDate, typologyFilter); + } + + @GetMapping("/firstCollected") + @Operation(summary = "return the datasources that were collected for the first time after the specified date", description = "Returns list of Datasource basic info.", tags = { + DS, + R + }) + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse(responseCode = "500", description = "unexpected error") + }) + public SimpleResponse firstCollectedAfter(@RequestParam final String fromDate, + @RequestParam(required = false) final String typologyFilter) throws Throwable { + final StopWatch stop = StopWatch.createStarted(); + final List list = dsmCore.getFirstCollectedAfter(fromDate, typologyFilter); + final SimpleResponse rsp = ResponseUtils.simpleResponse(list); + + return prepareResponse(1, list.size(), stop, rsp); + } + +} From 91d341c7be1e7bd5cadd9da24de7790d5c9a8c11 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Wed, 9 Nov 2022 14:29:42 +0100 Subject: [PATCH 19/20] search using OpenAIRE ID --- .../src/main/resources/sql/schema.sql | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql b/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql index 50c9e0f9..7ff4a386 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql +++ b/apps/dnet-orgs-database-application/src/main/resources/sql/schema.sql @@ -709,7 +709,7 @@ CREATE OR REPLACE FUNCTION refresh_index_search() RETURNS bigint AS $$ WITH d as ( INSERT INTO org_index_search(id, txt) SELECT o.id, - to_tsvector(o.id||' '||o.name||' '||array_to_string(array_agg(DISTINCT n.name), ' ','')||' '||array_to_string(array_agg(DISTINCT i.otherid), ' ','')||' '||array_to_string(array_agg(DISTINCT a.acronym), ' ','')||' '||array_to_string(array_agg(DISTINCT u.url), ' ','')) + to_tsvector(o.id||' '||substr(o.id, 1, 14)||md5(substr(o.id, 15))||' '||o.name||' '||array_to_string(array_agg(DISTINCT n.name), ' ','')||' '||array_to_string(array_agg(DISTINCT i.otherid), ' ','')||' '||array_to_string(array_agg(DISTINCT a.acronym), ' ','')||' '||array_to_string(array_agg(DISTINCT u.url), ' ','')) FROM organizations o LEFT OUTER JOIN other_names n on (o.id = n.id) LEFT OUTER JOIN other_ids i on (o.id = i.id) @@ -721,8 +721,6 @@ $$ LANGUAGE SQL; SELECT refresh_index_search(); - - CREATE OR REPLACE FUNCTION delete_index_search() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN DELETE FROM org_index_search WHERE id = old.id; @@ -734,7 +732,7 @@ CREATE OR REPLACE FUNCTION insert_or_update_index_search_trigger() RETURNS trigg BEGIN INSERT INTO org_index_search(id, txt) (SELECT o.id, - to_tsvector(o.id||' '||o.name||' '||array_to_string(array_agg(DISTINCT n.name), ' ','')||' '||array_to_string(array_agg(DISTINCT i.otherid), ' ','')||' '||array_to_string(array_agg(DISTINCT a.acronym), ' ','')||' '||array_to_string(array_agg(DISTINCT u.url), ' ','')) + to_tsvector(o.id||' '||substr(o.id, 1, 14)||md5(substr(o.id, 15))||' '||o.name||' '||array_to_string(array_agg(DISTINCT n.name), ' ','')||' '||array_to_string(array_agg(DISTINCT i.otherid), ' ','')||' '||array_to_string(array_agg(DISTINCT a.acronym), ' ','')||' '||array_to_string(array_agg(DISTINCT u.url), ' ','')) FROM organizations o LEFT OUTER JOIN other_names n on (o.id = n.id) LEFT OUTER JOIN other_ids i on (o.id = i.id) From 39d335439fa7ef7f30bc50c615341be4722a2d88 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Mon, 14 Nov 2022 11:59:31 +0100 Subject: [PATCH 20/20] persistent orgs filter and creation using OpenaireIDs --- .../readonly/OrganizationViewRepository.java | 3 + .../organizations/utils/DatabaseUtils.java | 27 ++++-- .../html/pages/admin/persistentOrgs.html | 91 ++++++++++--------- 3 files changed, 74 insertions(+), 47 deletions(-) diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/readonly/OrganizationViewRepository.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/readonly/OrganizationViewRepository.java index 2ea10b64..5af6cdb9 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/readonly/OrganizationViewRepository.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/readonly/OrganizationViewRepository.java @@ -1,5 +1,7 @@ package eu.dnetlib.organizations.repository.readonly; +import java.util.Optional; + import org.springframework.stereotype.Repository; import eu.dnetlib.organizations.model.view.OrganizationView; @@ -7,4 +9,5 @@ import eu.dnetlib.organizations.model.view.OrganizationView; @Repository public interface OrganizationViewRepository extends ReadOnlyRepository { + Optional findByOpenaireId(String openaireId); } diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java index 6b62494c..e58add34 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/utils/DatabaseUtils.java @@ -657,13 +657,28 @@ public class DatabaseUtils { return persistentOrganizationViewRepository.findAll(); } - public void addPersistentOrgs(final String id) { - final boolean valid = organizationRepository.findById(id) - .map(Organization::getStatus) - .filter(s -> s.equals(OrganizationStatus.approved.toString())) - .isPresent(); + public String addPersistentOrgs(final String id) { + + final boolean valid; + final String ooid; + + if (id.length() == 46) { + final Optional orgView = organizationViewRepository.findByOpenaireId(id); + valid = orgView.map(OrganizationView::getStatus) + .filter(s -> s.equals(OrganizationStatus.approved.toString())) + .isPresent(); + ooid = orgView.get().getId(); + } else { + valid = organizationRepository.findById(id) + .map(Organization::getStatus) + .filter(s -> s.equals(OrganizationStatus.approved.toString())) + .isPresent(); + ooid = id; + } + if (valid) { - persistentOrganizationRepository.save(new PersistentOrganization(id)); + persistentOrganizationRepository.save(new PersistentOrganization(ooid)); + return ooid; } else { throw new RuntimeException("The ID does not refer to an approved Organization"); } diff --git a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/admin/persistentOrgs.html b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/admin/persistentOrgs.html index 406d155a..28ef2fa4 100644 --- a/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/admin/persistentOrgs.html +++ b/apps/dnet-orgs-database-application/src/main/resources/static/resources/html/pages/admin/persistentOrgs.html @@ -4,44 +4,53 @@ It is necessary to persist the identifiers of the organizations associated to an Institutional Dashboard

-
Namenot specified
Registration RequestRegistration request 
{{currentUser.requestMessage}}
 not specified
Reference person>{{currentUser.referencePerson}}{{currentUser.referencePerson}} not specified
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
IDOA Graph Node IDNamePlace
{{o.id}}{{o.openaireId}}{{o.name}} {{o.city || '-'}}, {{o.country}} - -
No persistent organizazions
- - - -
\ No newline at end of file + +
+
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IDOA Graph Node IDNamePlace
{{o.id}}{{o.openaireId}}{{o.name}} {{o.city || '-'}}, {{o.country}} + +
No persistent organizazions
+ + + +
+
+
+