D-Net
/
dnet-applications
23
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

national_admin_functions #1

Manually merged
michele.artini merged 75 commits from national_admin_functions into master 2020-10-26 08:32:19 +01:00
Conversation 0 Commits 75 Files Changed 566 +25473 -1276
566 changed files with 25473 additions and 1276 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
.gitignore vendored
View File

@ -8,22 +8,24 @@
*~
.vscode
.classpath
/*/.classpath
/*/*/.classpath
/**/.classpath
.metadata
/*/.metadata
/*/*/.metadata
/**/.metadata
.project
/**/.project
.settings
/*/*/target
/*/target
/target
/*/*/build
/*/build
/build
/**/.settings
.factorypath
/**/.factorypath
target
/**/target
build
/**/build
spark-warehouse
/**/job-override.properties
/**/*.log
.svn
/**/.svn
apps/dnet-orgs-database-application/data/
apps/dnet-orgs-database-application/src/main/resources/tmp_data

120
dhp-broker-application/.factorypath → apps/dhp-broker-application/.factorypath
View File

@ -1,56 +1,35 @@
<factorypath>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/boot/spring-boot-starter-web/2.3.0.M4/spring-boot-starter-web-2.3.0.M4.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/boot/spring-boot-starter/2.3.0.M4/spring-boot-starter-2.3.0.M4.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/boot/spring-boot/2.3.0.M4/spring-boot-2.3.0.M4.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/boot/spring-boot-autoconfigure/2.3.0.M4/spring-boot-autoconfigure-2.3.0.M4.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/boot/spring-boot-starter-logging/2.3.0.M4/spring-boot-starter-logging-2.3.0.M4.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/boot/spring-boot-starter-data-elasticsearch/2.3.1.RELEASE/spring-boot-starter-data-elasticsearch-2.3.1.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/boot/spring-boot-starter/2.3.1.RELEASE/spring-boot-starter-2.3.1.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/boot/spring-boot/2.3.1.RELEASE/spring-boot-2.3.1.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/boot/spring-boot-autoconfigure/2.3.1.RELEASE/spring-boot-autoconfigure-2.3.1.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/boot/spring-boot-starter-logging/2.3.1.RELEASE/spring-boot-starter-logging-2.3.1.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/ch/qos/logback/logback-classic/1.2.3/logback-classic-1.2.3.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/ch/qos/logback/logback-core/1.2.3/logback-core-1.2.3.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/apache/logging/log4j/log4j-to-slf4j/2.13.1/log4j-to-slf4j-2.13.1.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/apache/logging/log4j/log4j-api/2.13.1/log4j-api-2.13.1.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/apache/logging/log4j/log4j-to-slf4j/2.13.3/log4j-to-slf4j-2.13.3.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/apache/logging/log4j/log4j-api/2.13.3/log4j-api-2.13.3.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/slf4j/jul-to-slf4j/1.7.30/jul-to-slf4j-1.7.30.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/jakarta/annotation/jakarta.annotation-api/1.3.5/jakarta.annotation-api-1.3.5.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/yaml/snakeyaml/1.26/snakeyaml-1.26.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/boot/spring-boot-starter-json/2.3.0.M4/spring-boot-starter-json-2.3.0.M4.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/fasterxml/jackson/core/jackson-databind/2.11.0.rc1/jackson-databind-2.11.0.rc1.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/fasterxml/jackson/datatype/jackson-datatype-jdk8/2.11.0.rc1/jackson-datatype-jdk8-2.11.0.rc1.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/fasterxml/jackson/datatype/jackson-datatype-jsr310/2.11.0.rc1/jackson-datatype-jsr310-2.11.0.rc1.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/fasterxml/jackson/module/jackson-module-parameter-names/2.11.0.rc1/jackson-module-parameter-names-2.11.0.rc1.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/boot/spring-boot-starter-tomcat/2.3.0.M4/spring-boot-starter-tomcat-2.3.0.M4.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/apache/tomcat/embed/tomcat-embed-core/9.0.33/tomcat-embed-core-9.0.33.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/glassfish/jakarta.el/3.0.3/jakarta.el-3.0.3.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/apache/tomcat/embed/tomcat-embed-websocket/9.0.33/tomcat-embed-websocket-9.0.33.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/spring-web/5.2.5.RELEASE/spring-web-5.2.5.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/spring-beans/5.2.5.RELEASE/spring-beans-5.2.5.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/spring-webmvc/5.2.5.RELEASE/spring-webmvc-5.2.5.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/spring-expression/5.2.5.RELEASE/spring-expression-5.2.5.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/boot/spring-boot-configuration-processor/2.3.0.M4/spring-boot-configuration-processor-2.3.0.M4.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/boot/spring-boot-starter-amqp/2.3.0.M4/spring-boot-starter-amqp-2.3.0.M4.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/spring-messaging/5.2.5.RELEASE/spring-messaging-5.2.5.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/amqp/spring-rabbit/2.2.5.RELEASE/spring-rabbit-2.2.5.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/rabbitmq/amqp-client/5.8.0/amqp-client-5.8.0.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/amqp/spring-amqp/2.2.5.RELEASE/spring-amqp-2.2.5.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/retry/spring-retry/1.2.5.RELEASE/spring-retry-1.2.5.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/spring-tx/5.2.5.RELEASE/spring-tx-5.2.5.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/boot/spring-boot-starter-data-elasticsearch/2.3.0.M4/spring-boot-starter-data-elasticsearch-2.3.0.M4.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/data/spring-data-elasticsearch/4.0.0.RC1/spring-data-elasticsearch-4.0.0.RC1.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/data/spring-data-commons/2.3.0.RC1/spring-data-commons-2.3.0.RC1.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/data/spring-data-elasticsearch/4.0.1.RELEASE/spring-data-elasticsearch-4.0.1.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/spring-tx/5.2.7.RELEASE/spring-tx-5.2.7.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/data/spring-data-commons/2.3.1.RELEASE/spring-data-commons-2.3.1.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/elasticsearch/plugin/transport-netty4-client/7.6.2/transport-netty4-client-7.6.2.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/netty/netty-buffer/4.1.48.Final/netty-buffer-4.1.48.Final.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/netty/netty-codec/4.1.48.Final/netty-codec-4.1.48.Final.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/netty/netty-codec-http/4.1.48.Final/netty-codec-http-4.1.48.Final.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/netty/netty-common/4.1.48.Final/netty-common-4.1.48.Final.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/netty/netty-handler/4.1.48.Final/netty-handler-4.1.48.Final.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/netty/netty-resolver/4.1.48.Final/netty-resolver-4.1.48.Final.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/netty/netty-transport/4.1.48.Final/netty-transport-4.1.48.Final.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/netty/netty-buffer/4.1.50.Final/netty-buffer-4.1.50.Final.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/netty/netty-codec/4.1.50.Final/netty-codec-4.1.50.Final.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/netty/netty-codec-http/4.1.50.Final/netty-codec-http-4.1.50.Final.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/netty/netty-common/4.1.50.Final/netty-common-4.1.50.Final.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/netty/netty-handler/4.1.50.Final/netty-handler-4.1.50.Final.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/netty/netty-resolver/4.1.50.Final/netty-resolver-4.1.50.Final.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/netty/netty-transport/4.1.50.Final/netty-transport-4.1.50.Final.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/elasticsearch/client/elasticsearch-rest-high-level-client/7.6.2/elasticsearch-rest-high-level-client-7.6.2.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/elasticsearch/elasticsearch/7.6.2/elasticsearch-7.6.2.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/elasticsearch/elasticsearch-core/7.6.2/elasticsearch-core-7.6.2.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/elasticsearch/elasticsearch-secure-sm/7.6.2/elasticsearch-secure-sm-7.6.2.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/elasticsearch/elasticsearch-x-content/7.6.2/elasticsearch-x-content-7.6.2.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/fasterxml/jackson/dataformat/jackson-dataformat-smile/2.11.0.rc1/jackson-dataformat-smile-2.11.0.rc1.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/fasterxml/jackson/dataformat/jackson-dataformat-yaml/2.11.0.rc1/jackson-dataformat-yaml-2.11.0.rc1.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/fasterxml/jackson/dataformat/jackson-dataformat-cbor/2.11.0.rc1/jackson-dataformat-cbor-2.11.0.rc1.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/fasterxml/jackson/dataformat/jackson-dataformat-smile/2.11.0/jackson-dataformat-smile-2.11.0.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/fasterxml/jackson/dataformat/jackson-dataformat-yaml/2.11.0/jackson-dataformat-yaml-2.11.0.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/fasterxml/jackson/dataformat/jackson-dataformat-cbor/2.11.0/jackson-dataformat-cbor-2.11.0.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/elasticsearch/elasticsearch-geo/7.6.2/elasticsearch-geo-7.6.2.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/apache/lucene/lucene-core/8.4.0/lucene-core-8.4.0.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/apache/lucene/lucene-analyzers-common/8.4.0/lucene-analyzers-common-8.4.0.jar" enabled="true" runInBatchMode="false"/>
@ -85,12 +64,13 @@
<factorypathentry kind="VARJAR" id="M2_REPO/org/elasticsearch/plugin/rank-eval-client/7.6.2/rank-eval-client-7.6.2.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/elasticsearch/plugin/lang-mustache-client/7.6.2/lang-mustache-client-7.6.2.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/github/spullara/mustache/java/compiler/0.9.6/compiler-0.9.6.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/fasterxml/jackson/core/jackson-core/2.11.0.rc1/jackson-core-2.11.0.rc1.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/boot/spring-boot-starter-data-jpa/2.3.0.M4/spring-boot-starter-data-jpa-2.3.0.M4.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/boot/spring-boot-starter-aop/2.3.0.M4/spring-boot-starter-aop-2.3.0.M4.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/boot/spring-boot-starter-jdbc/2.3.0.M4/spring-boot-starter-jdbc-2.3.0.M4.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/zaxxer/HikariCP/3.4.2/HikariCP-3.4.2.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/spring-jdbc/5.2.5.RELEASE/spring-jdbc-5.2.5.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/fasterxml/jackson/core/jackson-core/2.11.0/jackson-core-2.11.0.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/fasterxml/jackson/core/jackson-databind/2.11.0/jackson-databind-2.11.0.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/boot/spring-boot-starter-data-jpa/2.3.1.RELEASE/spring-boot-starter-data-jpa-2.3.1.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/boot/spring-boot-starter-aop/2.3.1.RELEASE/spring-boot-starter-aop-2.3.1.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/boot/spring-boot-starter-jdbc/2.3.1.RELEASE/spring-boot-starter-jdbc-2.3.1.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/zaxxer/HikariCP/3.4.5/HikariCP-3.4.5.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/spring-jdbc/5.2.7.RELEASE/spring-jdbc-5.2.7.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/jakarta/transaction/jakarta.transaction-api/1.3.3/jakarta.transaction-api-1.3.3.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/jakarta/persistence/jakarta.persistence-api/2.2.3/jakarta.persistence-api-2.2.3.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/hibernate/hibernate-core/5.2.0.Final/hibernate-core-5.2.0.Final.jar" enabled="true" runInBatchMode="false"/>
@ -100,30 +80,25 @@
<factorypathentry kind="VARJAR" id="M2_REPO/org/apache/geronimo/specs/geronimo-jta_1.1_spec/1.1.1/geronimo-jta_1.1_spec-1.1.1.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/jboss/jandex/2.0.0.Final/jandex-2.0.0.Final.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/dom4j/dom4j/1.6.1/dom4j-1.6.1.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/xml-apis/xml-apis/1.0.b2/xml-apis-1.0.b2.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/xml-apis/xml-apis/1.4.01/xml-apis-1.4.01.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/hibernate/common/hibernate-commons-annotations/5.0.1.Final/hibernate-commons-annotations-5.0.1.Final.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/javax/enterprise/cdi-api/1.1-PFD/cdi-api-1.1-PFD.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/javax/el/el-api/2.2/el-api-2.2.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/jboss/spec/javax/interceptor/jboss-interceptors-api_1.1_spec/1.0.0.Beta1/jboss-interceptors-api_1.1_spec-1.0.0.Beta1.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/javax/annotation/jsr250-api/1.0/jsr250-api-1.0.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/javax/inject/javax.inject/1/javax.inject-1.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/data/spring-data-jpa/2.3.0.RC1/spring-data-jpa-2.3.0.RC1.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/spring-orm/5.2.5.RELEASE/spring-orm-5.2.5.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/spring-aspects/5.2.5.RELEASE/spring-aspects-5.2.5.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/spring-core/5.2.5.RELEASE/spring-core-5.2.5.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/spring-jcl/5.2.5.RELEASE/spring-jcl-5.2.5.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/data/spring-data-jpa/2.3.1.RELEASE/spring-data-jpa-2.3.1.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/spring-orm/5.2.7.RELEASE/spring-orm-5.2.7.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/spring-beans/5.2.7.RELEASE/spring-beans-5.2.7.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/spring-aspects/5.2.7.RELEASE/spring-aspects-5.2.7.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/h2database/h2/1.4.200/h2-1.4.200.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/javax/validation/validation-api/2.0.1.Final/validation-api-2.0.1.Final.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/apache/commons/commons-lang3/3.10/commons-lang3-3.10.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/commons-codec/commons-codec/1.14/commons-codec-1.14.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/commons-io/commons-io/2.5/commons-io-2.5.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/google/code/gson/gson/2.8.6/gson-2.8.6.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/javax/mail/mail/1.4.7/mail-1.4.7.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/javax/activation/activation/1.1/activation-1.1.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/google/code/gson/gson/2.8.6/gson-2.8.6.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/springfox/springfox-swagger2/2.4.0/springfox-swagger2-2.4.0.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/swagger/swagger-annotations/1.5.6/swagger-annotations-1.5.6.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/swagger/swagger-models/1.5.6/swagger-models-1.5.6.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/fasterxml/jackson/core/jackson-annotations/2.11.0.rc1/jackson-annotations-2.11.0.rc1.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/fasterxml/jackson/core/jackson-annotations/2.11.0/jackson-annotations-2.11.0.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/springfox/springfox-spi/2.4.0/springfox-spi-2.4.0.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/springfox/springfox-core/2.4.0/springfox-core-2.4.0.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/springfox/springfox-schema/2.4.0/springfox-schema-2.4.0.jar" enabled="true" runInBatchMode="false"/>
@ -135,17 +110,36 @@
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/plugin/spring-plugin-core/1.2.0.RELEASE/spring-plugin-core-1.2.0.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/plugin/spring-plugin-metadata/1.2.0.RELEASE/spring-plugin-metadata-1.2.0.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/springfox/springfox-swagger-ui/2.4.0/springfox-swagger-ui-2.4.0.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/eu/dnetlib/dnet-openaire-broker-common/1.0.2/dnet-openaire-broker-common-1.0.2.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/eu/dnetlib/dnet-openaire-broker-common/3.0.4/dnet-openaire-broker-common-3.0.4.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/antlr/stringtemplate/3.2.1/stringtemplate-3.2.1.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/antlr/antlr/2.7.7/antlr-2.7.7.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/prometheus/simpleclient_spring_boot/0.2.0/simpleclient_spring_boot-0.2.0.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/prometheus/simpleclient/0.2.0/simpleclient-0.2.0.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/prometheus/simpleclient_common/0.2.0/simpleclient_common-0.2.0.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/boot/spring-boot-actuator/2.3.0.M4/spring-boot-actuator-2.3.0.M4.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/boot/spring-boot-actuator/2.3.1.RELEASE/spring-boot-actuator-2.3.1.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/prometheus/simpleclient_hotspot/0.2.0/simpleclient_hotspot-0.2.0.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/prometheus/simpleclient_servlet/0.2.0/simpleclient_servlet-0.2.0.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/io/prometheus/simpleclient_spring_web/0.3.0/simpleclient_spring_web-0.3.0.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/spring-aop/5.2.5.RELEASE/spring-aop-5.2.5.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/spring-context/5.2.5.RELEASE/spring-context-5.2.5.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/spring-web/5.2.7.RELEASE/spring-web-5.2.7.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/spring-aop/5.2.7.RELEASE/spring-aop-5.2.7.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/spring-context/5.2.7.RELEASE/spring-context-5.2.7.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/spring-expression/5.2.7.RELEASE/spring-expression-5.2.7.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/aspectj/aspectjweaver/1.9.5/aspectjweaver-1.9.5.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/boot/spring-boot-starter-web/2.3.1.RELEASE/spring-boot-starter-web-2.3.1.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/boot/spring-boot-starter-json/2.3.1.RELEASE/spring-boot-starter-json-2.3.1.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/fasterxml/jackson/datatype/jackson-datatype-jdk8/2.11.0/jackson-datatype-jdk8-2.11.0.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/fasterxml/jackson/datatype/jackson-datatype-jsr310/2.11.0/jackson-datatype-jsr310-2.11.0.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/com/fasterxml/jackson/module/jackson-module-parameter-names/2.11.0/jackson-module-parameter-names-2.11.0.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/boot/spring-boot-starter-tomcat/2.3.1.RELEASE/spring-boot-starter-tomcat-2.3.1.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/apache/tomcat/embed/tomcat-embed-core/9.0.36/tomcat-embed-core-9.0.36.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/glassfish/jakarta.el/3.0.3/jakarta.el-3.0.3.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/apache/tomcat/embed/tomcat-embed-websocket/9.0.36/tomcat-embed-websocket-9.0.36.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/spring-webmvc/5.2.7.RELEASE/spring-webmvc-5.2.7.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/boot/spring-boot-configuration-processor/2.3.1.RELEASE/spring-boot-configuration-processor-2.3.1.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/spring-core/5.2.7.RELEASE/spring-core-5.2.7.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/springframework/spring-jcl/5.2.7.RELEASE/spring-jcl-5.2.7.RELEASE.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/javax/validation/validation-api/2.0.1.Final/validation-api-2.0.1.Final.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/org/apache/commons/commons-lang3/3.10/commons-lang3-3.10.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/commons-codec/commons-codec/1.9/commons-codec-1.9.jar" enabled="true" runInBatchMode="false"/>
<factorypathentry kind="VARJAR" id="M2_REPO/commons-io/commons-io/2.4/commons-io-2.4.jar" enabled="true" runInBatchMode="false"/>
</factorypath>

0
dhp-broker-application/mvnw → apps/dhp-broker-application/mvnw vendored
View File

0
dhp-broker-application/mvnw.cmd → apps/dhp-broker-application/mvnw.cmd vendored
View File

50
apps/dhp-broker-application/pom.xml Normal file
View File

@ -0,0 +1,50 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>apps</artifactId>
<version>3.1.2-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dhp-broker-application</artifactId>
<packaging>jar</packaging>
<name>dhp-broker-application</name>
<description>D-Net Broker Service (new Hadoop implementation)</description>
<dependencies>
<!-- Mail -->
<dependency>
<groupId>javax.mail</groupId>
<artifactId>mail</artifactId>
<version>1.4.7</version>
</dependency>
<!-- Openaire -->
<dependency>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dnet-broker-apps-common</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<executable>true</executable>
</configuration>
</plugin>
</plugins>
</build>
</project>

2
dhp-broker-application/src/main/java/eu/dnetlib/lbs/LiteratureBrokerServiceApplication.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/LiteratureBrokerServiceApplication.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs;
package eu.dnetlib.broker;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;

51
dhp-broker-application/src/main/java/eu/dnetlib/lbs/LiteratureBrokerServiceConfiguration.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/LiteratureBrokerServiceConfiguration.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs;
package eu.dnetlib.broker;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -16,9 +16,9 @@ import org.springframework.data.elasticsearch.repository.config.EnableElasticsea
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.transaction.annotation.EnableTransactionManagement;
import eu.dnetlib.lbs.elasticsearch.Event;
import eu.dnetlib.lbs.elasticsearch.Notification;
import eu.dnetlib.lbs.properties.ElasticSearchProperties;
import eu.dnetlib.broker.common.elasticsearch.Event;
import eu.dnetlib.broker.common.elasticsearch.Notification;
import eu.dnetlib.broker.common.properties.ElasticSearchProperties;
import io.prometheus.client.exporter.MetricsServlet;
import io.prometheus.client.hotspot.DefaultExports;
import springfox.documentation.builders.RequestHandlerSelectors;
@ -32,7 +32,9 @@ import springfox.documentation.swagger2.annotations.EnableSwagger2;
@EnableSwagger2
@EnableScheduling
@EnableTransactionManagement
@EnableElasticsearchRepositories(basePackageClasses = { Event.class, Notification.class })
@EnableElasticsearchRepositories(basePackageClasses = {
Event.class, Notification.class
})
public class LiteratureBrokerServiceConfiguration extends AbstractElasticsearchConfiguration {
private static final Log log = LogFactory.getLog(LiteratureBrokerServiceConfiguration.class);
@ -45,13 +47,16 @@ public class LiteratureBrokerServiceConfiguration extends AbstractElasticsearchC
public static final String TAG_NOTIFICATIONS = "Notifications";
public static final String TAG_TOPIC_TYPES = "Topic Types";
public static final String TAG_OPENAIRE = "OpenAIRE";
public static final String TAG_MATCHING = "Subscription-Event Matching";
@Override
@Bean
public RestHighLevelClient elasticsearchClient() {
final ClientConfiguration clientConfiguration = ClientConfiguration.builder()
.connectedTo(elasticSearchProperties.getClusterNodes())
.build();
.connectedTo(elasticSearchProperties.getClusterNodes())
.withConnectTimeout(elasticSearchProperties.getConnectionTimeout())
.withSocketTimeout(elasticSearchProperties.getSocketTimeout())
.build();
return RestClients.create(clientConfiguration).rest();
}
@ -60,24 +65,20 @@ public class LiteratureBrokerServiceConfiguration extends AbstractElasticsearchC
log.info("Initializing SWAGGER...");
return new Docket(DocumentationType.SWAGGER_2)
.select()
.apis(RequestHandlerSelectors.any())
.paths(p -> p.startsWith("/api/"))
.build()
.tags(
new Tag(TAG_EVENTS, "Events management"),
new Tag(TAG_SUBSCRIPTIONS, "Subscriptions management"),
new Tag(TAG_NOTIFICATIONS, "Notifications management"),
new Tag(TAG_TOPIC_TYPES, "Topic types management"),
new Tag(TAG_OPENAIRE, "OpenAIRE use case"))
.apiInfo(new ApiInfo(
"Literature Broker Service",
"APIs documentation",
"1.1",
"urn:tos",
ApiInfo.DEFAULT_CONTACT,
"Apache 2.0",
"http://www.apache.org/licenses/LICENSE-2.0"));
.select()
.apis(RequestHandlerSelectors.any())
.paths(p -> p.startsWith("/api/"))
.build()
.tags(new Tag(TAG_EVENTS, "Events management"), new Tag(TAG_SUBSCRIPTIONS, "Subscriptions management"), new Tag(TAG_NOTIFICATIONS,
"Notifications management"), new Tag(TAG_TOPIC_TYPES, "Topic types management"), new Tag(TAG_OPENAIRE, "OpenAIRE use case"))
.apiInfo(new ApiInfo(
"Literature Broker Service",
"APIs documentation",
"1.1",
"urn:tos",
ApiInfo.DEFAULT_CONTACT,
"Apache 2.0",
"http://www.apache.org/licenses/LICENSE-2.0"));
}

33
dhp-broker-application/src/main/java/eu/dnetlib/lbs/controllers/AjaxController.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/controllers/AjaxController.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs.controllers;
package eu.dnetlib.broker.controllers;
import java.util.ArrayList;
import java.util.List;
@ -10,20 +10,21 @@ import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.lbs.controllers.objects.BufferStatus;
import eu.dnetlib.lbs.controllers.objects.CurrentStatus;
import eu.dnetlib.lbs.controllers.objects.DispatcherStatus;
import eu.dnetlib.lbs.controllers.objects.ThreadStatus;
import eu.dnetlib.lbs.controllers.objects.Tool;
import eu.dnetlib.lbs.elasticsearch.EventRepository;
import eu.dnetlib.lbs.elasticsearch.NotificationRepository;
import eu.dnetlib.lbs.events.output.DispatcherManager;
import eu.dnetlib.lbs.events.output.NotificationDispatcher;
import eu.dnetlib.lbs.subscriptions.SubscriptionRepository;
import eu.dnetlib.lbs.topics.TopicTypeRepository;
import eu.dnetlib.lbs.utils.LbsQueue;
import eu.dnetlib.lbs.utils.QueueManager;
import eu.dnetlib.lbs.utils.ThreadManager;
import eu.dnetlib.broker.common.controllers.AbstractLbsController;
import eu.dnetlib.broker.common.elasticsearch.EventRepository;
import eu.dnetlib.broker.common.elasticsearch.NotificationRepository;
import eu.dnetlib.broker.common.subscriptions.SubscriptionRepository;
import eu.dnetlib.broker.common.topics.TopicTypeRepository;
import eu.dnetlib.broker.controllers.objects.BufferStatus;
import eu.dnetlib.broker.controllers.objects.CurrentStatus;
import eu.dnetlib.broker.controllers.objects.DispatcherStatus;
import eu.dnetlib.broker.controllers.objects.ThreadStatus;
import eu.dnetlib.broker.controllers.objects.Tool;
import eu.dnetlib.broker.events.output.DispatcherManager;
import eu.dnetlib.broker.events.output.NotificationDispatcher;
import eu.dnetlib.broker.utils.LbsQueue;
import eu.dnetlib.broker.utils.QueueManager;
import eu.dnetlib.broker.utils.ThreadManager;
@RestController
@RequestMapping("/ajax")
@ -76,7 +77,7 @@ public class AjaxController extends AbstractLbsController {
final List<DispatcherStatus> dispatchers = dispatcherManager.getDispatchers()
.stream()
.map(d -> new DispatcherStatus(d.getDispatcherName(), d.count()))
.map(d -> new DispatcherStatus(d.getDispatcherName(), d.count(), d.countErrors(), d.lastError()))
.sorted()
.collect(Collectors.toList());

2
dhp-broker-application/src/main/java/eu/dnetlib/lbs/controllers/ApiDocController.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/controllers/ApiDocController.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs.controllers;
package eu.dnetlib.broker.controllers;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;

76
dhp-broker-application/src/main/java/eu/dnetlib/lbs/controllers/EventsController.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/controllers/EventsController.java
View File

@ -1,5 +1,6 @@
package eu.dnetlib.lbs.controllers;
package eu.dnetlib.broker.controllers;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -7,6 +8,8 @@ import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest;
import org.springframework.web.bind.annotation.DeleteMapping;
@ -20,14 +23,14 @@ import org.springframework.web.bind.annotation.RestController;
import com.google.common.collect.Lists;
import eu.dnetlib.lbs.LiteratureBrokerServiceConfiguration;
import eu.dnetlib.lbs.cron.ScheduledActions;
import eu.dnetlib.lbs.elasticsearch.Event;
import eu.dnetlib.lbs.elasticsearch.EventRepository;
import eu.dnetlib.lbs.elasticsearch.EventStatsManager;
import eu.dnetlib.lbs.elasticsearch.EventStatsManager.BrowseEntry;
import eu.dnetlib.lbs.subscriptions.Subscription;
import eu.dnetlib.lbs.subscriptions.SubscriptionRepository;
import eu.dnetlib.broker.LiteratureBrokerServiceConfiguration;
import eu.dnetlib.broker.common.controllers.AbstractLbsController;
import eu.dnetlib.broker.common.elasticsearch.Event;
import eu.dnetlib.broker.common.elasticsearch.EventRepository;
import eu.dnetlib.broker.common.elasticsearch.EventStatsManager;
import eu.dnetlib.broker.common.elasticsearch.EventStatsManager.BrowseEntry;
import eu.dnetlib.broker.common.subscriptions.Subscription;
import eu.dnetlib.broker.common.subscriptions.SubscriptionRepository;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
@ -36,6 +39,8 @@ import io.swagger.annotations.ApiOperation;
@Api(tags = LiteratureBrokerServiceConfiguration.TAG_EVENTS)
public class EventsController extends AbstractLbsController {
private static final Log log = LogFactory.getLog(EventsController.class);
@Autowired
private EventRepository eventRepository;
@ -45,9 +50,6 @@ public class EventsController extends AbstractLbsController {
@Autowired
private EventStatsManager eventStatsManager;
@Autowired
private ScheduledActions scheduledActions;
@ApiOperation("Return an event by ID")
@GetMapping("/{id}")
public Event getEvent(@PathVariable final String id) {
@ -69,17 +71,17 @@ public class EventsController extends AbstractLbsController {
@ApiOperation("Return a page of events")
@GetMapping("/list/{page}/{pageSize}")
public List<Event> events(
@PathVariable final int page,
@PathVariable final int pageSize) {
@PathVariable final int page,
@PathVariable final int pageSize) {
return Lists.newArrayList(eventRepository.findAll(PageRequest.of(page, pageSize)));
}
@ApiOperation("Return a page of events by topic")
@GetMapping("/byTopic/{page}/{pageSize}")
public List<Event> eventsByTopic(
@PathVariable final int page,
@PathVariable final int pageSize,
@RequestParam final String topic) {
@PathVariable final int page,
@PathVariable final int pageSize,
@RequestParam final String topic) {
return Lists.newArrayList(eventRepository.findByTopic(topic, PageRequest.of(page, pageSize)));
}
@ -94,9 +96,35 @@ public class EventsController extends AbstractLbsController {
@ApiOperation("Delete the expired events")
@DeleteMapping("/expired")
public Map<String, Long> deleteExpiredEvents() {
public Map<String, Object> deleteExpiredEvents() {
return deleteEventsByExpiryDate(0, new Date().getTime());
}
@ApiOperation("Delete the events with the creationDate in a range")
@DeleteMapping("/byCreationDate/{from}/{to}")
public Map<String, Long> deleteEventsByCreationDate(@PathVariable final long from, @PathVariable final long to) {
final Map<String, Long> res = new HashMap<>();
res.put("deleted", scheduledActions.deleteExpiredEvents());
res.put("deleted", eventRepository.deleteByCreationDateBetween(from, to));
res.put("from", from);
res.put("to", to);
log.info(String.format("Deleted %s events in creationDate range %s-%s", res.get("deleted"), from, to));
return res;
}
@ApiOperation("Delete the events with the expiryDate in a range")
@DeleteMapping("/byExpiryDate/{from}/{to}")
public Map<String, Object> deleteEventsByExpiryDate(@PathVariable final long from, @PathVariable final long to) {
new Thread(() -> {
final long n = eventRepository.deleteByExpiryDateBetween(from, to);
log.info(String.format("Deleted %s events in expiryDate range %s-%s", n, from, to));
}).start();
final Map<String, Object> res = new HashMap<>();
res.put("status", "deleting...");
res.put("from", from);
res.put("to", to);
return res;
}
@ -113,13 +141,13 @@ public class EventsController extends AbstractLbsController {
final Iterable<Subscription> iter = subscriptionRepo.findAll();
final Set<String> validTopics = StreamSupport.stream(iter.spliterator(), false)
.map(Subscription::getTopic)
.collect(Collectors.toSet());
.map(Subscription::getTopic)
.collect(Collectors.toSet());
return this.browseTopics()
.stream()
.filter(e -> validTopics.contains(e.getValue()))
.collect(Collectors.toList());
.stream()
.filter(e -> validTopics.contains(e.getValue()))
.collect(Collectors.toList());
}
}

31
dhp-broker-application/src/main/java/eu/dnetlib/lbs/controllers/NotificationsController.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/controllers/NotificationsController.java
View File

@ -1,5 +1,10 @@
package eu.dnetlib.lbs.controllers;
package eu.dnetlib.broker.controllers;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
@ -9,9 +14,10 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.lbs.LiteratureBrokerServiceConfiguration;
import eu.dnetlib.lbs.elasticsearch.Notification;
import eu.dnetlib.lbs.elasticsearch.NotificationRepository;
import eu.dnetlib.broker.LiteratureBrokerServiceConfiguration;
import eu.dnetlib.broker.common.controllers.AbstractLbsController;
import eu.dnetlib.broker.common.elasticsearch.Notification;
import eu.dnetlib.broker.common.elasticsearch.NotificationRepository;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
@ -20,6 +26,8 @@ import io.swagger.annotations.ApiOperation;
@Api(tags = LiteratureBrokerServiceConfiguration.TAG_NOTIFICATIONS)
public class NotificationsController extends AbstractLbsController {
private static final Log log = LogFactory.getLog(NotificationsController.class);
@Autowired
private NotificationRepository notificationRepository;
@ -47,4 +55,19 @@ public class NotificationsController extends AbstractLbsController {
notificationRepository.deleteAll();
}
@ApiOperation("Delete the notifications with the date in a range")
@DeleteMapping("/byDate/{from}/{to}")
public Map<String, Object> deleteNotificationsByDate(@PathVariable final long from, @PathVariable final long to) {
new Thread(() -> {
final long n = notificationRepository.deleteByDateBetween(from, to);
log.info(String.format("Deleted %s notifications in date range %s-%s", n, from, to));
}).start();
final Map<String, Object> res = new HashMap<>();
res.put("status", "deleting...");
res.put("from", from);
res.put("to", to);
return res;
}
}

60
apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/controllers/StartMatchingController.java Normal file
View File

@ -0,0 +1,60 @@
package eu.dnetlib.broker.controllers;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Profile;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.broker.LiteratureBrokerServiceConfiguration;
import eu.dnetlib.broker.common.controllers.AbstractLbsController;
import eu.dnetlib.broker.common.subscriptions.Subscription;
import eu.dnetlib.broker.common.subscriptions.SubscriptionRepository;
import eu.dnetlib.broker.matchers.SubscriptionEventMatcher;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
@Profile("!openaire")
@RestController
@RequestMapping("/api/matching")
@Api(tags = LiteratureBrokerServiceConfiguration.TAG_MATCHING)
public class StartMatchingController extends AbstractLbsController {
@Autowired
private SubscriptionRepository subscriptionRepo;
@Autowired(required = false)
private SubscriptionEventMatcher subscriptionEventMatcher;
@ApiOperation("Launch the thread that produces new notifications")
@GetMapping("/start")
public List<String> startMatching() {
if (subscriptionEventMatcher != null) {
this.subscriptionRepo.findAll().forEach(this.subscriptionEventMatcher::startMatching);
return Arrays.asList("done");
} else {
return Arrays.asList("matching is disabled");
}
}
@ApiOperation("Launch the thread that produces new notifications by subscriptuion id")
@GetMapping("/start/{subscriptionId}")
public List<String> startMatching(@PathVariable final String subscriptionId) {
final Optional<Subscription> s = subscriptionRepo.findById(subscriptionId);
if (subscriptionEventMatcher == null) {
return Arrays.asList("matching is disabled");
} else if (s.isPresent()) {
subscriptionEventMatcher.startMatching(s.get());
return Arrays.asList("done");
} else {
return Arrays.asList("subscription not present");
}
}
}

50
dhp-broker-application/src/main/java/eu/dnetlib/lbs/controllers/SubscriptionsController.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/controllers/SubscriptionsController.java
View File

@ -1,11 +1,9 @@
package eu.dnetlib.lbs.controllers;
package eu.dnetlib.broker.controllers;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.function.Predicate;
import java.util.regex.Pattern;
@ -23,15 +21,14 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.lbs.LiteratureBrokerServiceConfiguration;
import eu.dnetlib.lbs.cron.ScheduledActions;
import eu.dnetlib.lbs.elasticsearch.NotificationRepository;
import eu.dnetlib.lbs.matchers.SubscriptionEventMatcher;
import eu.dnetlib.lbs.subscriptions.MapCondition;
import eu.dnetlib.lbs.subscriptions.NotificationFrequency;
import eu.dnetlib.lbs.subscriptions.NotificationMode;
import eu.dnetlib.lbs.subscriptions.Subscription;
import eu.dnetlib.lbs.subscriptions.SubscriptionRepository;
import eu.dnetlib.broker.LiteratureBrokerServiceConfiguration;
import eu.dnetlib.broker.common.controllers.AbstractLbsController;
import eu.dnetlib.broker.common.elasticsearch.NotificationRepository;
import eu.dnetlib.broker.common.subscriptions.MapCondition;
import eu.dnetlib.broker.common.subscriptions.NotificationFrequency;
import eu.dnetlib.broker.common.subscriptions.NotificationMode;
import eu.dnetlib.broker.common.subscriptions.Subscription;
import eu.dnetlib.broker.common.subscriptions.SubscriptionRepository;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
@ -46,13 +43,7 @@ public class SubscriptionsController extends AbstractLbsController {
@Autowired
private NotificationRepository notificationRepo;
@Autowired
private ScheduledActions schedulerActions;
@Autowired
private SubscriptionEventMatcher subscriptionEventMatcher;
public static final Predicate<String> verifyTopic = Pattern.compile("^[a-zA-Z0-9._-]+(\\/[a-zA-Z0-9._-]+)+$").asPredicate();
public static final Predicate<String> verifyTopic = Pattern.compile("^([a-zA-Z0-9._-]+(\\/[a-zA-Z0-9._-]+)+)|\\*$").asPredicate();
public static final Predicate<String> verifyEmail = email -> {
try {
new InternetAddress(email).validate();
@ -99,25 +90,6 @@ public class SubscriptionsController extends AbstractLbsController {
return res;
}
@ApiOperation("Launch the thread that produces new notifications")
@GetMapping("/startMatching")
public List<String> startMatching() {
schedulerActions.startSubscriptionEventsMatcher();
return Arrays.asList("done");
}
@ApiOperation("Launch the thread that produces new notifications by subscriptuion id")
@GetMapping("/startMatching/{subscriptionId}")
public List<String> startMatching(@PathVariable final String subscriptionId) {
final Optional<Subscription> s = subscriptionRepo.findById(subscriptionId);
if (s.isPresent()) {
subscriptionEventMatcher.startMatching(s.get());
return Arrays.asList("done");
} else {
return Arrays.asList("subscription not present");
}
}
@ApiOperation("Reset the last notification date")
@DeleteMapping("/{id}/date")
public void deleteNotificationDate(@PathVariable final String id) {
@ -147,7 +119,7 @@ class InSubscription {
public InSubscription() {}
public InSubscription(final String subscriber, final String topic, final NotificationFrequency frequency, final NotificationMode mode,
final List<MapCondition> conditions) {
final List<MapCondition> conditions) {
this.subscriber = subscriber;
this.topic = topic;
this.frequency = frequency;

17
dhp-broker-application/src/main/java/eu/dnetlib/lbs/controllers/TopicsController.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/controllers/TopicsController.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs.controllers;
package eu.dnetlib.broker.controllers;
import java.util.Arrays;
import java.util.HashMap;
@ -18,9 +18,10 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.lbs.LiteratureBrokerServiceConfiguration;
import eu.dnetlib.lbs.topics.TopicType;
import eu.dnetlib.lbs.topics.TopicTypeRepository;
import eu.dnetlib.broker.LiteratureBrokerServiceConfiguration;
import eu.dnetlib.broker.common.controllers.AbstractLbsController;
import eu.dnetlib.broker.common.topics.TopicType;
import eu.dnetlib.broker.common.topics.TopicTypeRepository;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
@ -33,7 +34,7 @@ public class TopicsController extends AbstractLbsController {
private TopicTypeRepository topicTypeRepo;
private final Predicate<String> verifyExpression =
Pattern.compile("^([a-zA-Z0-9._-]+|<[a-zA-Z0-9._-]+>)(\\/([a-zA-Z0-9._-]+|<[a-zA-Z0-9._-]+>))+$").asPredicate();
Pattern.compile("^([a-zA-Z0-9._-]+|<[a-zA-Z0-9._-]+>)(\\/([a-zA-Z0-9._-]+|<[a-zA-Z0-9._-]+>))+$").asPredicate();
@ApiOperation("Return the list of topic types")
@GetMapping("")
@ -44,9 +45,9 @@ public class TopicsController extends AbstractLbsController {
@ApiOperation("Register a new topic type")
@PostMapping("/add")
public TopicType registerTopicType(@RequestParam final String name,
@RequestParam final String expression,
@RequestParam final String producerId,
@RequestParam final String mapKeys) {
@RequestParam final String expression,
@RequestParam final String producerId,
@RequestParam final String mapKeys) {
if (StringUtils.isBlank(name)) { throw new IllegalArgumentException("name is empty"); }
if (StringUtils.isBlank(expression)) { throw new IllegalArgumentException("expression is empty"); }

2
dhp-broker-application/src/main/java/eu/dnetlib/lbs/controllers/objects/BufferStatus.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/controllers/objects/BufferStatus.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs.controllers.objects;
package eu.dnetlib.broker.controllers.objects;
public class BufferStatus implements Comparable<BufferStatus> {

18
dhp-broker-application/src/main/java/eu/dnetlib/lbs/controllers/objects/CurrentStatus.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/controllers/objects/CurrentStatus.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs.controllers.objects;
package eu.dnetlib.broker.controllers.objects;
import java.util.ArrayList;
import java.util.HashMap;
@ -8,15 +8,15 @@ import java.util.Map;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import eu.dnetlib.lbs.properties.DatabaseProperties;
import eu.dnetlib.lbs.properties.ElasticSearchProperties;
import eu.dnetlib.lbs.properties.EmailProperties;
import eu.dnetlib.broker.common.properties.DatabaseProperties;
import eu.dnetlib.broker.common.properties.ElasticSearchProperties;
import eu.dnetlib.broker.properties.EmailProperties;
@Component
public class CurrentStatus {
@Autowired
private DatabaseProperties h2;
private DatabaseProperties db;
@Autowired
private ElasticSearchProperties elasticSearch;
@ -72,12 +72,12 @@ public class CurrentStatus {
this.elasticSearch = elasticSearch;
}
public DatabaseProperties getH2() {
return this.h2;
public DatabaseProperties getDb() {
return this.db;
}
public void setH2(final DatabaseProperties h2) {
this.h2 = h2;
public void setDb(final DatabaseProperties db) {
this.db = db;
}
}

40
apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/controllers/objects/DispatcherStatus.java Normal file
View File

@ -0,0 +1,40 @@
package eu.dnetlib.broker.controllers.objects;
public class DispatcherStatus implements Comparable<DispatcherStatus> {
private final String name;
private final long count;
private final long countErrors;
private final String lastError;
public DispatcherStatus(final String name, final long count, final long countErrors, final String lastError) {
this.name = name;
this.count = count;
this.countErrors = countErrors;
this.lastError = lastError;
}
public String getName() {
return name;
}
public long getCount() {
return count;
}
public long getCountErrors() {
return countErrors;
}
public String getLastError() {
return lastError;
}
@Override
public int compareTo(final DispatcherStatus o) {
if (this.name == null) { return -1; }
if (o.name == null) { return 1; }
return this.name.compareTo(o.name);
}
}

2
dhp-broker-application/src/main/java/eu/dnetlib/lbs/controllers/objects/ThreadStatus.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/controllers/objects/ThreadStatus.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs.controllers.objects;
package eu.dnetlib.broker.controllers.objects;
public class ThreadStatus implements Comparable<ThreadStatus> {

2
dhp-broker-application/src/main/java/eu/dnetlib/lbs/controllers/objects/Tool.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/controllers/objects/Tool.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs.controllers.objects;
package eu.dnetlib.broker.controllers.objects;
public class Tool {

26
dhp-broker-application/src/main/java/eu/dnetlib/lbs/cron/ScheduledActions.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/cron/ScheduledActions.java
View File

@ -1,27 +1,25 @@
package eu.dnetlib.lbs.cron;
package eu.dnetlib.broker.cron;
import java.util.Date;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Profile;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import eu.dnetlib.lbs.elasticsearch.EventRepository;
import eu.dnetlib.lbs.elasticsearch.NotificationRepository;
import eu.dnetlib.lbs.matchers.SubscriptionEventMatcher;
import eu.dnetlib.lbs.subscriptions.SubscriptionRepository;
import eu.dnetlib.broker.common.elasticsearch.EventRepository;
import eu.dnetlib.broker.common.subscriptions.SubscriptionRepository;
import eu.dnetlib.broker.matchers.SubscriptionEventMatcher;
@Component
@Profile("!openaire")
public class ScheduledActions {
@Autowired
private EventRepository eventRepository;
@Autowired
private NotificationRepository notificationRepository;
@Autowired
private SubscriptionRepository subscriptionRepo;
@ -32,21 +30,11 @@ public class ScheduledActions {
@Scheduled(cron = "${lbs.task.deleteOldEvents.cron}")
public long deleteExpiredEvents() {
// TODO : the date used in the query should be the min among the dates of the subscriptions (considering only the subscriptions with
// creation_date > events.creationDate)
final long n = this.eventRepository.deleteByExpiryDateBefore(new Date().getTime());
final long n = this.eventRepository.deleteByExpiryDateBetween(0, new Date().getTime());
log.info("Number of deleted expired events: " + n);
return n;
}
@Scheduled(cron = "${lbs.task.deleteOldNotifications.cron}")
public long deleteExpiredNotifications() {
// TODO
return 0;
}
@Scheduled(cron = "${lbs.task.subscriptionEventsMatcher.cron}")
public void startSubscriptionEventsMatcher() {
this.subscriptionRepo.findAll().forEach(this.subscriptionEventMatcher::startMatching);

40
dhp-broker-application/src/main/java/eu/dnetlib/lbs/events/manager/EventManager.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/events/manager/EventManager.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs.events.manager;
package eu.dnetlib.broker.events.manager;
import java.util.List;
import java.util.stream.StreamSupport;
@ -6,15 +6,15 @@ import java.util.stream.StreamSupport;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import eu.dnetlib.lbs.elasticsearch.Event;
import eu.dnetlib.lbs.elasticsearch.EventRepository;
import eu.dnetlib.lbs.elasticsearch.Notification;
import eu.dnetlib.lbs.elasticsearch.NotificationRepository;
import eu.dnetlib.lbs.events.output.DispatcherManager;
import eu.dnetlib.lbs.subscriptions.NotificationFrequency;
import eu.dnetlib.lbs.subscriptions.Subscription;
import eu.dnetlib.lbs.subscriptions.SubscriptionRepository;
import eu.dnetlib.lbs.utils.LbsQueue;
import eu.dnetlib.broker.common.elasticsearch.Event;
import eu.dnetlib.broker.common.elasticsearch.EventRepository;
import eu.dnetlib.broker.common.elasticsearch.Notification;
import eu.dnetlib.broker.common.elasticsearch.NotificationRepository;
import eu.dnetlib.broker.common.subscriptions.NotificationFrequency;
import eu.dnetlib.broker.common.subscriptions.Subscription;
import eu.dnetlib.broker.common.subscriptions.SubscriptionRepository;
import eu.dnetlib.broker.events.output.DispatcherManager;
import eu.dnetlib.broker.utils.LbsQueue;
public class EventManager implements Runnable {
@ -27,9 +27,9 @@ public class EventManager implements Runnable {
private static final Log log = LogFactory.getLog(EventManager.class);
public EventManager(final EventRepository eventRepository, final NotificationRepository notificationRepository,
final SubscriptionRepository subscriptionRepo,
final DispatcherManager dispatcherManager,
final LbsQueue<String, Event> queue) {
final SubscriptionRepository subscriptionRepo,
final DispatcherManager dispatcherManager,
final LbsQueue<String, Event> queue) {
this.eventRepository = eventRepository;
this.notificationRepository = notificationRepository;
this.subscriptionRepo = subscriptionRepo;
@ -52,13 +52,13 @@ public class EventManager implements Runnable {
list.stream().filter(Event::isInstantMessage).forEach(e -> {
final Iterable<Subscription> iter = this.subscriptionRepo.findByTopic(e.getTopic());
StreamSupport.stream(iter.spliterator(), false)
.filter(s -> s.verifyEventConditions(e))
.filter(s -> s.getFrequency() == NotificationFrequency.realtime)
.forEach(s -> {
final Notification n = new Notification(s, e);
this.notificationRepository.save(n);
this.dispatcherManager.dispatch(s, e);
});
.filter(s -> s.verifyEventConditions(e))
.filter(s -> s.getFrequency() == NotificationFrequency.realtime)
.forEach(s -> {
final Notification n = new Notification(s, e);
this.notificationRepository.save(n);
this.dispatcherManager.dispatch(s, e);
});
});
}
}

14
dhp-broker-application/src/main/java/eu/dnetlib/lbs/events/manager/EventManagerFactory.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/events/manager/EventManagerFactory.java
View File

@ -1,14 +1,14 @@
package eu.dnetlib.lbs.events.manager;
package eu.dnetlib.broker.events.manager;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import eu.dnetlib.lbs.elasticsearch.Event;
import eu.dnetlib.lbs.elasticsearch.EventRepository;
import eu.dnetlib.lbs.elasticsearch.NotificationRepository;
import eu.dnetlib.lbs.events.output.DispatcherManager;
import eu.dnetlib.lbs.subscriptions.SubscriptionRepository;
import eu.dnetlib.lbs.utils.LbsQueue;
import eu.dnetlib.broker.common.elasticsearch.Event;
import eu.dnetlib.broker.common.elasticsearch.EventRepository;
import eu.dnetlib.broker.common.elasticsearch.NotificationRepository;
import eu.dnetlib.broker.common.subscriptions.SubscriptionRepository;
import eu.dnetlib.broker.events.output.DispatcherManager;
import eu.dnetlib.broker.utils.LbsQueue;
@Component
public class EventManagerFactory {

45
dhp-broker-application/src/main/java/eu/dnetlib/lbs/events/output/AbstractNotificationDispatcher.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/events/output/AbstractNotificationDispatcher.java
View File

@ -1,5 +1,6 @@
package eu.dnetlib.lbs.events.output;
package eu.dnetlib.broker.events.output;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import javax.annotation.PostConstruct;
@ -10,11 +11,13 @@ import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.BeanNameAware;
import org.springframework.beans.factory.annotation.Autowired;
import eu.dnetlib.lbs.elasticsearch.Event;
import eu.dnetlib.lbs.subscriptions.Subscription;
import eu.dnetlib.lbs.utils.LbsQueue;
import eu.dnetlib.lbs.utils.QueueManager;
import eu.dnetlib.lbs.utils.ThreadManager;
import com.google.common.base.Throwables;
import eu.dnetlib.broker.common.elasticsearch.Event;
import eu.dnetlib.broker.common.subscriptions.Subscription;
import eu.dnetlib.broker.utils.LbsQueue;
import eu.dnetlib.broker.utils.QueueManager;
import eu.dnetlib.broker.utils.ThreadManager;
public abstract class AbstractNotificationDispatcher<T> implements NotificationDispatcher, BeanNameAware, Runnable {
@ -30,6 +33,10 @@ public abstract class AbstractNotificationDispatcher<T> implements NotificationD
private final AtomicLong count = new AtomicLong(0);
private final AtomicLong countErrors = new AtomicLong(0);
private String lastError = "";
private static final Log log = LogFactory.getLog(AbstractNotificationDispatcher.class);
@PostConstruct
@ -48,7 +55,8 @@ public abstract class AbstractNotificationDispatcher<T> implements NotificationD
this.count.incrementAndGet();
} catch (final Throwable e) {
log.error("Error sending notification", e);
this.queue.offer(message);
this.countErrors.incrementAndGet();
this.lastError = e.getMessage() + "\nStacktrave:\n" + Throwables.getStackTraceAsString(e);
}
}
}
@ -64,6 +72,17 @@ public abstract class AbstractNotificationDispatcher<T> implements NotificationD
}
}
@Override
public void sendNotification(final Subscription subscription, final Map<String, Object> params) {
try {
this.queue.offer(prepareAction(subscription, params));
} catch (final Exception e) {
log.error("Error sending notification", e);
}
}
abstract protected T prepareAction(final Subscription subscription, final Map<String, Object> params) throws Exception;
abstract protected T prepareAction(final Subscription subscription, final Event... events) throws Exception;
abstract protected void performAction(final T message) throws Exception;
@ -82,9 +101,21 @@ public abstract class AbstractNotificationDispatcher<T> implements NotificationD
return this.count.get();
}
@Override
public long countErrors() {
return this.countErrors.get();
}
@Override
public String lastError() {
return lastError;
}
@Override
public void resetCount() {
this.count.set(0);
this.countErrors.set(0);
this.lastError = "";
}
@Override

26
dhp-broker-application/src/main/java/eu/dnetlib/lbs/events/output/DispatcherManager.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/events/output/DispatcherManager.java
View File

@ -1,7 +1,8 @@
package eu.dnetlib.lbs.events.output;
package eu.dnetlib.broker.events.output;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.apache.commons.logging.Log;
@ -9,8 +10,8 @@ import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import eu.dnetlib.lbs.elasticsearch.Event;
import eu.dnetlib.lbs.subscriptions.Subscription;
import eu.dnetlib.broker.common.elasticsearch.Event;
import eu.dnetlib.broker.common.subscriptions.Subscription;
@Component
public class DispatcherManager {
@ -22,9 +23,9 @@ public class DispatcherManager {
public void dispatch(final Subscription s, final Event... events) {
final Optional<NotificationDispatcher> dispatcher = this.dispatchers
.stream()
.filter(d -> d.getMode() == s.getMode())
.findFirst();
.stream()
.filter(d -> d.getMode() == s.getMode())
.findFirst();
if (dispatcher.isPresent()) {
dispatcher.get().sendNotification(s, events);
@ -33,6 +34,19 @@ public class DispatcherManager {
}
}
public void sendNotification(final Subscription s, final Map<String, Object> params) {
final Optional<NotificationDispatcher> dispatcher = this.dispatchers
.stream()
.filter(d -> d.getMode() == s.getMode())
.findFirst();
if (dispatcher.isPresent()) {
dispatcher.get().sendNotification(s, params);
} else {
log.error("Notification dispatcher not found, mode=" + s.getMode());
}
}
public List<NotificationDispatcher> getDispatchers() {
return this.dispatchers;
}

74
dhp-broker-application/src/main/java/eu/dnetlib/lbs/events/output/EmailDispatcher.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/events/output/EmailDispatcher.java
View File

@ -1,17 +1,20 @@
package eu.dnetlib.lbs.events.output;
package eu.dnetlib.broker.events.output;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Date;
import java.util.Map;
import java.util.Properties;
import java.util.stream.Collectors;
import javax.mail.Authenticator;
import javax.mail.Message;
import javax.mail.MessagingException;
import javax.mail.PasswordAuthentication;
import javax.mail.Session;
import javax.mail.Transport;
import javax.mail.internet.AddressException;
import javax.mail.internet.InternetAddress;
import javax.mail.internet.MimeMessage;
@ -27,10 +30,10 @@ import org.springframework.stereotype.Component;
import com.google.common.base.Splitter;
import eu.dnetlib.lbs.elasticsearch.Event;
import eu.dnetlib.lbs.properties.EmailProperties;
import eu.dnetlib.lbs.subscriptions.NotificationMode;
import eu.dnetlib.lbs.subscriptions.Subscription;
import eu.dnetlib.broker.common.elasticsearch.Event;
import eu.dnetlib.broker.common.subscriptions.NotificationMode;
import eu.dnetlib.broker.common.subscriptions.Subscription;
import eu.dnetlib.broker.properties.EmailProperties;
@Component
public class EmailDispatcher extends AbstractNotificationDispatcher<Message> {
@ -54,17 +57,51 @@ public class EmailDispatcher extends AbstractNotificationDispatcher<Message> {
}
if (events.length == 0) {
log.warn("Event list is empty");
throw new IllegalArgumentException("Event list is empty");
}
final String topics = Arrays.stream(events).map(e -> e.getTopic()).distinct().collect(Collectors.joining(", "));
final String content = generateMailContent(subscription, null, events);
return prepareMimeMessage(subscription, content);
}
@Override
protected Message prepareAction(final Subscription subscription, final Map<String, Object> params) throws Exception {
if (subscription == null || StringUtils.isBlank(subscription.getSubscriber())) {
log.warn("Invalid subscription");
throw new IllegalArgumentException("Invalid subscription");
}
final String content = generateMailContent(subscription, params);
return prepareMimeMessage(subscription, content);
}
private String generateMailContent(final Subscription subscription, final Map<String, Object> params, final Event... events) throws IOException {
final StringTemplate st = new StringTemplate(IOUtils.toString(emailTemplate.getInputStream(), StandardCharsets.UTF_8));
st.setAttribute("sub", subscription);
st.setAttribute("total", events.length);
st.setAttribute("max", MAX_NUMBER_OF_EVENTS);
if (events.length > MAX_NUMBER_OF_EVENTS) {
st.setAttribute("events", Arrays.copyOfRange(events, 0, MAX_NUMBER_OF_EVENTS));
} else {
st.setAttribute("events", events);
}
if (params != null) {
params.entrySet().forEach(e -> st.setAttribute(e.getKey(), e.getValue()));
}
return st.toString();
}
private Message prepareMimeMessage(final Subscription subscription, final String content)
throws MessagingException, UnsupportedEncodingException, AddressException {
final Session session = Session.getInstance(obtainProperties(), obtainAuthenticator());
final MimeMessage mimeMessage = new MimeMessage(session);
mimeMessage.setFrom(new InternetAddress(props.getFrom(), props.getFromName()));
mimeMessage.setSubject("Notification for topic(s): " + topics);
mimeMessage.setContent(generateMailContent(subscription, events), "text/html; charset=utf-8");
mimeMessage.setSubject("Notification for topic: " + subscription.getTopic());
mimeMessage.setContent(content, "text/html; charset=utf-8");
mimeMessage.setSentDate(new Date());
mimeMessage.addRecipient(Message.RecipientType.TO, new InternetAddress(subscription.getSubscriber()));
@ -80,21 +117,6 @@ public class EmailDispatcher extends AbstractNotificationDispatcher<Message> {
return mimeMessage;
}
private String generateMailContent(final Subscription subscription, final Event... events) throws IOException {
final StringTemplate st = new StringTemplate(IOUtils.toString(emailTemplate.getInputStream(), StandardCharsets.UTF_8));
st.setAttribute("sub", subscription);
st.setAttribute("total", events.length);
st.setAttribute("max", MAX_NUMBER_OF_EVENTS);
if (events.length > MAX_NUMBER_OF_EVENTS) {
st.setAttribute("events", Arrays.copyOfRange(events, 0, MAX_NUMBER_OF_EVENTS));
} else {
st.setAttribute("events", events);
}
return st.toString();
}
@Override
protected void performAction(final Message message) throws Exception {
log.info("Sending mail to " + Arrays.toString(message.getAllRecipients()) + "...");
@ -117,7 +139,7 @@ public class EmailDispatcher extends AbstractNotificationDispatcher<Message> {
return new Authenticator() {
private final PasswordAuthentication authentication =
new PasswordAuthentication(props.getSmtpUser(), props.getSmtpPassword());
new PasswordAuthentication(props.getSmtpUser(), props.getSmtpPassword());
@Override
protected PasswordAuthentication getPasswordAuthentication() {

20
dhp-broker-application/src/main/java/eu/dnetlib/lbs/events/output/MockDispatcher.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/events/output/MockDispatcher.java
View File

@ -1,15 +1,16 @@
package eu.dnetlib.lbs.events.output;
package eu.dnetlib.broker.events.output;
import java.io.StringWriter;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Component;
import eu.dnetlib.lbs.elasticsearch.Event;
import eu.dnetlib.lbs.subscriptions.NotificationMode;
import eu.dnetlib.lbs.subscriptions.Subscription;
import eu.dnetlib.broker.common.elasticsearch.Event;
import eu.dnetlib.broker.common.subscriptions.NotificationMode;
import eu.dnetlib.broker.common.subscriptions.Subscription;
@Component
@Profile("dev")
@ -28,6 +29,17 @@ public class MockDispatcher extends AbstractNotificationDispatcher<String> {
return sw.toString();
}
@Override
protected String prepareAction(final Subscription subscription, final Map<String, Object> params) throws Exception {
final StringWriter sw = new StringWriter();
sw.write("\n********************************");
sw.write("\n* New notification");
sw.write("\n* - Subscription : " + subscription);
params.entrySet().forEach(e -> sw.write("\n* - " + e.getKey() + ": " + e.getValue()));
sw.write("\n********************************");
return sw.toString();
}
@Override
protected void performAction(final String message) throws Exception {
log.info(message);

26
apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/events/output/NotificationDispatcher.java Normal file
View File

@ -0,0 +1,26 @@
package eu.dnetlib.broker.events.output;
import java.util.Map;
import eu.dnetlib.broker.common.elasticsearch.Event;
import eu.dnetlib.broker.common.subscriptions.NotificationMode;
import eu.dnetlib.broker.common.subscriptions.Subscription;
public interface NotificationDispatcher {
void sendNotification(Subscription subscription, Event... events);
void resetCount();
String getDispatcherName();
long count();
NotificationMode getMode();
void sendNotification(Subscription subscription, Map<String, Object> params);
long countErrors();
String lastError();
}

53
dhp-broker-application/src/main/java/eu/dnetlib/lbs/matchers/SubscriptionEventMatcher.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/matchers/SubscriptionEventMatcher.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs.matchers;
package eu.dnetlib.broker.matchers;
import java.util.ArrayList;
import java.util.Date;
@ -12,8 +12,10 @@ import org.apache.commons.logging.LogFactory;
import org.apache.lucene.search.join.ScoreMode;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.Operator;
import org.elasticsearch.index.query.QueryBuilders;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Profile;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
import org.springframework.data.elasticsearch.core.SearchHit;
@ -23,18 +25,19 @@ import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilde
import org.springframework.data.util.CloseableIterator;
import org.springframework.stereotype.Component;
import eu.dnetlib.lbs.elasticsearch.Event;
import eu.dnetlib.lbs.elasticsearch.Notification;
import eu.dnetlib.lbs.elasticsearch.NotificationRepository;
import eu.dnetlib.lbs.events.output.DispatcherManager;
import eu.dnetlib.lbs.properties.ElasticSearchProperties;
import eu.dnetlib.lbs.subscriptions.MapCondition;
import eu.dnetlib.lbs.subscriptions.Subscription;
import eu.dnetlib.lbs.subscriptions.SubscriptionRepository;
import eu.dnetlib.lbs.utils.LbsQueue;
import eu.dnetlib.lbs.utils.QueueManager;
import eu.dnetlib.lbs.utils.ThreadManager;
import eu.dnetlib.broker.common.elasticsearch.Event;
import eu.dnetlib.broker.common.elasticsearch.Notification;
import eu.dnetlib.broker.common.elasticsearch.NotificationRepository;
import eu.dnetlib.broker.common.properties.ElasticSearchProperties;
import eu.dnetlib.broker.common.subscriptions.MapCondition;
import eu.dnetlib.broker.common.subscriptions.Subscription;
import eu.dnetlib.broker.common.subscriptions.SubscriptionRepository;
import eu.dnetlib.broker.events.output.DispatcherManager;
import eu.dnetlib.broker.utils.LbsQueue;
import eu.dnetlib.broker.utils.QueueManager;
import eu.dnetlib.broker.utils.ThreadManager;
@Profile("!openaire")
@Component
public class SubscriptionEventMatcher implements Runnable {
@ -99,25 +102,23 @@ public class SubscriptionEventMatcher implements Runnable {
final BoolQueryBuilder mapQuery = QueryBuilders.boolQuery();
s.getConditionsAsList()
.stream()
.map(MapCondition::asQueryBuilder)
.filter(Objects::nonNull)
.forEach(mapQuery::must);
.stream()
.map(MapCondition::asQueryBuilder)
.filter(Objects::nonNull)
.forEach(mapQuery::must);
final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
.withQuery(QueryBuilders.boolQuery()
.must(QueryBuilders.matchQuery("topic", s.getTopic()))
.must(QueryBuilders.rangeQuery("creationDate").from(s.getLastNotificationDate().getTime()))
.must(QueryBuilders.nestedQuery("map", mapQuery, ScoreMode.None)))
.withSearchType(SearchType.DEFAULT)
// .withIndices(elasticSearchProperties.getEventsIndexName())
// .withTypes(elasticSearchProperties.getEventsIndexType())
.withPageable(PageRequest.of(0, 10))
.build();
.withQuery(QueryBuilders.boolQuery()
.must(QueryBuilders.matchQuery("topic", s.getTopic()).operator(Operator.AND))
.must(QueryBuilders.rangeQuery("creationDate").from(s.getLastNotificationDate() != null ? s.getLastNotificationDate().getTime() : 0))
.must(QueryBuilders.nestedQuery("map", mapQuery, ScoreMode.None)))
.withSearchType(SearchType.DEFAULT)
.withPageable(PageRequest.of(0, 10))
.build();
final List<Event> events = new ArrayList<>();
final CloseableIterator<SearchHit<Event>> it =
esOperations.searchForStream(searchQuery, Event.class, IndexCoordinates.of(elasticSearchProperties.getEventsIndexName()));
esOperations.searchForStream(searchQuery, Event.class, IndexCoordinates.of(elasticSearchProperties.getEventsIndexName()));
while (it.hasNext()) {
final Event e = it.next().getContent();
final Notification n = new Notification(s, e);

2
dhp-broker-application/src/main/java/eu/dnetlib/lbs/metrics/MetricInterceptor.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/metrics/MetricInterceptor.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs.metrics;
package eu.dnetlib.broker.metrics;
public interface MetricInterceptor {

2
dhp-broker-application/src/main/java/eu/dnetlib/lbs/metrics/RequestCounterInterceptor.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/metrics/RequestCounterInterceptor.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs.metrics;
package eu.dnetlib.broker.metrics;
import java.lang.reflect.Method;
import javax.servlet.http.HttpServletRequest;

2
dhp-broker-application/src/main/java/eu/dnetlib/lbs/metrics/RequestTimingInterceptor.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/metrics/RequestTimingInterceptor.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs.metrics;
package eu.dnetlib.broker.metrics;
import java.lang.reflect.Method;
import javax.servlet.http.HttpServletRequest;

2
dhp-broker-application/src/main/java/eu/dnetlib/lbs/metrics/WebMvcConfig.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/metrics/WebMvcConfig.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs.metrics;
package eu.dnetlib.broker.metrics;
import java.util.List;

40
dhp-broker-application/src/main/java/eu/dnetlib/lbs/openaire/AdvQueryObject.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/openaire/AdvQueryObject.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs.openaire;
package eu.dnetlib.broker.openaire;
import java.util.ArrayList;
import java.util.Arrays;
@ -7,10 +7,10 @@ import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import eu.dnetlib.lbs.events.input.MapValueType;
import eu.dnetlib.lbs.subscriptions.ConditionOperator;
import eu.dnetlib.lbs.subscriptions.ConditionParams;
import eu.dnetlib.lbs.subscriptions.MapCondition;
import eu.dnetlib.broker.common.subscriptions.ConditionOperator;
import eu.dnetlib.broker.common.subscriptions.ConditionParams;
import eu.dnetlib.broker.common.subscriptions.MapCondition;
import eu.dnetlib.broker.common.utils.MapValueType;
public class AdvQueryObject {
@ -25,7 +25,7 @@ public class AdvQueryObject {
public AdvQueryObject() {}
public AdvQueryObject(final String datasource, final String topic, final List<String> titles, final List<String> subjects, final List<String> authors,
final List<Range> dates, final Range trust) {
final List<Range> dates, final Range trust) {
super();
this.datasource = datasource;
this.topic = topic;
@ -96,32 +96,32 @@ public class AdvQueryObject {
final List<MapCondition> list = new ArrayList<>();
final List<ConditionParams> list_dates = this.dates.stream()
.filter(Range::isValid)
.map(d -> new ConditionParams(d.getMin(), d.getMax()))
.collect(Collectors.toList());
.filter(Range::isValid)
.map(d -> new ConditionParams(d.getMin(), d.getMax()))
.collect(Collectors.toList());
final List<ConditionParams> list_titles = this.titles.stream()
.filter(StringUtils::isNotBlank)
.map(t -> new ConditionParams(t, null))
.collect(Collectors.toList());
.filter(StringUtils::isNotBlank)
.map(t -> new ConditionParams(t, null))
.collect(Collectors.toList());
final List<ConditionParams> list_authors = this.authors.stream()
.filter(StringUtils::isNotBlank)
.map(aut -> new ConditionParams(aut, null))
.collect(Collectors.toList());
.filter(StringUtils::isNotBlank)
.map(aut -> new ConditionParams(aut, null))
.collect(Collectors.toList());
final List<ConditionParams> list_subjects = this.subjects.stream()
.filter(StringUtils::isNotBlank)
.map(s -> new ConditionParams(s, null))
.collect(Collectors.toList());
.filter(StringUtils::isNotBlank)
.map(s -> new ConditionParams(s, null))
.collect(Collectors.toList());
if (StringUtils.isNotBlank(this.datasource)) {
list.add(new MapCondition("targetDatasourceName", MapValueType.STRING, ConditionOperator.EXACT,
Arrays.asList(new ConditionParams(this.datasource, null))));
Arrays.asList(new ConditionParams(this.datasource, null))));
}
if (this.trust.isValid()) {
list.add(new MapCondition("trust", MapValueType.FLOAT, ConditionOperator.RANGE,
Arrays.asList(new ConditionParams(this.trust.getMin(), this.trust.getMax()))));
Arrays.asList(new ConditionParams(this.trust.getMin(), this.trust.getMax()))));
}
if (!list_dates.isEmpty()) {
list.add(new MapCondition("targetDateofacceptance", MapValueType.DATE, ConditionOperator.RANGE, list_dates));

2
dhp-broker-application/src/main/java/eu/dnetlib/lbs/openaire/BrowseEntry.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/openaire/BrowseEntry.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs.openaire;
package eu.dnetlib.broker.openaire;
public class BrowseEntry implements Comparable<BrowseEntry> {

4
dhp-broker-application/src/main/java/eu/dnetlib/lbs/openaire/ElasticSearchQueryUtils.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/openaire/ElasticSearchQueryUtils.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs.openaire;
package eu.dnetlib.broker.openaire;
import java.util.Date;
import java.util.List;
@ -10,7 +10,7 @@ import org.elasticsearch.index.query.Operator;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.search.MatchQuery.ZeroTermsQuery;
import eu.dnetlib.lbs.utils.DateParser;
import eu.dnetlib.broker.common.utils.DateParser;
public class ElasticSearchQueryUtils {

2
dhp-broker-application/src/main/java/eu/dnetlib/lbs/openaire/EventsPage.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/openaire/EventsPage.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs.openaire;
package eu.dnetlib.broker.openaire;
import java.util.List;

2
dhp-broker-application/src/main/java/eu/dnetlib/lbs/openaire/HomeController.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/openaire/HomeController.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs.openaire;
package eu.dnetlib.broker.openaire;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;

199
dhp-broker-application/src/main/java/eu/dnetlib/lbs/openaire/OpenaireBrokerController.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/openaire/OpenaireBrokerController.java
View File

@ -1,17 +1,25 @@
package eu.dnetlib.lbs.openaire;
package eu.dnetlib.broker.openaire;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.lucene.search.join.ScoreMode;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.NestedQueryBuilder;
import org.elasticsearch.index.query.Operator;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
@ -19,16 +27,16 @@ import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.nested.ParsedNested;
import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Profile;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.elasticsearch.core.AbstractElasticsearchTemplate;
import org.springframework.data.elasticsearch.core.ElasticSearchScrollUtils;
import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
import org.springframework.data.elasticsearch.core.SearchHit;
import org.springframework.data.elasticsearch.core.SearchHits;
import org.springframework.data.elasticsearch.core.SearchScrollHits;
import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
import org.springframework.data.elasticsearch.core.query.NativeSearchQuery;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.RowMapper;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
@ -37,21 +45,21 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import com.google.gson.Gson;
import eu.dnetlib.broker.LiteratureBrokerServiceConfiguration;
import eu.dnetlib.broker.common.controllers.AbstractLbsController;
import eu.dnetlib.broker.common.elasticsearch.Event;
import eu.dnetlib.broker.common.elasticsearch.Notification;
import eu.dnetlib.broker.common.elasticsearch.NotificationRepository;
import eu.dnetlib.broker.common.properties.ElasticSearchProperties;
import eu.dnetlib.broker.common.subscriptions.MapCondition;
import eu.dnetlib.broker.common.subscriptions.Subscription;
import eu.dnetlib.broker.common.subscriptions.SubscriptionRepository;
import eu.dnetlib.broker.events.output.DispatcherManager;
import eu.dnetlib.broker.objects.OaBrokerEventPayload;
import eu.dnetlib.lbs.LiteratureBrokerServiceConfiguration;
import eu.dnetlib.lbs.controllers.AbstractLbsController;
import eu.dnetlib.lbs.elasticsearch.Event;
import eu.dnetlib.lbs.elasticsearch.Notification;
import eu.dnetlib.lbs.elasticsearch.NotificationRepository;
import eu.dnetlib.lbs.properties.ElasticSearchProperties;
import eu.dnetlib.lbs.subscriptions.MapCondition;
import eu.dnetlib.lbs.subscriptions.Subscription;
import eu.dnetlib.lbs.subscriptions.SubscriptionRepository;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
@Profile("openaire")
@RestController
@RequestMapping("/api/openaireBroker")
@Api(tags = LiteratureBrokerServiceConfiguration.TAG_OPENAIRE)
@ -69,12 +77,21 @@ public class OpenaireBrokerController extends AbstractLbsController {
@Autowired
private ElasticSearchProperties props;
@Autowired
private DispatcherManager dispatcher;
@Autowired
private JdbcTemplate jdbcTemplate;
private static final Log log = LogFactory.getLog(OpenaireBrokerController.class);
@ApiOperation("Return the datasources having events")
@GetMapping("/datasources")
public List<BrowseEntry> findDatasourcesWithEvents() {
public List<BrowseEntry> findDatasourcesWithEvents(@RequestParam(defaultValue = "false", required = false) final boolean useIndex) {
return useIndex ? findDatasourcesWithEventsUsingIndex() : findDatasourcesWithEventsUsingDb();
}
private List<BrowseEntry> findDatasourcesWithEventsUsingIndex() {
final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
.withQuery(QueryBuilders.matchAllQuery())
.withSearchType(SearchType.DEFAULT)
@ -95,34 +112,69 @@ public class OpenaireBrokerController extends AbstractLbsController {
.collect(Collectors.toList());
}
private List<BrowseEntry> findDatasourcesWithEventsUsingDb() {
try {
final String sql = IOUtils.toString(getClass().getResourceAsStream("/sql/datasourceTopics.sql"));
final RowMapper<BrowseEntry> mapper = (rs, rowNum) -> new BrowseEntry(rs.getString("name"), rs.getLong("size"));
return jdbcTemplate.query(sql, mapper);
} catch (final Exception e) {
log.error("Error executing query", e);
return new ArrayList<>();
}
}
@ApiOperation("Return the topics of the events of a datasource")
@GetMapping("/topicsForDatasource")
public List<BrowseEntry> findTopicsForDatasource(@RequestParam final String ds) {
public List<BrowseEntry> findTopicsForDatasource(@RequestParam final String ds,
@RequestParam(defaultValue = "false", required = false) final boolean useIndex) {
return useIndex ? findTopicsForDatasourceUsingIndex(ds) : findTopicsForDatasourceUsingDb(ds);
}
private List<BrowseEntry> findTopicsForDatasourceUsingIndex(final String ds) {
final String term = "topic.keyword";
final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
.withQuery(QueryBuilders.nestedQuery("map", QueryBuilders.matchQuery("map.targetDatasourceName", ds), ScoreMode.None))
.withSearchType(SearchType.DEFAULT)
.addAggregation(AggregationBuilders.terms("topic").field("topic").size(1000).minDocCount(1))
.addAggregation(AggregationBuilders.terms(term).field(term).size(1000).minDocCount(1))
.build();
final SearchHits<Event> hits = esOperations.search(searchQuery, Event.class, IndexCoordinates.of(props.getEventsIndexName()));
final Aggregations aggregations = hits.getAggregations();
return ((ParsedStringTerms) aggregations.asMap().get("topic")).getBuckets()
return ((ParsedStringTerms) aggregations.asMap().get(term)).getBuckets()
.stream()
.map(b -> new BrowseEntry(b.getKeyAsString(), b.getDocCount()))
.collect(Collectors.toList());
}
private List<BrowseEntry> findTopicsForDatasourceUsingDb(final String ds) {
try {
final String sql = IOUtils.toString(getClass().getResourceAsStream("/sql/datasourceTopicsDetailed.sql"));
final RowMapper<BrowseEntry> mapper = (rs, rowNum) -> new BrowseEntry(rs.getString("topic"), rs.getLong("size"));
return jdbcTemplate.query(sql, new Object[] {
ds
}, mapper);
} catch (final Exception e) {
log.error("Error executing query", e);
return new ArrayList<>();
}
}
@ApiOperation("Return a page of events of a datasource (by topic)")
@GetMapping("/events/{nPage}/{size}")
public EventsPage showEvents(@RequestParam final String ds, @RequestParam final String topic, @PathVariable final int nPage, @PathVariable final int size) {
final NestedQueryBuilder nestedQuery = QueryBuilders.nestedQuery("map", QueryBuilders.matchQuery("map.targetDatasourceName", ds), ScoreMode.None);
final QueryBuilder q = StringUtils.isNotBlank(topic) && !topic.equals("*") ? QueryBuilders.boolQuery()
.must(QueryBuilders.matchQuery("topic", topic).operator(Operator.AND))
.must(nestedQuery) : nestedQuery;
final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
.withQuery(QueryBuilders.boolQuery()
.must(QueryBuilders.matchQuery("topic", topic))
.must(QueryBuilders.nestedQuery("map", QueryBuilders.matchQuery("map.targetDatasourceName", ds), ScoreMode.None)))
.withQuery(q)
.withSearchType(SearchType.DEFAULT)
.withFields("payload")
.withPageable(PageRequest.of(nPage, size))
@ -151,10 +203,14 @@ public class OpenaireBrokerController extends AbstractLbsController {
ElasticSearchQueryUtils.addMapConditionForTrust(mapQuery, "map.trust", qObj.getTrust());
ElasticSearchQueryUtils.addMapConditionForDates(mapQuery, "map.targetDateofacceptance", qObj.getDates());
final NestedQueryBuilder nestedQuery = QueryBuilders.nestedQuery("map", mapQuery, ScoreMode.None);
final QueryBuilder q = StringUtils.isNotBlank(qObj.getTopic()) && !qObj.getTopic().equals("*") ? QueryBuilders.boolQuery()
.must(QueryBuilders.matchQuery("topic", qObj.getTopic()).operator(Operator.AND))
.must(nestedQuery) : nestedQuery;
final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
.withQuery(QueryBuilders.boolQuery()
.must(QueryBuilders.matchQuery("topic", qObj.getTopic()))
.must(QueryBuilders.nestedQuery("map", mapQuery, ScoreMode.None)))
.withQuery(q)
.withSearchType(SearchType.DEFAULT)
.withFields("payload")
.withPageable(PageRequest.of(nPage, size))
@ -181,12 +237,13 @@ public class OpenaireBrokerController extends AbstractLbsController {
return sub;
}
@ApiOperation("Return the subscriptions of an user (by email)")
@ApiOperation("Return the subscriptions of an user (by email and datasource (optional))")
@GetMapping("/subscriptions")
public Map<String, List<SimpleSubscriptionDesc>> subscriptions(@RequestParam final String email) {
public Map<String, List<SimpleSubscriptionDesc>> subscriptions(@RequestParam final String email, @RequestParam(required = false) final String ds) {
final Iterable<Subscription> iter = subscriptionRepo.findBySubscriber(email);
return StreamSupport.stream(iter.spliterator(), false)
.map(this::subscriptionDesc)
.filter(s -> StringUtils.isBlank(ds) || StringUtils.equalsIgnoreCase(s.getDatasource(), ds))
.collect(Collectors.groupingBy(SimpleSubscriptionDesc::getDatasource));
}
@ -200,7 +257,7 @@ public class OpenaireBrokerController extends AbstractLbsController {
final Subscription sub = optSub.get();
final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
.withQuery(QueryBuilders.matchQuery("subscriptionId", subscrId))
.withQuery(QueryBuilders.termQuery("subscriptionId.keyword", subscrId))
.withSearchType(SearchType.DEFAULT)
.withFields("payload")
.withPageable(PageRequest.of(nPage, size))
@ -222,71 +279,39 @@ public class OpenaireBrokerController extends AbstractLbsController {
}
@ApiOperation("Returns notifications using scrolls (first page)")
@GetMapping("/scroll/notifications/start/{subscrId}")
public ScrollPage prepareScrollNotifications(@PathVariable final String subscrId) {
@ApiOperation("Send notifications")
@GetMapping("/notifications/send/{date}")
private List<String> sendMailForNotifications(@PathVariable final long date) {
new Thread(() -> innerSendMailForNotifications(date)).start();
return Arrays.asList("Sending ...");
}
final Optional<Subscription> optSub = subscriptionRepo.findById(subscrId);
@ApiOperation("Update stats")
@GetMapping("/stats/update")
private List<String> updateStats() {
new Thread(() -> {
try {
jdbcTemplate.update(IOUtils.toString(getClass().getResourceAsStream("/sql/updateStats.sql")));
} catch (final Exception e) {
log.error("Error updating stats", e);
}
}).start();
return Arrays.asList("Sending ...");
}
if (optSub.isPresent()) {
private void innerSendMailForNotifications(final long date) {
for (final Subscription s : subscriptionRepo.findAll()) {
final AbstractElasticsearchTemplate esTemplate = (AbstractElasticsearchTemplate) esOperations;
final NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
.withQuery(QueryBuilders.matchQuery("subscriptionId", subscrId))
.withSearchType(SearchType.DEFAULT)
.withFields("payload")
.build();
final SearchScrollHits<Notification> scroll =
ElasticSearchScrollUtils.startScroll(esTemplate, IndexCoordinates.of(props.getNotificationsIndexName()), searchQuery, Notification.class);
if (scroll.hasSearchHits()) {
final List<OaBrokerEventPayload> values = calculateEventPayloads(scroll);
return new ScrollPage(scroll.getScrollId(), values.isEmpty() || scroll.getScrollId() == null, values);
} else {
ElasticSearchScrollUtils.clearScroll(esTemplate, scroll.getScrollId());
return new ScrollPage(null, true, new ArrayList<>());
final long count = notificationRepository.countBySubscriptionIdAndDateAfter(s.getSubscriptionId(), date);
if (count > 0) {
final Map<String, Object> params = new HashMap<>();
params.put("oa_notifications_total", count);
dispatcher.sendNotification(s, params);
}
} else {
log.warn("Invalid subscription: " + subscrId);
return new ScrollPage();
s.setLastNotificationDate(new Date());
subscriptionRepo.save(s);
}
}
@ApiOperation("Returns notifications using scrolls (other pages)")
@GetMapping("/scroll/notifications/{scrollId}")
public ScrollPage scrollNotifications(@PathVariable final String scrollId) {
final AbstractElasticsearchTemplate esTemplate = (AbstractElasticsearchTemplate) esOperations;
final SearchScrollHits<Notification> scroll =
ElasticSearchScrollUtils.continueScroll(esTemplate, IndexCoordinates.of(props.getNotificationsIndexName()), scrollId, Notification.class);
if (scroll.hasSearchHits()) {
final List<OaBrokerEventPayload> values = calculateEventPayloads(scroll);
return new ScrollPage(scroll.getScrollId(), values.isEmpty() || scroll.getScrollId() == null, values);
} else {
ElasticSearchScrollUtils.clearScroll(esTemplate, scroll.getScrollId());
return new ScrollPage(null, true, new ArrayList<>());
}
}
private List<OaBrokerEventPayload> calculateEventPayloads(final SearchScrollHits<Notification> scroll) {
if (scroll.getSearchHits().size() > 0) {
final Gson gson = new Gson();
return scroll.stream()
.map(SearchHit::getContent)
.map(Notification::getPayload)
.map(s -> gson.fromJson(s, OaBrokerEventPayload.class))
.collect(Collectors.toList());
} else {
return new ArrayList<>();
}
}
private SimpleSubscriptionDesc subscriptionDesc(final Subscription s) {

12
dhp-broker-application/src/main/java/eu/dnetlib/lbs/openaire/OpenaireSubscription.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/openaire/OpenaireSubscription.java
View File

@ -1,13 +1,13 @@
package eu.dnetlib.lbs.openaire;
package eu.dnetlib.broker.openaire;
import java.util.Date;
import java.util.UUID;
import org.apache.commons.lang3.StringUtils;
import eu.dnetlib.lbs.subscriptions.NotificationFrequency;
import eu.dnetlib.lbs.subscriptions.NotificationMode;
import eu.dnetlib.lbs.subscriptions.Subscription;
import eu.dnetlib.broker.common.subscriptions.NotificationFrequency;
import eu.dnetlib.broker.common.subscriptions.NotificationMode;
import eu.dnetlib.broker.common.subscriptions.Subscription;
public class OpenaireSubscription {
@ -19,7 +19,7 @@ public class OpenaireSubscription {
public OpenaireSubscription() {}
public OpenaireSubscription(final String subscriber, final NotificationFrequency frequency, final NotificationMode mode,
final AdvQueryObject query) {
final AdvQueryObject query) {
this.subscriber = subscriber;
this.frequency = frequency;
this.mode = mode;
@ -65,7 +65,7 @@ public class OpenaireSubscription {
final String subscriptionId = "sub-" + UUID.randomUUID();
return new Subscription(subscriptionId, this.subscriber, this.query.getTopic(), this.frequency, this.mode, null, new Date(),
this.query.asListOfConditions());
this.query.asListOfConditions());
}
}

2
dhp-broker-application/src/main/java/eu/dnetlib/lbs/openaire/Range.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/openaire/Range.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs.openaire;
package eu.dnetlib.broker.openaire;
import org.apache.commons.lang3.StringUtils;

2
dhp-broker-application/src/main/java/eu/dnetlib/lbs/openaire/ScrollPage.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/openaire/ScrollPage.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs.openaire;
package eu.dnetlib.broker.openaire;
import java.util.List;

2
dhp-broker-application/src/main/java/eu/dnetlib/lbs/openaire/SimpleSubscriptionDesc.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/openaire/SimpleSubscriptionDesc.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs.openaire;
package eu.dnetlib.broker.openaire;
import java.util.Date;

2
dhp-broker-application/src/main/java/eu/dnetlib/lbs/properties/EmailProperties.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/properties/EmailProperties.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs.properties;
package eu.dnetlib.broker.properties;
import javax.validation.constraints.NotNull;

14
dhp-broker-application/src/main/java/eu/dnetlib/lbs/utils/EventVerifier.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/utils/EventVerifier.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs.utils;
package eu.dnetlib.broker.utils;
import java.time.Duration;
import java.time.Instant;
@ -12,14 +12,14 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import eu.dnetlib.lbs.elasticsearch.Event;
import eu.dnetlib.lbs.topics.TopicType;
import eu.dnetlib.lbs.topics.TopicTypeRepository;
import eu.dnetlib.broker.common.elasticsearch.Event;
import eu.dnetlib.broker.common.topics.TopicType;
import eu.dnetlib.broker.common.topics.TopicTypeRepository;
public class EventVerifier implements Predicate<Event> {
private final TopicTypeRepository topicTypeRepo;
private final long MAX_DELAY = 5000; // 10 seconds
private final long MAX_DELAY = 30000; // 30 seconds
private Instant last = Instant.EPOCH;
private List<Predicate<Event>> validators = new ArrayList<>();
@ -40,8 +40,8 @@ public class EventVerifier implements Predicate<Event> {
final Iterable<TopicType> iter = this.topicTypeRepo.findAll();
this.validators = StreamSupport.stream(iter.spliterator(), false)
.map(TopicType::asValidator)
.collect(Collectors.toList());
.map(TopicType::asValidator)
.collect(Collectors.toList());
}
try {

2
dhp-broker-application/src/main/java/eu/dnetlib/lbs/utils/LbsQueue.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/utils/LbsQueue.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs.utils;
package eu.dnetlib.broker.utils;
import java.util.ArrayList;
import java.util.List;

2
dhp-broker-application/src/main/java/eu/dnetlib/lbs/utils/QueueManager.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/utils/QueueManager.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs.utils;
package eu.dnetlib.broker.utils;
import java.util.ArrayList;
import java.util.List;

2
dhp-broker-application/src/main/java/eu/dnetlib/lbs/utils/ThreadManager.java → apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/utils/ThreadManager.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs.utils;
package eu.dnetlib.broker.utils;
import java.util.ArrayList;
import java.util.List;

54
apps/dhp-broker-application/src/main/resources/application.properties Normal file
View File

@ -0,0 +1,54 @@
spring.profiles.active = dev,openaire
#logging.level.root=DEBUG
spring.datasource.url=jdbc:postgresql://localhost:5432/dnet_broker
spring.datasource.username=
spring.datasource.password=
spring.jpa.properties.hibernate.dialect = org.hibernate.dialect.PostgreSQLDialect
# Hibernate ddl auto (create, create-drop, validate, update)
spring.jpa.hibernate.ddl-auto = validate
spring.jpa.properties.hibernate.hbm2dll.extra_physical_table_types = MATERIALIZED VIEW
spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation = true
spring.jpa.open-in-view = true
spring.jpa.properties.hibernate.show_sql = false
spring.jpa.properties.hibernate.use_sql_comments = false
spring.jpa.properties.hibernate.format_sql = false
lbs.database.url = ${spring.datasource.url}
# for development server
#lbs.elastic.clusterNodes = broker1-dev-dnet.d4science.org:9300
#lbs.elastic.homepage = http://broker1-dev-dnet.d4science.org:9200/_plugin/hq
# for localhost
lbs.elastic.clusterNodes = 127.0.0.1:9200
lbs.elastic.homepage = http://
lbs.elastic.socketTimeout = 30000
lbs.elastic.connectionTimeout = 30000
lbs.elastic.eventsIndexName = oa_events_beta
lbs.elastic.notificationsIndexName = oa_notifications_beta
lbs.mail.from = notifications-noreply@openaire.eu
lbs.mail.fromName = Literature Broker Service
lbs.mail.cc = michele.artini@isti.cnr.it
lbs.mail.smtpHost = smtp.isti.cnr.it
lbs.mail.smtpPort = 587
lbs.mail.smtpUser = smtp-dnet
lbs.mail.smtpPassword = hhr*7932
lbs.mail.message.template = classpath:/templates/dhp_openaire_mail.st
lbs.queues.maxReturnedValues = 1000
lbs.task.deleteOldEvents.cron = 0 0 4 1/1 * ?
lbs.task.deleteOldNotifications.cron = 0 0 5 1/1 * ?
lbs.task.subscriptionEventsMatcher.cron = 0 0 6 1/1 * ?
spring.data.elasticsearch.cluster-name = ${lbs.elastic.clusterName}
spring.data.elasticsearch.cluster-nodes = ${lbs.elastic.clusterNodes}

110
apps/dhp-broker-application/src/main/resources/config/mapping_es7.sh Executable file
View File

@ -0,0 +1,110 @@
#!/bin/bash
INDEXBASEURL=http://localhost:9200
EVENTINDEXNAME=oa_events_beta
NOTIFICATIONINDEXNAME=oa_notifications_beta
echo
echo "Update settings of events index..."
curl -XPOST "$INDEXBASEURL/$EVENTINDEXNAME/_close"
echo
curl -XPUT -H 'Content-Type: application/json' "$INDEXBASEURL/$EVENTINDEXNAME/_settings" -d '
{
"index": {
"translog": {
"sync_interval": "15s",
"durability": "ASYNC"
},
"analysis": {
"analyzer": {
"analyzer_keyword": {
"filter": "lowercase",
"tokenizer": "keyword"
}
}
}
}
}'
echo
curl -XPOST "$INDEXBASEURL/$EVENTINDEXNAME/_open"
echo
echo
echo "Update settings of notifications index..."
curl -XPOST "$INDEXBASEURL/$NOTIFICATIONINDEXNAME/_close"
echo
curl -XPUT -H 'Content-Type: application/json' "$INDEXBASEURL/$NOTIFICATIONINDEXNAME/_settings" -d '
{
"index": {
"translog": {
"sync_interval": "15s",
"durability": "ASYNC"
},
"analysis": {
"analyzer": {
"analyzer_keyword": {
"filter": "lowercase",
"tokenizer": "keyword"
}
}
}
}
}'
echo
curl -XPOST "$INDEXBASEURL/$NOTIFICATIONINDEXNAME/_open"
echo
echo
echo "Fixing events index..."
curl -XPUT -H 'Content-Type: application/json' "$INDEXBASEURL/$EVENTINDEXNAME/_mapping" -d '
{
"properties" : {
"map": {
"type": "nested",
"properties": {
"targetDatasourceId": {
"type": "keyword",
"index": true
},
"targetDatasourceName": {
"type": "keyword",
"index": true
},
"targetDatasourceType": {
"type": "keyword",
"index": true
}
}
}
}
}'
echo
echo
echo "Fixing notifications index..."
curl -XPUT -H 'Content-Type: application/json' "$INDEXBASEURL/$NOTIFICATIONINDEXNAME/_mapping" -d '
{
"properties" : {
"map": {
"type": "nested",
"properties": {
"targetDatasourceId": {
"type": "keyword",
"index": true
},
"targetDatasourceName": {
"type": "keyword",
"index": true
},
"targetDatasourceType": {
"type": "keyword",
"index": true
}
}
}
}
}'
echo
echo

34
apps/dhp-broker-application/src/main/resources/config/schema.sql Normal file
View File

@ -0,0 +1,34 @@
CREATE TABLE subscriptions (
subscriptionid text PRIMARY KEY,
conditions text,
creation_date timestamp without time zone DEFAULT '2018-01-01 00:00:00'::timestamp without time zone NOT NULL,
frequency text NOT NULL,
last_notification_date timestamp without time zone,
mode text NOT NULL,
subscriber text NOT NULL,
topic text NOT NULL
);
CREATE TABLE topic_types (
id text PRIMARY KEY,
name text UNIQUE NOT NULL,
expression text UNIQUE NOT NULL,
map_keys text,
producer_id text,
regex text UNIQUE NOT NULL
);
CREATE TABLE oa_datasource_stats (
id text NOT NULL,
name text NOT NULL,
type text NOT NULL,
topic text NOT NULL,
size bigint NOT NULL,
PRIMARY KEY (id, topic)
);
INSERT INTO public.topic_types (id, name, expression, map_keys, producer_id, regex) VALUES ('tt-a739fa2b-fde0-4eb2-bcee-6e7f277347db', 'ENRICH', 'ENRICH/<cond>/<field>', 'targetDatasourceName', 'OpenAIRE', '^ENRICH\/[a-zA-Z0-9._-]+\/[a-zA-Z0-9._-]+$');
INSERT INTO public.topic_types (id, name, expression, map_keys, producer_id, regex) VALUES ('tt-93be0404-e7fb-43bb-9a0a-d317f418ed6d', 'ENRICH_WITH_SUBFIELD', 'ENRICH/<cond>/<field>/<subfield>', 'targetDatasourceName', 'OpenAIRE', '^ENRICH\/[a-zA-Z0-9._-]+\/[a-zA-Z0-9._-]+\/[a-zA-Z0-9._-]+$');
INSERT INTO public.topic_types (id, name, expression, map_keys, producer_id, regex) VALUES ('tt-80978da9-1859-47aa-9897-0a0c372365a1', 'ADD', 'ADD/<field>', 'targetDatasourceName', 'OpenAIRE', '^ADD\/[a-zA-Z0-9._-]+$');
INSERT INTO public.topic_types (id, name, expression, map_keys, producer_id, regex) VALUES ('tt-75e4b98f-17b8-4c2a-b3c6-d0ad7dfb4d51', '*', '*', 'targetDatasourceName', 'OpenAIRE', '^\*$');

1
apps/dhp-broker-application/src/main/resources/sql/datasourceTopics.sql Normal file
View File

@ -0,0 +1 @@
select (array_agg(name))[1] as name, sum(size) as size from oa_datasource_stats group by lower(name) order by sum(size) desc;

7
apps/dhp-broker-application/src/main/resources/sql/datasourceTopicsDetailed.sql Normal file
View File

@ -0,0 +1,7 @@
select
topic as topic,
sum(size) as size
from oa_datasource_stats
where lower(name) = lower(?)
group by topic
order by sum(size) desc;

5
apps/dhp-broker-application/src/main/resources/sql/updateStats.sql Normal file
View File

@ -0,0 +1,5 @@
begin;
delete from oa_datasource_stats;
insert into oa_datasource_stats (id, name, type, topic, size) select id, name, type, topic, size from oa_datasource_stats_temp;
drop table oa_datasource_stats_temp;
commit;

0
dhp-broker-application/src/main/resources/static/css/angular-backtop.css → apps/dhp-broker-application/src/main/resources/static/css/angular-backtop.css
View File

0
dhp-broker-application/src/main/resources/static/css/bootstrap-theme.min.css → apps/dhp-broker-application/src/main/resources/static/css/bootstrap-theme.min.css vendored
View File

0
dhp-broker-application/src/main/resources/static/css/bootstrap.min.css → apps/dhp-broker-application/src/main/resources/static/css/bootstrap.min.css vendored
View File

0
dhp-broker-application/src/main/resources/static/css/rzslider.min.css → apps/dhp-broker-application/src/main/resources/static/css/rzslider.min.css vendored
View File

0
dhp-broker-application/src/main/resources/static/fonts/glyphicons-halflings-regular.eot → apps/dhp-broker-application/src/main/resources/static/fonts/glyphicons-halflings-regular.eot
View File

0
dhp-broker-application/src/main/resources/static/fonts/glyphicons-halflings-regular.svg → apps/dhp-broker-application/src/main/resources/static/fonts/glyphicons-halflings-regular.svg
View File

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 106 KiB

After

Width:  |  Height:  |  Size: 106 KiB

0
dhp-broker-application/src/main/resources/static/fonts/glyphicons-halflings-regular.ttf → apps/dhp-broker-application/src/main/resources/static/fonts/glyphicons-halflings-regular.ttf
View File

0
dhp-broker-application/src/main/resources/static/fonts/glyphicons-halflings-regular.woff → apps/dhp-broker-application/src/main/resources/static/fonts/glyphicons-halflings-regular.woff
View File

0
dhp-broker-application/src/main/resources/static/fonts/glyphicons-halflings-regular.woff2 → apps/dhp-broker-application/src/main/resources/static/fonts/glyphicons-halflings-regular.woff2
View File

0
dhp-broker-application/src/main/resources/static/html/adminOperation.template.html → apps/dhp-broker-application/src/main/resources/static/html/adminOperation.template.html
View File

0
dhp-broker-application/src/main/resources/static/html/eventsByTopic.html → apps/dhp-broker-application/src/main/resources/static/html/eventsByTopic.html
View File

0
dhp-broker-application/src/main/resources/static/html/filterEvents.html → apps/dhp-broker-application/src/main/resources/static/html/filterEvents.html
View File

2
dhp-broker-application/src/main/resources/static/html/subscriptions.html → apps/dhp-broker-application/src/main/resources/static/html/subscriptions.html
View File

@ -1,4 +1,4 @@
<h3>Subscriptions (H2 database)</h3>
<h3>Subscriptions (Postgres database)</h3>
<form class="form-inline text-right" style="margin-top:40px; margin-bottom:20px">
<button class="btn btn-primary" data-target="#addSubscriptionModal" data-toggle="modal" ng-click="resetFormFields()">

36
dhp-broker-application/src/main/resources/static/html/summary.html → apps/dhp-broker-application/src/main/resources/static/html/summary.html
View File

@ -19,28 +19,24 @@
<th class="col-xs-6">n. subscriptions</th>
<td class="col-xs-6 text-right">
{{summary.totals.subscriptions}}
<two-step-button label="clear" action="deleteAllSubscriptions()" />
</td>
</tr>
<tr>
<th class="col-xs-6">n. events</th>
<td class="col-xs-6 text-right">
{{summary.totals.events}}
<two-step-button label="clear" action="deleteAllEvents()" />
</td>
</tr>
<tr>
<th class="col-xs-6">n. topic types</th>
<td class="col-xs-6 text-right">
{{summary.totals.topics}}
<two-step-button label="clear" action="deleteAllTopicTypes()" />
</td>
</tr>
<tr>
<th class="col-xs-6">n. notifications</th>
<td class="col-xs-6 text-right">
{{summary.totals.notifications}}
<two-step-button label="clear" action="deleteAllNotifications()" />
</td>
</tr>
</table>
@ -87,12 +83,14 @@
<tr>
<th></th>
<th class="text-right">Sent messages</th>
<th class="text-right">Errors</th>
</tr>
</thead>
<tbody>
<tr ng-repeat="b in summary.dispatchers">
<th>{{b.name}}</th>
<td class="text-right">{{b.count}}</td>
<td class="text-right" ng-class="{ 'text-danger' : b.countErrors > 0 }" title="{{b.lastError}}">{{b.countErrors}}</td>
</tr>
</tbody>
</table>
@ -133,13 +131,17 @@
</tr>
<tr>
<th class="col-xs-4" rowspan="2">indexes</th>
<td class="col-xs-4">{{summary.elasticSearch.eventsIndexName}}</td>
<th class="col-xs-1">type</th>
<td class="col-xs-3">{{summary.elasticSearch.eventsIndexType}}</td>
<td class="col-xs-8">{{summary.elasticSearch.eventsIndexName}}</td>
<tr>
<td class="col-xs-4">{{summary.elasticSearch.notificationsIndexName}}</td>
<th class="col-xs-1">type</th>
<td class="col-xs-3">{{summary.elasticSearch.notificationsIndexType}}</td>
<td class="col-xs-8">{{summary.elasticSearch.notificationsIndexName}}</td>
</tr>
<tr>
<th class="col-xs-4">socket timeout</th>
<td class="col-xs-8" colspan="3">{{summary.elasticSearch.socketTimeout}} ms</td>
</tr>
<tr>
<th class="col-xs-4">connection timeout</th>
<td class="col-xs-8" colspan="3">{{summary.elasticSearch.connectionTimeout}} ms</td>
</tr>
</table>
</div>
@ -149,14 +151,11 @@
<div class="row">
<div class="col-xs-12 col-md-6">
<div class="panel panel-default">
<div class="panel-heading">H2 database</div>
<div class="panel-heading">Postgres database</div>
<table class="table">
<tr>
<th class="col-xs-4">path</th>
<td class="col-xs-8">{{summary.h2.url}}</td>
</tr><tr>
<th class="col-xs-4">driver</th>
<td class="col-xs-8">{{summary.h2.driverName}}</td>
<td class="col-xs-8">{{summary.db.url}}</td>
</tr>
</table>
</div>
@ -212,13 +211,6 @@
</div>
<div class="modal-body">
<table class="table table-bordered">
<tr admin-operation label="delete expired events"
description="Delete the expired events"
alert="It can't be UNDONE."
action="deleteExpiredEvents()"></tr>
<tr admin-operation label="subscription/events matcher"
description="Start the subscription/events matcher"
action="startSubscriptionEventsMatcher()"></admin-operation>
<tr admin-operation label="reset counters"
description="Reset the counters of the internal queues."
action="resetCounters()"></tr>

0
dhp-broker-application/src/main/resources/static/html/topics.html → apps/dhp-broker-application/src/main/resources/static/html/topics.html
View File

0
dhp-broker-application/src/main/resources/static/html/twoStepButton.template.html → apps/dhp-broker-application/src/main/resources/static/html/twoStepButton.template.html
View File

0
dhp-broker-application/src/main/resources/static/index.html → apps/dhp-broker-application/src/main/resources/static/index.html
View File

0
dhp-broker-application/src/main/resources/static/js/angular-backtop.min.js → apps/dhp-broker-application/src/main/resources/static/js/angular-backtop.min.js vendored
View File

0
dhp-broker-application/src/main/resources/static/js/angular-base64.min.js → apps/dhp-broker-application/src/main/resources/static/js/angular-base64.min.js vendored
View File

0
dhp-broker-application/src/main/resources/static/js/angular-local-storage.min.js → apps/dhp-broker-application/src/main/resources/static/js/angular-local-storage.min.js vendored
View File

0
dhp-broker-application/src/main/resources/static/js/angular-route.min.js → apps/dhp-broker-application/src/main/resources/static/js/angular-route.min.js vendored
View File

0
dhp-broker-application/src/main/resources/static/js/angular.min.js → apps/dhp-broker-application/src/main/resources/static/js/angular.min.js vendored
View File

0
dhp-broker-application/src/main/resources/static/js/bootstrap.min.js → apps/dhp-broker-application/src/main/resources/static/js/bootstrap.min.js vendored
View File

0
dhp-broker-application/src/main/resources/static/js/jquery-1.12.3.min.js → apps/dhp-broker-application/src/main/resources/static/js/jquery-1.12.3.min.js vendored
View File

0
dhp-broker-application/src/main/resources/static/js/jquery-1.12.3.min.map → apps/dhp-broker-application/src/main/resources/static/js/jquery-1.12.3.min.map
View File

52
dhp-broker-application/src/main/resources/static/js/lbs-ui.js → apps/dhp-broker-application/src/main/resources/static/js/lbs-ui.js
View File

@ -112,54 +112,6 @@ lbsControllers.controller('lbsSummaryCtrl', function ($scope, $http, $sce, $loca
});
};
$scope.deleteAllSubscriptions = function() {
$http.delete('api/subscriptions').success(function(data) {
$scope.summary.totals.subscriptions = 0;
}).error(function() {
alert("error");
});
};
$scope.deleteAllEvents = function() {
$http.delete('api/events/all').success(function(data) {
$scope.summary.totals.events = 0;
}).error(function() {
alert("error");
});
};
$scope.deleteExpiredEvents = function() {
$http.delete('api/events/expired').success(function(data) {
$scope.summary.totals.events = $scope.summary.totals.events - data.deleted;
}).error(function() {
alert("error");
});
};
$scope.deleteAllTopicTypes = function() {
$http.delete('api/topic-types').success(function(data) {
$scope.summary.totals.topics = 0;
}).error(function() {
alert("error");
});
};
$scope.deleteAllNotifications = function() {
$http.delete('api/notifications').success(function(data) {
$scope.summary.totals.notifications = 0;
}).error(function() {
alert("error");
});
};
$scope. startSubscriptionEventsMatcher = function() {
$http.get('api/subscriptions/startMatching').success(function(data) {
alert("matching is running");
}).error(function() {
alert("error");
});
};
$scope.refresh();
});
@ -262,10 +214,10 @@ lbsControllers.controller('lbsSubscriptionsCtrl', function ($scope, $http, $sce,
};
$scope.startMatching = function(id) {
$http.get('api/subscriptions/startMatching/' + id).success(function(data) {
$http.get('api/matching/start/' + id).success(function(data) {
alert("Matching in queue");
}).error(function() {
alert("error");
alert("Error: The functionality could be disabled in current configuration profile !");
});
};

0
dhp-broker-application/src/main/resources/static/js/rzslider.min.js → apps/dhp-broker-application/src/main/resources/static/js/rzslider.min.js vendored
View File

2
dhp-broker-application/src/main/resources/static/openaire/datasources.html → apps/dhp-broker-application/src/main/resources/static/openaire/datasources.html
View File

@ -38,7 +38,7 @@
<td class="text-right">{{t.size}}</td>
</tr>
<tr>
<th>Total</th>
<th><a href="javascript:void(0)" ng-click="gotoEventsPage(dsName, '*')" data-dismiss="modal">Total</a></th>
<th class="text-right">{{calculateTotal(topics)}}</th>
</tr>
</tbody>

0
dhp-broker-application/src/main/resources/static/openaire/index.html → apps/dhp-broker-application/src/main/resources/static/openaire/index.html
View File

0
dhp-broker-application/src/main/resources/static/openaire/login.html → apps/dhp-broker-application/src/main/resources/static/openaire/login.html
View File

253
apps/dhp-broker-application/src/main/resources/static/openaire/notifications.html Normal file
View File

@ -0,0 +1,253 @@
<div class="col-xs-12" ng-show="page">
<p>
<b>Datasource:</b> {{page.datasource}}<br />
<b>Topic:</b> {{page.topic}}<br />
<b>Total:</b> {{page.total}}<br />
<b>Page:</b> {{page.currPage + 1}} / {{page.totalPages}}<br />
</p>
<p ng-if="page.values.length == 0">
<b>No records</b>
</p>
<div ng-if="page.values.length > 0">
<ul class="pager">
<li class="disabled" ng-if="page.currPage <= 0"><a href="javascript:void(0)">&larr; Prev</a></li>
<li ng-if="page.currPage > 0"><a href="javascript:void(0)" ng-click="gotoPage(page.currPage - 1)">&larr; Prev</a></li>
<li class="disabled" ng-if="page.totalPages <= page.currPage + 1"><a href="javascript:void(0)">Next &rarr;</a></li>
<li ng-if="page.currPage + 1 < page.totalPages"><a href="javascript:void(0)" ng-click="gotoPage(page.currPage + 1)">Next &rarr;</a></li>
</ul>
<div class="well" ng-repeat="val in page.values">
<p class="text-right"><span class="label label-primary"><b>Trust:</b> {{val.trust}}</span></p>
<table class="table table-striped">
<tr>
<th class="col-xs-4 col-md-3 col-lg-2">ID</th>
<td class="col-xs-8 col-md-9 col-lg-10">
{{val.result.openaireId}}
</td>
</tr>
<tr>
<th>Title(s)</th>
<td>
<span ng-repeat="t in val.result.titles track by $index">
<br ng-if="$index > 0"/>
{{t}}
</span>
</td>
</tr>
<tr>
<th>Type</th>
<td>{{val.result.typology}}</td>
</tr>
<tr ng-class="{'success' : val.highlight.creators && val.highlight.creators.length > 0}">
<th>Author(s)</th>
<td>
<span ng-repeat="a in val.result.creators track by $index">
<br ng-if="$index > 0"/>
{{a.fullname}}<span ng-if="a.orcid"> (ORCID: {{a.orcid}})</span>
</span>
<span ng-if="val.highlight.creators && isAuthorHighlighted(a, val.highlight.creators)">
<span class="label label-success">added</span>
<span class="pull-right">
<b>from: </b>
<a href="{{val.provenance.url}}" target="_blank" ng-if="val.provenance.url">{{val.provenance.repositoryName}}</a>
<span ng-if="!val.provenance.url">{{val.provenance.repositoryName}}</span>
<span ng-if="val.provenance.id">(ID: {{val.provenance.id}})</span>
</span>
</span>
</td>
</tr>
<tr ng-class="{'success' : val.highlight.pids && val.highlight.pids.length > 0}">
<th>PID(s)</th>
<td>
<span ng-repeat="pid in val.result.pids">
<br ng-if="$index > 0"/>
{{pid.value}} <b>({{pid.type}})</b>
<span ng-if="val.highlight.pids && isPidHighlighted(pid, val.highlight.pids)">
<span class="label label-success">added</span>
<span class="pull-right">
<b>from: </b>
<a href="{{val.provenance.url}}" target="_blank" ng-if="val.provenance.url">{{val.provenance.repositoryName}}</a>
<span ng-if="!val.provenance.url">{{val.provenance.repositoryName}}</span>
<span ng-if="val.provenance.id">(ID: {{val.provenance.id}})</span>
</span>
</span>
</span>
</td>
</tr>
<tr ng-class="{'success' : val.highlight.abstracts && val.highlight.abstracts.length > 0}">
<th>Abstract(s)</th>
<td>
<span ng-repeat="d in val.result.abstracts track by $index">
<br ng-if="$index > 0"/>
{{d}}
</span>
<span class="pull-right" ng-if="val.highlight.abstracts">
<b>from: </b>
<a href="{{val.provenance.url}}" target="_blank" ng-if="val.provenance.url">{{val.provenance.repositoryName}}</a>
<span ng-if="!val.provenance.url">{{val.provenance.repositoryName}}</span>
<span ng-if="val.provenance.id">(ID: {{val.provenance.id}})</span>
</span>
</td>
</tr>
<tr ng-class="{'success' : val.highlight.subjects && val.highlight.subjects.length > 0}">
<th>Subject(s)</th>
<td>
<span ng-repeat="s in val.result.subjects track by $index">
<br ng-if="$index > 0"/>
{{s.value}}<b ng-if="s.type"> ({{s.type}})</b>
<span ng-if="val.highlight.subjects && isSubjectHighlighted(s, val.highlight.subjects)">
<span class="label label-success">added</span>
<span class="pull-right">
<b>from: </b>
<a href="{{val.provenance.url}}" target="_blank" ng-if="val.provenance.url">{{val.provenance.repositoryName}}</a>
<span ng-if="!val.provenance.url">{{val.provenance.repositoryName}}</span>
<span ng-if="val.provenance.id">(ID: {{val.provenance.id}})</span>
</span>
</span>
</span>
</td>
</tr>
<tr ng-class="{'success' : val.highlight.publicationdate }">
<th>Publication date</th>
<td>
{{val.result.publicationdate}}
<span class="pull-right" ng-if="val.highlight.publicationdate">
<b>from: </b>
<a href="{{val.provenance.url}}" target="_blank" ng-if="val.provenance.url">{{val.provenance.repositoryName}}</a>
<span ng-if="!val.provenance.url">{{val.provenance.repositoryName}}</span>
<span ng-if="val.provenance.id">(ID: {{val.provenance.id}})</span>
</span>
</td>
</tr>
<tr ng-class="{'success' : val.highlight.instances && val.highlight.instances.length > 0}">
<th>Rights</th>
<td>
<span ng-repeat="i in val.result.instances">
<br ng-if="$index > 0"/>
{{i.license}} <b>({{i.hostedby}})</b>
<span ng-if="val.highlight.instances && isInstanceHighlighted(i, val.highlight.instances)">
<span class="label label-success">added</span>
<span class="pull-right">
<b>from: </b>
<a href="{{val.provenance.url}}" target="_blank" ng-if="val.provenance.url">{{val.provenance.repositoryName}}</a>
<span ng-if="!val.provenance.url">{{val.provenance.repositoryName}}</span>
<span ng-if="val.provenance.id">(ID: {{val.provenance.id}})</span>
</span>
</span>
<br />
<a href="{{i.url}}" target="_blank">{{i.url}}</a>
</span>
</td>
</tr>
<tr ng-class="{'success' : val.highlight.projects && val.highlight.projects.length > 0}">
<th>Related project(s)</th>
<td>
<span ng-repeat="p in val.result.projects">
<br ng-if="$index > 0"/>
{{p.title}}
<span ng-if="val.highlight.projects && isProjectHighlighted(p, val.highlight.projects)">
<span class="label label-success">added</span>
<span class="pull-right">
<b>from: </b>
<a href="{{val.provenance.url}}" target="_blank" ng-if="val.provenance.url">{{val.provenance.repositoryName}}</a>
<span ng-if="!val.provenance.url">{{val.provenance.repositoryName}}</span>
<span ng-if="val.provenance.id">(ID: {{val.provenance.id}})</span>
</span>
</span>
<ul>
<li ng-if="p.acronym">acronym: <i>{{p.acronym}}</i></li>
<li ng-if="p.code">code: <i>{{p.code}}</i></li>
<li ng-if="p.funder">funder: <i>{{p.funder}}</i></li>
<li ng-if="p.fundingProgram">fundingProgram: <i>{{p.fundingProgram}}</i></li>
<li ng-if="p.jurisdiction">jurisdiction: <i>{{p.jurisdiction}}</i></li>
</ul>
</span>
</td>
</tr>
<!-- DATASETS, PUBLICATIONS AND SOFTWARES -->
<tr ng-class="{'success' : val.highlight.datasets && val.highlight.datasets.length > 0}">
<th>Related dataset(s)</th>
<td>
<span ng-repeat="d in val.result.datasets">
<br ng-if="$index > 0"/>
{{d.title}}
<span ng-if="val.highlight.datasets && isDatasetHighlighted(d, val.highlight.datasets)">
<span class="label label-success">added</span>
<span class="pull-right">
<b>from: </b>
<a href="{{val.provenance.url}}" target="_blank" ng-if="val.provenance.url">{{val.provenance.repositoryName}}</a>
<span ng-if="!val.provenance.url">{{val.provenance.repositoryName}}</span>
<span ng-if="val.provenance.id">(ID: {{val.provenance.id}})</span>
</span>
</span>
<ul class="small">
<li ng-repeat="pid in d.pids"><b>{{pid.type}}:</b> {{pid.value}}</li>
</ul>
</span>
</td>
</tr>
<tr ng-class="{'success' : val.highlight.publications && val.highlight.publications.length > 0}">
<th>Related publications(s)</th>
<td>
<span ng-repeat="p in val.result.publications">
<br ng-if="$index > 0"/>
{{p.title}}
<span ng-if="val.highlight.publications && isRelatedPublicationHighlighted(p, val.highlight.publications)">
<span class="label label-success">added</span>
<span class="pull-right">
<b>from: </b>
<a href="{{val.provenance.url}}" target="_blank" ng-if="val.provenance.url">{{val.provenance.repositoryName}}</a>
<span ng-if="!val.provenance.url">{{val.provenance.repositoryName}}</span>
<span ng-if="val.provenance.id">(ID: {{val.provenance.id}})</span>
</span>
</span>
<ul class="small">
<li ng-repeat="pid in p.pids"><b>{{pid.type}}:</b> {{pid.value}}</li>
</ul>
</span>
</td>
</tr>
<tr ng-class="{'success' : val.highlight.softwares && val.highlight.softwares.length > 0}">
<th>Related softwares(s)</th>
<td>
<span ng-repeat="sw in val.result.softwares">
<br ng-if="$index > 0"/>
{{sw}}
<span ng-if="val.highlight.softwares && isSoftwareHighlighted(sw, val.highlight.softwares)">
<span class="label label-success">added</span>
<span class="pull-right">
<b>from: </b>
<a href="{{val.provenance.url}}" target="_blank" ng-if="val.provenance.url">{{val.provenance.repositoryName}}</a>
<span ng-if="!val.provenance.url">{{val.provenance.repositoryName}}</span>
<span ng-if="val.provenance.id">(ID: {{val.provenance.id}})</span>
</span>
</span>
<ul class="small">
<li ng-repeat="pid in d.pids"><b>{{pid.type}}:</b> {{pid.value}}</li>
</ul>
</span>
</td>
</tr>
</table>
</div>
<ul class="pager">
<li class="disabled" ng-if="page.currPage <= 0"><a href="javascript:void(0)">&larr; Prev</a></li>
<li ng-if="page.currPage > 0"><a href="javascript:void(0)" ng-click="gotoPage(page.currPage - 1)">&larr; Prev</a></li>
<li class="disabled" ng-if="page.totalPages <= page.currPage + 1"><a href="javascript:void(0)">Next &rarr;</a></li>
<li ng-if="page.currPage + 1 < page.totalPages"><a href="javascript:void(0)" ng-click="gotoPage(page.currPage + 1)">Next &rarr;</a></li>
</ul>
</div>
</div>
<back-top><span class="glyphicon glyphicon-chevron-up"></span> top</back-top>

100
dhp-broker-application/src/main/resources/static/openaire/openaire-lbs-ui.js → apps/dhp-broker-application/src/main/resources/static/openaire/openaire-lbs-ui.js
View File

@ -193,11 +193,21 @@ openaireBrokerControllers.controller('queryCtrl', function ($scope, $http, $sce,
return (s && ((s.min && s.min.trim().length > 0) || (s.max && s.max.trim().length > 0)));
}
$scope.isAuthorHighlighted = function(author, list) {
if (list) {
for (var i=0; i<list.length; i++) {
if (author.fullname + author.orcid == list[i].fullname + list[i].orcid) {
return true;
}
}
}
return false;
}
$scope.isSubjectHighlighted = function(subject, list) {
if (list) {
for (var i=0; i<list.length; i++) {
if (subject == list[i]) {
if (subject.value == list[i].value) {
return true;
}
}
@ -219,7 +229,7 @@ openaireBrokerControllers.controller('queryCtrl', function ($scope, $http, $sce,
$scope.isProjectHighlighted= function(project, list) {
if (list) {
for (var i=0; i<list.length; i++) {
if (project.code == list[i].code) {
if (project.openaireId == list[i].openaireId) {
return true;
}
}
@ -241,18 +251,30 @@ openaireBrokerControllers.controller('queryCtrl', function ($scope, $http, $sce,
$scope.isDatasetHighlighted= function(dataset, list) {
if (list) {
for (var i=0; i<list.length; i++) {
if (dataset.originalId == list[i].originalId) {
if (dataset.openaireId == list[i].openaireId) {
return true;
}
}
}
return false;
}
$scope.isSoftwareHighlighted= function(sw, list) {
if (list) {
for (var i=0; i<list.length; i++) {
if (sw.openaireId == list[i].openaireId) {
return true;
}
}
}
return false;
}
$scope.isRelatedPublicationHighlighted= function(pub, list) {
if (list) {
for (var i=0; i<list.length; i++) {
if (pub.originalId == list[i].originalId) {
if (pub.openaireId == list[i].openaireId) {
return true;
}
}
@ -260,9 +282,6 @@ openaireBrokerControllers.controller('queryCtrl', function ($scope, $http, $sce,
return false;
}
$scope.prepareAdvancedSearchForm = function() {
$scope.tempQuery = angular.copy($scope.query);
if ($scope.tempQuery.titles.length == 0) { $scope.tempQuery.titles.push(''); }
@ -329,6 +348,28 @@ openaireBrokerControllers.controller('notificationsCtrl', function ($scope, $htt
$location.path('/ntf/' + n + "/" + $scope.subscrId);
}
$scope.isAuthorHighlighted = function(author, list) {
if (list) {
for (var i=0; i<list.length; i++) {
if (author.fullname + author.orcid == list[i].fullname + list[i].orcid) {
return true;
}
}
}
return false;
}
$scope.isSubjectHighlighted = function(subject, list) {
if (list) {
for (var i=0; i<list.length; i++) {
if (subject.value == list[i].value) {
return true;
}
}
}
return false;
}
$scope.isInstanceHighlighted = function(instance, list) {
if (list) {
for (var i=0; i<list.length; i++) {
@ -340,6 +381,17 @@ openaireBrokerControllers.controller('notificationsCtrl', function ($scope, $htt
return false;
}
$scope.isProjectHighlighted= function(project, list) {
if (list) {
for (var i=0; i<list.length; i++) {
if (project.openaireId == list[i].openaireId) {
return true;
}
}
}
return false;
}
$scope.isPidHighlighted = function(pid, list) {
if (list) {
for (var i=0; i<list.length; i++) {
@ -351,5 +403,39 @@ openaireBrokerControllers.controller('notificationsCtrl', function ($scope, $htt
return false;
}
$scope.isDatasetHighlighted= function(dataset, list) {
if (list) {
for (var i=0; i<list.length; i++) {
if (dataset.openaireId == list[i].openaireId) {
return true;
}
}
}
return false;
}
$scope.isSoftwareHighlighted= function(sw, list) {
if (list) {
for (var i=0; i<list.length; i++) {
if (sw.openaireId == list[i].openaireId) {
return true;
}
}
}
return false;
}
$scope.isRelatedPublicationHighlighted= function(pub, list) {
if (list) {
for (var i=0; i<list.length; i++) {
if (pub.openaireId == list[i].openaireId) {
return true;
}
}
}
return false;
}
$scope.refresh();
});

52
dhp-broker-application/src/main/resources/static/openaire/query.html → apps/dhp-broker-application/src/main/resources/static/openaire/query.html
View File

@ -20,14 +20,13 @@
<li ng-if="eventsPage.currPage + 1 < eventsPage.totalPages"><a href="javascript:void(0)" ng-click="gotoPage(eventsPage.currPage + 1)">Next &rarr;</a></li>
</ul>
<div class="well" ng-repeat="val in eventsPage.values">
<p class="text-right"><span class="label label-primary"><b>Trust:</b> {{val.trust}}</span></p>
<table class="table table-striped">
<tr>
<th class="col-xs-4 col-md-3 col-lg-2">ID</th>
<td class="col-xs-8 col-md-9 col-lg-10">
{{val.result.originalId}}
{{val.result.openaireId}}
</td>
</tr>
<tr>
@ -40,11 +39,24 @@
</td>
</tr>
<tr>
<th>Type</th>
<td>{{val.result.typology}}</td>
</tr>
<tr ng-class="{'success' : val.highlight.creators && val.highlight.creators.length > 0}">
<th>Author(s)</th>
<td>
<span ng-repeat="a in val.result.creators track by $index">
<br ng-if="$index > 0"/>
{{a}}
{{a.fullname}}<span ng-if="a.orcid"> (ORCID: {{a.orcid}})</span>
<span ng-if="val.highlight.creators && isAuthorHighlighted(a, val.highlight.creators)">
<span class="label label-success">added</span>
<span class="pull-right">
<b>from: </b>
<a href="{{val.provenance.url}}" target="_blank" ng-if="val.provenance.url">{{val.provenance.repositoryName}}</a>
<span ng-if="!val.provenance.url">{{val.provenance.repositoryName}}</span>
<span ng-if="val.provenance.id">(ID: {{val.provenance.id}})</span>
</span>
</span>
</span>
</td>
</tr>
@ -88,7 +100,7 @@
<td>
<span ng-repeat="s in val.result.subjects track by $index">
<br ng-if="$index > 0"/>
{{s}}
{{s.value}}<b ng-if="s.type"> ({{s.type}})</b>
<span ng-if="val.highlight.subjects && isSubjectHighlighted(s, val.highlight.subjects)">
<span class="label label-success">added</span>
<span class="pull-right">
@ -134,7 +146,7 @@
</td>
</tr>
<tr ng-class="{'success' : val.highlight.projects && val.highlight.projects.length > 0}">
<th>Project(s)</th>
<th>Related project(s)</th>
<td>
<span ng-repeat="p in val.result.projects">
<br ng-if="$index > 0"/>
@ -161,13 +173,13 @@
<!-- DATASETS AND PLUBLICATIONS -->
<!-- DATASETS, PUBLICATIONS AND SOFTWARES -->
<tr ng-class="{'success' : val.highlight.datasets && val.highlight.datasets.length > 0}">
<th>Dataset(s)</th>
<th>Related dataset(s)</th>
<td>
<span ng-repeat="d in val.result.datasets">
<br ng-if="$index > 0"/>
{{d.titles[0]}}
{{d.title}}
<span ng-if="val.highlight.datasets && isDatasetHighlighted(d, val.highlight.datasets)">
<span class="label label-success">added</span>
<span class="pull-right">
@ -189,7 +201,7 @@
<td>
<span ng-repeat="p in val.result.publications">
<br ng-if="$index > 0"/>
{{p.titles[0]}}
{{p.title}}
<span ng-if="val.highlight.publications && isRelatedPublicationHighlighted(p, val.highlight.publications)">
<span class="label label-success">added</span>
<span class="pull-right">
@ -206,6 +218,28 @@
</td>
</tr>
<tr ng-class="{'success' : val.highlight.softwares && val.highlight.softwares.length > 0}">
<th>Related softwares(s)</th>
<td>
<span ng-repeat="sw in val.result.softwares">
<br ng-if="$index > 0"/>
{{sw}}
<span ng-if="val.highlight.softwares && isSoftwareHighlighted(sw, val.highlight.softwares)">
<span class="label label-success">added</span>
<span class="pull-right">
<b>from: </b>
<a href="{{val.provenance.url}}" target="_blank" ng-if="val.provenance.url">{{val.provenance.repositoryName}}</a>
<span ng-if="!val.provenance.url">{{val.provenance.repositoryName}}</span>
<span ng-if="val.provenance.id">(ID: {{val.provenance.id}})</span>
</span>
</span>
<ul class="small">
<li ng-repeat="pid in d.pids"><b>{{pid.type}}:</b> {{pid.value}}</li>
</ul>
</span>
</td>
</tr>
</table>
</div>

0
dhp-broker-application/src/main/resources/static/openaire/subscriptions.html → apps/dhp-broker-application/src/main/resources/static/openaire/subscriptions.html
View File

17
apps/dhp-broker-application/src/main/resources/templates/dhp_openaire_mail.st Normal file
View File

@ -0,0 +1,17 @@
You have received this mail because you are owner of the following subscription:
<ul>
<li><b>ID: </b>$sub.subscriptionId$</li>
<li><b>Topic: </b>$sub.topic$</li>
<li><b>Number of events: </b>$oa_notifications_total$</li>
</ul>
<hr />
<p>
You can access the notified events at this address: .... <br />
or using the Public Broker Service API.
</p>
<hr />
This email message was auto-generated. Please do not respond.

0
dhp-broker-application/src/main/resources/templates/mail.st → apps/dhp-broker-application/src/main/resources/templates/mail.st
View File

25
apps/dhp-broker-application/src/main/resources/templates/openaire_mail.st Normal file
View File

@ -0,0 +1,25 @@
You have received this mail because you are owner of the following subscription:
<ul>
<li><b>Topic: </b>$sub.topic$</li>
<li><b>Frequency: </b>$sub.frequency$</li>
<li><b>Mode: </b>$sub.mode$</li>
</ul>
<hr />
<p>
There are $total$ event(s) related to the following publications<br />
(The list is limited to $max$ events, see the Content Provider Dashboard for all):
</p>
<ol>
$events:{e|
<li>
<b>$e.map.targetResultTitle$</b><br/>
<small style="color: grey">$e.map.targetAuthors$</small>
</li>
}$
</ol>
<hr />
This email message was auto-generated. Please do not respond.

2
dhp-broker-application/src/test/java/eu/dnetlib/lbs/LiteratureBrokerServiceApplicationTests.java → apps/dhp-broker-application/src/test/java/eu/dnetlib/broker/LiteratureBrokerServiceApplicationTests.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs;
package eu.dnetlib.broker;
import org.junit.Ignore;
import org.junit.Test;

4
dhp-broker-application/src/test/java/eu/dnetlib/lbs/integration/ScrollTest.java → apps/dhp-broker-application/src/test/java/eu/dnetlib/broker/integration/ScrollTest.java
View File

@ -1,11 +1,11 @@
package eu.dnetlib.lbs.integration;
package eu.dnetlib.broker.integration;
import org.junit.Ignore;
import org.junit.Test;
import org.springframework.web.client.RestTemplate;
import eu.dnetlib.broker.objects.OaBrokerEventPayload;
import eu.dnetlib.lbs.openaire.ScrollPage;
import eu.dnetlib.broker.openaire.ScrollPage;
public class ScrollTest {

10
dhp-broker-application/src/test/java/eu/dnetlib/lbs/tests/OpenaireQueriesTest.java → apps/dhp-broker-application/src/test/java/eu/dnetlib/broker/tests/OpenaireQueriesTest.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.lbs.tests;
package eu.dnetlib.broker.tests;
import java.util.Arrays;
@ -22,12 +22,12 @@ import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilde
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import eu.dnetlib.broker.common.elasticsearch.Event;
import eu.dnetlib.broker.objects.OaBrokerEventPayload;
import eu.dnetlib.broker.objects.OaBrokerMainEntity;
import eu.dnetlib.lbs.elasticsearch.Event;
import eu.dnetlib.lbs.openaire.AdvQueryObject;
import eu.dnetlib.lbs.openaire.ElasticSearchQueryUtils;
import eu.dnetlib.lbs.openaire.Range;
import eu.dnetlib.broker.openaire.AdvQueryObject;
import eu.dnetlib.broker.openaire.ElasticSearchQueryUtils;
import eu.dnetlib.broker.openaire.Range;
@Ignore
@RunWith(SpringJUnit4ClassRunner.class)

20
apps/dhp-broker-application/src/test/resources/applicationContext-test-queries.xml Normal file
View File

@ -0,0 +1,20 @@
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:elasticsearch="http://www.springframework.org/schema/data/elasticsearch"
xmlns:p="http://www.springframework.org/schema/p"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/data/elasticsearch http://www.springframework.org/schema/data/elasticsearch/spring-elasticsearch.xsd">
<elasticsearch:transport-client id="elasticSearchClient"
cluster-nodes="localhost:9300"
cluster-name="elasticsearch_michele" />
<elasticsearch:repositories
base-package="eu.dnetlib.broker.elasticsearch"
elasticsearch-template-ref="elasticsearchTemplate" />
<bean id="elasticsearchTemplate" class="org.springframework.data.elasticsearch.core.ElasticsearchTemplate">
<constructor-arg name="client" ref="elasticSearchClient" />
</bean>
</beans>

13
apps/dhp-broker-application/src/test/resources/logback-test.xml Normal file
View File

@ -0,0 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<include resource="org/springframework/boot/logging/logback/base.xml"/>
<logger name="org.springframework" level="OFF">
<appender-ref ref="CONSOLE" />
</logger>
<logger name="root" level="OFF">
<appender-ref ref="CONSOLE" />
</logger>
<logger name="eu.dnetlib.broker" level="INFO">
<appender-ref ref="CONSOLE" />
</logger>
</configuration>

0
dhp-broker-application/src/test/resources/scripts/perl/eventImporter.pl → apps/dhp-broker-application/src/test/resources/scripts/perl/eventImporter.pl
View File

0
dhp-broker-application/src/test/resources/scripts/perl/mockImporter.pl → apps/dhp-broker-application/src/test/resources/scripts/perl/mockImporter.pl
View File

0
dhp-broker-application/src/test/resources/scripts/perl/test.pl → apps/dhp-broker-application/src/test/resources/scripts/perl/test.pl
View File

0
dhp-broker-application/src/test/resources/test-event-1.json → apps/dhp-broker-application/src/test/resources/test-event-1.json
View File

0
dhp-broker-application/src/test/resources/test-search-on-map.json → apps/dhp-broker-application/src/test/resources/test-search-on-map.json
View File

Some files were not shown because too many files have changed in this diff Show More