andreas.czerniak
/
BrBETA_dnet-hadoop
forked from D-Net/dnet-hadoop
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

changed the implemetation of RabitMQ Comunication

This commit is contained in:
Sandro La Bruzzo 2019-04-16 12:28:01 +02:00
parent 403c13eebf
commit 53ec9bccca
5 changed files with 137 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
dhp-applications/dhp-collector-worker/src/main/java/eu/dnetlib/collector/worker/DnetCollectorWorkerApplication.java
View File

@ -192,16 +192,18 @@ public class DnetCollectorWorkerApplication implements CommandLineRunner {
log.info("Created path "+hdfswritepath.toString()); log.info("Created path "+hdfswritepath.toString());
final Map<String, String> ongoingMap = new HashMap<>();
final Map<String, String> reportMap = new HashMap<>();
final AtomicInteger counter = new AtomicInteger(0);
try(SequenceFile.Writer writer = SequenceFile.createWriter(conf, try(SequenceFile.Writer writer = SequenceFile.createWriter(conf,
SequenceFile.Writer.file(hdfswritepath), SequenceFile.Writer.keyClass(IntWritable.class), SequenceFile.Writer.file(hdfswritepath), SequenceFile.Writer.keyClass(IntWritable.class),
SequenceFile.Writer.valueClass(Text.class))) { SequenceFile.Writer.valueClass(Text.class))) {
final AtomicInteger counter = new AtomicInteger(0);
final IntWritable key = new IntWritable(counter.get()); final IntWritable key = new IntWritable(counter.get());
final Text value = new Text(); final Text value = new Text();
final Map<String, String> ongoingMap = new HashMap<>();
final Map<String, String> reportMap = new HashMap<>();
plugin.collect(api).forEach(content -> { plugin.collect(api).forEach(content -> {
@ -223,12 +225,13 @@ public class DnetCollectorWorkerApplication implements CommandLineRunner {
} }
}); });
ongoingMap.put("ongoing", ""+counter.get());
manager.sendMessage(new Message(workflowId,"Collection", MessageType.ONGOING, ongoingMap ), rabbitOngoingQueue, true, false);
reportMap.put("collected", ""+counter.get());
manager.sendMessage(new Message(workflowId,"Collection", MessageType.REPORT, reportMap ), rabbitReportQueue, true, false);
} }
ongoingMap.put("ongoing", ""+counter.get());
manager.sendMessage(new Message(workflowId,"Collection", MessageType.ONGOING, ongoingMap ), rabbitOngoingQueue, true, false);
reportMap.put("collected", ""+counter.get());
manager.sendMessage(new Message(workflowId,"Collection", MessageType.REPORT, reportMap ), rabbitReportQueue, true, false);
manager.close();
} }
} }

161
dhp-applications/dhp-mdstore-manager-app/pom.xml
View File

@ -1,91 +1,98 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" <project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<artifactId>dhp-mdstore-manager-app</artifactId> <artifactId>dhp-mdstore-manager-app</artifactId>
<version>1.1.0-SNAPSHOT</version> <version>1.1.0-SNAPSHOT</version>
<!-- Inherit defaults from Spring Boot --> <!-- Inherit defaults from Spring Boot -->
<parent> <parent>
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId> <artifactId>spring-boot-starter-parent</artifactId>
<version>2.1.3.RELEASE</version> <version>2.1.3.RELEASE</version>
<relativePath>../</relativePath> <relativePath>../</relativePath>
</parent> </parent>
<!-- <repositories> <repository> <id>dnet-deps</id> <name>dnet-dependencies</name> <!-- <repositories> <repository> <id>dnet-deps</id> <name>dnet-dependencies</name>
<url>http://maven.research-infrastructures.eu/nexus/content/repositories/dnet-deps</url> <url>http://maven.research-infrastructures.eu/nexus/content/repositories/dnet-deps</url>
<layout>default</layout> </repository> <repository> <id>dnet45-releases</id> <layout>default</layout> </repository> <repository> <id>dnet45-releases</id>
<name>D-Net 45 Releases</name> <url>http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-releases</url> <name>D-Net 45 Releases</name> <url>http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-releases</url>
<layout>default</layout> <snapshots> <enabled>true</enabled> </snapshots> <layout>default</layout> <snapshots> <enabled>true</enabled> </snapshots>
</repository> <repository> <id>dnet45-snapshots</id> <name>D-Net 45 Snapshots</name> </repository> <repository> <id>dnet45-snapshots</id> <name>D-Net 45 Snapshots</name>
<url>http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots</url> <url>http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots</url>
<layout>default</layout> <snapshots> <enabled>true</enabled> </snapshots> <layout>default</layout> <snapshots> <enabled>true</enabled> </snapshots>
</repository> </repositories> --> </repository> </repositories> -->
<!-- Add typical dependencies for a web application -->
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-json</artifactId>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
</dependency>
<dependency>
<groupId>com.vladmihalcea</groupId>
<artifactId>hibernate-types-52</artifactId>
<version>2.3.5</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.6</version>
</dependency>
<!-- Common -->
<dependency>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dhp-common</artifactId>
<version>1.0.0-SNAPSHOT</version>
</dependency>
<!-- Swagger -->
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger2</artifactId>
<version>2.9.2</version>
</dependency>
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger-ui</artifactId>
<version>2.9.2</version>
</dependency>
<!-- Add typical dependencies for a web application --> <!-- JUnit -->
<dependencies> <dependency>
<dependency> <groupId>junit</groupId>
<groupId>org.springframework.boot</groupId> <artifactId>junit</artifactId>
<artifactId>spring-boot-starter-web</artifactId> <scope>test</scope>
</dependency> </dependency>
<dependency> </dependencies>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-json</artifactId>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
</dependency>
<dependency>
<groupId>com.vladmihalcea</groupId>
<artifactId>hibernate-types-52</artifactId>
<version>2.3.5</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.6</version>
</dependency>
<!-- Common -->
<dependency>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dhp-common</artifactId>
<version>1.0.0-SNAPSHOT</version>
</dependency>
<!-- Swagger -->
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger2</artifactId>
<version>2.9.2</version>
</dependency>
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger-ui</artifactId>
<version>2.9.2</version>
</dependency>
<!-- JUnit -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<executable>true</executable>
</configuration>
</plugin>
</plugins>
</build>
</project> </project>

4
dhp-applications/dhp-mdstore-manager-app/src/main/resources/application.properties
View File

@ -2,8 +2,8 @@ spring.main.banner-mode = console
logging.level.root = INFO logging.level.root = INFO
spring.datasource.url=jdbc:postgresql://localhost:5432/mdstoremanager spring.datasource.url=jdbc:postgresql://localhost:5432/mdstoremanager
spring.datasource.username= spring.datasource.username=dnet
spring.datasource.password= spring.datasource.password=dnetPwd
spring.jpa.properties.hibernate.dialect = org.hibernate.dialect.PostgreSQLDialect spring.jpa.properties.hibernate.dialect = org.hibernate.dialect.PostgreSQLDialect

38
dhp-common/src/main/java/eu/dnetlib/message/MessageManager.java
View File

@ -19,6 +19,9 @@ public class MessageManager {
private final String password; private final String password;
private Connection connection;
private Map<String , Channel> channels = new HashMap<>();
private boolean durable; private boolean durable;
@ -59,9 +62,36 @@ public class MessageManager {
channel.queueDeclare(queueName, durable, false, this.autodelete, args); channel.queueDeclare(queueName, durable, false, this.autodelete, args);
return channel; return channel;
} }
public boolean sendMessage(final Message message, String queueName) throws Exception {
try (Connection connection = createConnection(); Channel channel = createChannel(connection, queueName, this.durable, this.autodelete)) {
private Channel getOrCreateChannel(final String queueName, boolean durable, boolean autodelete) throws Exception {
if (channels.containsKey(queueName)) {
return channels.get(queueName);
}
if (this.connection == null) {
this.connection = createConnection();
}
channels.put(queueName, createChannel(this.connection, queueName, durable, autodelete));
return channels.get(queueName);
}
public void close() throws IOException {
channels.values().forEach(ch-> {
try {
ch.close();
} catch (Exception e) {
//TODO LOG
}
});
this.connection.close();
}
public boolean sendMessage(final Message message, String queueName) throws Exception {
try {
Channel channel = getOrCreateChannel(queueName, this.durable, this.autodelete);
channel.basicPublish("", queueName,null, message.toString().getBytes()); channel.basicPublish("", queueName,null, message.toString().getBytes());
return true; return true;
} catch (Throwable e) { } catch (Throwable e) {
@ -70,8 +100,8 @@ public class MessageManager {
} }
public boolean sendMessage(final Message message, String queueName, boolean durable_var, boolean autodelete_var) throws Exception { public boolean sendMessage(final Message message, String queueName, boolean durable_var, boolean autodelete_var) throws Exception {
try (Connection connection = createConnection(); Channel channel = createChannel(connection, queueName, durable_var, autodelete_var)) { try {
Channel channel = getOrCreateChannel(queueName, durable_var, autodelete_var);
channel.basicPublish("", queueName,null, message.toString().getBytes()); channel.basicPublish("", queueName,null, message.toString().getBytes());
return true; return true;
} catch (Throwable e) { } catch (Throwable e) {

7
dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/transformation/TransformSparkJobNode.java
View File

@ -140,12 +140,19 @@ public class TransformSparkJobNode {
if (rabbitHost != null) { if (rabbitHost != null) {
System.out.println("SEND FINAL REPORT");
final Map<String, String> reportMap = new HashMap<>(); final Map<String, String> reportMap = new HashMap<>();
reportMap.put("inputItem" , ""+ totalItems.value()); reportMap.put("inputItem" , ""+ totalItems.value());
reportMap.put("invalidRecords", "" + errorItems.value()); reportMap.put("invalidRecords", "" + errorItems.value());
reportMap.put("mdStoreSize", "" + transformedItems.value()); reportMap.put("mdStoreSize", "" + transformedItems.value());
final MessageManager manager = new MessageManager(rabbitHost, rabbitUser, rabbitPassword, false, false, null); final MessageManager manager = new MessageManager(rabbitHost, rabbitUser, rabbitPassword, false, false, null);
System.out.println(new Message(workflowId, "Transform", MessageType.REPORT, reportMap));
manager.sendMessage(new Message(workflowId, "Transform", MessageType.REPORT, reportMap), rabbitReportQueue, true, false); manager.sendMessage(new Message(workflowId, "Transform", MessageType.REPORT, reportMap), rabbitReportQueue, true, false);
manager.close();
} }
} }