1
0
Fork 0

changed the implemetation of RabitMQ Comunication

This commit is contained in:
Sandro La Bruzzo 2019-04-16 12:28:01 +02:00
parent 403c13eebf
commit 53ec9bccca
5 changed files with 137 additions and 90 deletions

View File

@ -192,16 +192,18 @@ public class DnetCollectorWorkerApplication implements CommandLineRunner {
log.info("Created path "+hdfswritepath.toString()); log.info("Created path "+hdfswritepath.toString());
final Map<String, String> ongoingMap = new HashMap<>();
final Map<String, String> reportMap = new HashMap<>();
final AtomicInteger counter = new AtomicInteger(0);
try(SequenceFile.Writer writer = SequenceFile.createWriter(conf, try(SequenceFile.Writer writer = SequenceFile.createWriter(conf,
SequenceFile.Writer.file(hdfswritepath), SequenceFile.Writer.keyClass(IntWritable.class), SequenceFile.Writer.file(hdfswritepath), SequenceFile.Writer.keyClass(IntWritable.class),
SequenceFile.Writer.valueClass(Text.class))) { SequenceFile.Writer.valueClass(Text.class))) {
final AtomicInteger counter = new AtomicInteger(0);
final IntWritable key = new IntWritable(counter.get()); final IntWritable key = new IntWritable(counter.get());
final Text value = new Text(); final Text value = new Text();
final Map<String, String> ongoingMap = new HashMap<>();
final Map<String, String> reportMap = new HashMap<>();
plugin.collect(api).forEach(content -> { plugin.collect(api).forEach(content -> {
@ -223,12 +225,13 @@ public class DnetCollectorWorkerApplication implements CommandLineRunner {
} }
}); });
}
ongoingMap.put("ongoing", ""+counter.get()); ongoingMap.put("ongoing", ""+counter.get());
manager.sendMessage(new Message(workflowId,"Collection", MessageType.ONGOING, ongoingMap ), rabbitOngoingQueue, true, false); manager.sendMessage(new Message(workflowId,"Collection", MessageType.ONGOING, ongoingMap ), rabbitOngoingQueue, true, false);
reportMap.put("collected", ""+counter.get()); reportMap.put("collected", ""+counter.get());
manager.sendMessage(new Message(workflowId,"Collection", MessageType.REPORT, reportMap ), rabbitReportQueue, true, false); manager.sendMessage(new Message(workflowId,"Collection", MessageType.REPORT, reportMap ), rabbitReportQueue, true, false);
manager.close();
}
} }
} }

View File

@ -25,10 +25,6 @@
</repository> </repositories> --> </repository> </repositories> -->
<!-- Add typical dependencies for a web application --> <!-- Add typical dependencies for a web application -->
<dependencies> <dependencies>
<dependency> <dependency>
@ -86,6 +82,17 @@
</dependency> </dependency>
</dependencies> </dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<executable>true</executable>
</configuration>
</plugin>
</plugins>
</build>
</project> </project>

View File

@ -2,8 +2,8 @@ spring.main.banner-mode = console
logging.level.root = INFO logging.level.root = INFO
spring.datasource.url=jdbc:postgresql://localhost:5432/mdstoremanager spring.datasource.url=jdbc:postgresql://localhost:5432/mdstoremanager
spring.datasource.username= spring.datasource.username=dnet
spring.datasource.password= spring.datasource.password=dnetPwd
spring.jpa.properties.hibernate.dialect = org.hibernate.dialect.PostgreSQLDialect spring.jpa.properties.hibernate.dialect = org.hibernate.dialect.PostgreSQLDialect

View File

@ -19,6 +19,9 @@ public class MessageManager {
private final String password; private final String password;
private Connection connection;
private Map<String , Channel> channels = new HashMap<>();
private boolean durable; private boolean durable;
@ -59,9 +62,36 @@ public class MessageManager {
channel.queueDeclare(queueName, durable, false, this.autodelete, args); channel.queueDeclare(queueName, durable, false, this.autodelete, args);
return channel; return channel;
} }
public boolean sendMessage(final Message message, String queueName) throws Exception {
try (Connection connection = createConnection(); Channel channel = createChannel(connection, queueName, this.durable, this.autodelete)) {
private Channel getOrCreateChannel(final String queueName, boolean durable, boolean autodelete) throws Exception {
if (channels.containsKey(queueName)) {
return channels.get(queueName);
}
if (this.connection == null) {
this.connection = createConnection();
}
channels.put(queueName, createChannel(this.connection, queueName, durable, autodelete));
return channels.get(queueName);
}
public void close() throws IOException {
channels.values().forEach(ch-> {
try {
ch.close();
} catch (Exception e) {
//TODO LOG
}
});
this.connection.close();
}
public boolean sendMessage(final Message message, String queueName) throws Exception {
try {
Channel channel = getOrCreateChannel(queueName, this.durable, this.autodelete);
channel.basicPublish("", queueName,null, message.toString().getBytes()); channel.basicPublish("", queueName,null, message.toString().getBytes());
return true; return true;
} catch (Throwable e) { } catch (Throwable e) {
@ -70,8 +100,8 @@ public class MessageManager {
} }
public boolean sendMessage(final Message message, String queueName, boolean durable_var, boolean autodelete_var) throws Exception { public boolean sendMessage(final Message message, String queueName, boolean durable_var, boolean autodelete_var) throws Exception {
try (Connection connection = createConnection(); Channel channel = createChannel(connection, queueName, durable_var, autodelete_var)) { try {
Channel channel = getOrCreateChannel(queueName, durable_var, autodelete_var);
channel.basicPublish("", queueName,null, message.toString().getBytes()); channel.basicPublish("", queueName,null, message.toString().getBytes());
return true; return true;
} catch (Throwable e) { } catch (Throwable e) {

View File

@ -140,12 +140,19 @@ public class TransformSparkJobNode {
if (rabbitHost != null) { if (rabbitHost != null) {
System.out.println("SEND FINAL REPORT");
final Map<String, String> reportMap = new HashMap<>(); final Map<String, String> reportMap = new HashMap<>();
reportMap.put("inputItem" , ""+ totalItems.value()); reportMap.put("inputItem" , ""+ totalItems.value());
reportMap.put("invalidRecords", "" + errorItems.value()); reportMap.put("invalidRecords", "" + errorItems.value());
reportMap.put("mdStoreSize", "" + transformedItems.value()); reportMap.put("mdStoreSize", "" + transformedItems.value());
final MessageManager manager = new MessageManager(rabbitHost, rabbitUser, rabbitPassword, false, false, null); final MessageManager manager = new MessageManager(rabbitHost, rabbitUser, rabbitPassword, false, false, null);
System.out.println(new Message(workflowId, "Transform", MessageType.REPORT, reportMap));
manager.sendMessage(new Message(workflowId, "Transform", MessageType.REPORT, reportMap), rabbitReportQueue, true, false); manager.sendMessage(new Message(workflowId, "Transform", MessageType.REPORT, reportMap), rabbitReportQueue, true, false);
manager.close();
} }
} }