merge from private folder on SVN
git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/content-management/storage-manager-trigger@93065 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
b3cd12bbe2
commit
6001d5a0b4
|
@ -0,0 +1,10 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<classpath>
|
||||||
|
<classpathentry kind="src" output="target/classes" path="src/main/java"/>
|
||||||
|
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources"/>
|
||||||
|
<classpathentry kind="src" output="target/test-classes" path="src/test/java"/>
|
||||||
|
<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources"/>
|
||||||
|
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7"/>
|
||||||
|
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER"/>
|
||||||
|
<classpathentry kind="output" path="target/classes"/>
|
||||||
|
</classpath>
|
|
@ -0,0 +1,23 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<projectDescription>
|
||||||
|
<name>storage-manager-trigger</name>
|
||||||
|
<comment></comment>
|
||||||
|
<projects>
|
||||||
|
</projects>
|
||||||
|
<buildSpec>
|
||||||
|
<buildCommand>
|
||||||
|
<name>org.eclipse.jdt.core.javabuilder</name>
|
||||||
|
<arguments>
|
||||||
|
</arguments>
|
||||||
|
</buildCommand>
|
||||||
|
<buildCommand>
|
||||||
|
<name>org.eclipse.m2e.core.maven2Builder</name>
|
||||||
|
<arguments>
|
||||||
|
</arguments>
|
||||||
|
</buildCommand>
|
||||||
|
</buildSpec>
|
||||||
|
<natures>
|
||||||
|
<nature>org.eclipse.jdt.core.javanature</nature>
|
||||||
|
<nature>org.eclipse.m2e.core.maven2Nature</nature>
|
||||||
|
</natures>
|
||||||
|
</projectDescription>
|
|
@ -0,0 +1,5 @@
|
||||||
|
eclipse.preferences.version=1
|
||||||
|
encoding//src/main/java=UTF-8
|
||||||
|
encoding//src/main/resources=UTF-8
|
||||||
|
encoding//src/test/java=UTF-8
|
||||||
|
encoding//src/test/resources=UTF-8
|
|
@ -0,0 +1,5 @@
|
||||||
|
eclipse.preferences.version=1
|
||||||
|
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
|
||||||
|
org.eclipse.jdt.core.compiler.compliance=1.7
|
||||||
|
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
|
||||||
|
org.eclipse.jdt.core.compiler.source=1.7
|
|
@ -0,0 +1,4 @@
|
||||||
|
activeProfiles=
|
||||||
|
eclipse.preferences.version=1
|
||||||
|
resolveWorkspaceProjects=true
|
||||||
|
version=1
|
|
@ -0,0 +1,121 @@
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
<parent>
|
||||||
|
<artifactId>maven-parent</artifactId>
|
||||||
|
<groupId>org.gcube.tools</groupId>
|
||||||
|
<version>1.0.0</version>
|
||||||
|
<relativePath />
|
||||||
|
</parent>
|
||||||
|
<groupId>org.gcube.contentmanagement</groupId>
|
||||||
|
<artifactId>storage-manager-trigger</artifactId>
|
||||||
|
<version>0.0.1-SNAPSHOT</version>
|
||||||
|
<scm>
|
||||||
|
<connection>scm:svn:http://svn.d4science.research-infrastructures.eu/gcube/trunk/content-management/${project.artifactId}</connection>
|
||||||
|
<developerConnection>scm:svn:https://svn.d4science.research-infrastructures.eu/gcube/trunk/content-management/${project.artifactId}</developerConnection>
|
||||||
|
<url>http://svn.d4science.research-infrastructures.eu/gcube/trunk/content-management/${project.artifactId}</url>
|
||||||
|
</scm>
|
||||||
|
<properties>
|
||||||
|
<distroDirectory>${project.basedir}/distro</distroDirectory>
|
||||||
|
</properties>
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.mongodb</groupId>
|
||||||
|
<artifactId>mongo-java-driver</artifactId>
|
||||||
|
<version>[2.6.2,)</version>
|
||||||
|
</dependency>
|
||||||
|
<!-- <dependency> -->
|
||||||
|
<!-- <groupId>org.slf4j</groupId> -->
|
||||||
|
<!-- <artifactId>slf4j-nop</artifactId> -->
|
||||||
|
<!-- <version>1.6.4</version> -->
|
||||||
|
<!-- </dependency> -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-log4j12</artifactId>
|
||||||
|
<version>1.6.4</version>
|
||||||
|
</dependency>
|
||||||
|
<!-- <dependency> -->
|
||||||
|
<!-- <groupId>org.gcube.resources</groupId> -->
|
||||||
|
<!-- <artifactId>common-gcore-resources</artifactId> -->
|
||||||
|
<!-- <version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version> -->
|
||||||
|
<!-- </dependency> -->
|
||||||
|
<!-- <dependency> -->
|
||||||
|
<!-- <groupId>org.gcube.core</groupId> -->
|
||||||
|
<!-- <artifactId>common-gcore-stubs</artifactId> -->
|
||||||
|
<!-- <version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version> -->
|
||||||
|
<!-- </dependency> -->
|
||||||
|
<!-- <dependency> -->
|
||||||
|
<!-- <groupId>org.gcube.resources.discovery</groupId> -->
|
||||||
|
<!-- <artifactId>ic-client</artifactId> -->
|
||||||
|
<!-- <version>[1.0.0-SNAPSHOT, 2.0.0-SNAPSHOT)</version> -->
|
||||||
|
<!-- </dependency> -->
|
||||||
|
<!-- <dependency> -->
|
||||||
|
<!-- <groupId>org.gcube.core</groupId> -->
|
||||||
|
<!-- <artifactId>common-encryption</artifactId> -->
|
||||||
|
<!-- <version>[1.0.0-SNAPSHOT, 2.0.0-SNAPSHOT)</version> -->
|
||||||
|
<!-- </dependency> -->
|
||||||
|
<!-- <dependency> -->
|
||||||
|
<!-- <groupId>org.gcube.core</groupId> -->
|
||||||
|
<!-- <artifactId>common-scope-maps</artifactId> -->
|
||||||
|
<!-- <version>[1.0.0-SNAPSHOT, 2.0.0-SNAPSHOT)</version> -->
|
||||||
|
<!-- </dependency> -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>junit</groupId>
|
||||||
|
<artifactId>junit</artifactId>
|
||||||
|
<version>4.11</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.gcube.accounting</groupId>
|
||||||
|
<artifactId>common-accounting-lib</artifactId>
|
||||||
|
<version>[1.0.0-SNAPSHOT,2.0.0-SNAPSHOT)</version>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-resources-plugin</artifactId>
|
||||||
|
<version>2.5</version>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>copy-profile</id>
|
||||||
|
<phase>install</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>copy-resources</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<outputDirectory>target</outputDirectory>
|
||||||
|
<resources>
|
||||||
|
<resource>
|
||||||
|
<directory>${distroDirectory}</directory>
|
||||||
|
<filtering>true</filtering>
|
||||||
|
<includes>
|
||||||
|
<include>profile.xml</include>
|
||||||
|
</includes>
|
||||||
|
</resource>
|
||||||
|
</resources>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-assembly-plugin</artifactId>
|
||||||
|
<configuration>
|
||||||
|
<descriptors>
|
||||||
|
<descriptor>${distroDirectory}/descriptor.xml</descriptor>
|
||||||
|
</descriptors>
|
||||||
|
</configuration>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>servicearchive</id>
|
||||||
|
<phase>install</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>single</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
</project>
|
|
@ -0,0 +1,15 @@
|
||||||
|
log4j.rootLogger=INFO, A1, stdout
|
||||||
|
log4j.appender.A1=org.apache.log4j.RollingFileAppender
|
||||||
|
log4j.appender.A1.File=log.txt
|
||||||
|
log4j.appender.A1.layout=org.apache.log4j.PatternLayout
|
||||||
|
log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
|
||||||
|
# ***** Max file size is set to 100KB
|
||||||
|
log4j.appender.A1.MaxFileSize=100MB
|
||||||
|
# ***** Keep one backup file
|
||||||
|
log4j.appender.A1.MaxBackupIndex=1
|
||||||
|
|
||||||
|
#CONSOLE
|
||||||
|
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
|
||||||
|
log4j.appender.stdout.Threshold=INFO
|
||||||
|
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
|
||||||
|
log4j.appender.stdout.layout.ConversionPattern=[%t] %-5p %c %d{dd MMM yyyy ;HH:mm:ss.SSS} - %m%n
|
|
@ -0,0 +1,31 @@
|
||||||
|
package org.gcube.contentmanager.storageserver.accounting;
|
||||||
|
|
||||||
|
public interface Report {
|
||||||
|
/**
|
||||||
|
* Set generic properties of report
|
||||||
|
* @param resourceType
|
||||||
|
* @param consumerId
|
||||||
|
* @param resourceOwner
|
||||||
|
* @param resourceScope
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public void init(String consumerId, String resourceScope);
|
||||||
|
/**
|
||||||
|
* set start time of the operation
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public void timeUpdate();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set end time of operation and other specific properties
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public void ultimate(String owner, String uri, String operation, String size, String filePath, String id);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* send report
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public void send();
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,109 @@
|
||||||
|
package org.gcube.contentmanager.storageserver.accounting;
|
||||||
|
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.InetAddress;
|
||||||
|
import java.net.UnknownHostException;
|
||||||
|
import java.util.Calendar;
|
||||||
|
import java.util.Date;
|
||||||
|
import java.util.GregorianCalendar;
|
||||||
|
|
||||||
|
import org.gcube.accounting.datamodel.RawUsageRecord;
|
||||||
|
import org.gcube.accounting.exception.InvalidValueException;
|
||||||
|
import org.gcube.accounting.messaging.ResourceAccounting;
|
||||||
|
import org.gcube.accounting.messaging.ResourceAccountingFactory;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
public class ReportAccountingImpl implements Report {
|
||||||
|
final Logger logger = LoggerFactory.getLogger(ReportAccountingImpl.class);
|
||||||
|
public RawUsageRecord ur;
|
||||||
|
public ResourceAccounting raFactory;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(String consumerId, String resourceScope) {
|
||||||
|
logger.info("set accounting properties: consumerId "+consumerId+" scope: "+resourceScope);
|
||||||
|
raFactory = null;
|
||||||
|
try {
|
||||||
|
raFactory = ResourceAccountingFactory.getResourceAccountingInstance();
|
||||||
|
}
|
||||||
|
catch (IOException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
catch (Exception e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
this.ur = new RawUsageRecord();
|
||||||
|
//generic properties
|
||||||
|
ur.setResourceType("storage-usage");
|
||||||
|
if(consumerId!=null) ur.setConsumerId(consumerId);
|
||||||
|
// ur.setResourceOwner("paolo.fabriani");
|
||||||
|
if(resourceScope !=null) ur.setResourceScope(resourceScope);
|
||||||
|
//set creation time
|
||||||
|
Calendar createTime = new GregorianCalendar();
|
||||||
|
ur.setCreateTime(createTime.getTime());
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void timeUpdate() {
|
||||||
|
// setIpAddress();
|
||||||
|
setStartTime();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void ultimate(String owner, String uri, String operation, String size, String filePath, String id) {
|
||||||
|
setEndTime();
|
||||||
|
logger.info("set accounting properties: owner "+owner+" uri: "+uri+" operation: "+operation+" size: "+size);
|
||||||
|
//specific properties TODO
|
||||||
|
if(owner != null) ur.setResourceOwner(owner);
|
||||||
|
if(uri != null) ur.setResourceSpecificProperty("objectURI", uri);
|
||||||
|
if (operation!=null) ur.setResourceSpecificProperty("operationType",operation);
|
||||||
|
ur.setResourceSpecificProperty("dataType","STORAGE");
|
||||||
|
if(size!= null) ur.setResourceSpecificProperty("dataVolume", size);
|
||||||
|
if(filePath != null) ur.setResourceSpecificProperty("remotePath", filePath);
|
||||||
|
if(id!= null) ur.setResourceSpecificProperty("id", id);
|
||||||
|
|
||||||
|
}
|
||||||
|
@Override
|
||||||
|
public void send() {
|
||||||
|
logger.info("report sending ");
|
||||||
|
raFactory.sendAccountingMessage(ur);
|
||||||
|
}
|
||||||
|
|
||||||
|
// private void setIpAddress() {
|
||||||
|
// String address=null;
|
||||||
|
// try {
|
||||||
|
// address=InetAddress.getLocalHost().toString();
|
||||||
|
// } catch (UnknownHostException e) {
|
||||||
|
//
|
||||||
|
// }
|
||||||
|
// logger.info("caller ip: "+address);
|
||||||
|
// if (address!=null) ur.setResourceSpecificProperty("callerIP", address);;
|
||||||
|
// }
|
||||||
|
|
||||||
|
private void setStartTime() {
|
||||||
|
//set start time
|
||||||
|
Calendar startTime = new GregorianCalendar();
|
||||||
|
Date time=startTime.getTime();
|
||||||
|
logger.info("set start time: "+time);
|
||||||
|
try {
|
||||||
|
ur.setStartTime(time);
|
||||||
|
}
|
||||||
|
catch (InvalidValueException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
private void setEndTime() {
|
||||||
|
// set end time
|
||||||
|
Calendar endTime = new GregorianCalendar();
|
||||||
|
Date time=endTime.getTime();
|
||||||
|
logger.info("set end time: "+time);
|
||||||
|
try {
|
||||||
|
ur.setEndTime(time);
|
||||||
|
}
|
||||||
|
catch (InvalidValueException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,8 @@
|
||||||
|
package org.gcube.contentmanager.storageserver.accounting;
|
||||||
|
|
||||||
|
public class ReportConfig {
|
||||||
|
|
||||||
|
/** Report type - used by : Report factory class */
|
||||||
|
public static final int ACCOUNTING_TYPE = 1;
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,24 @@
|
||||||
|
package org.gcube.contentmanager.storageserver.accounting;
|
||||||
|
|
||||||
|
public class ReportException extends Exception {
|
||||||
|
|
||||||
|
private Exception exc = null;
|
||||||
|
|
||||||
|
/** The no-arg constructor */
|
||||||
|
public ReportException() {
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Construct a ReportException with an error message
|
||||||
|
* @param message the error message
|
||||||
|
*/
|
||||||
|
public ReportException(String message) {
|
||||||
|
super(message);
|
||||||
|
}
|
||||||
|
|
||||||
|
public ReportException (Exception e)
|
||||||
|
{
|
||||||
|
this.exc = e;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,37 @@
|
||||||
|
package org.gcube.contentmanager.storageserver.accounting;
|
||||||
|
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
public class ReportFactory {
|
||||||
|
|
||||||
|
final static Logger logger=LoggerFactory.getLogger(ReportFactory.class);
|
||||||
|
/**
|
||||||
|
* <p> Instantiate the class specified by user </p>
|
||||||
|
* @return the Dao class
|
||||||
|
* @throws DAOException
|
||||||
|
*/
|
||||||
|
public static Report getReport(int ReportType) throws ReportException {
|
||||||
|
|
||||||
|
Report report = null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
switch(ReportType) {
|
||||||
|
case ReportConfig.ACCOUNTING_TYPE :
|
||||||
|
report = new ReportAccountingImpl();
|
||||||
|
break;
|
||||||
|
default :
|
||||||
|
throw new ReportException("MyDAOFactory.getDAO: ["+ReportType+"] is an UNKNOWN TYPE !");
|
||||||
|
}
|
||||||
|
logger.trace("ReportFactory.getDao : returning class ["+report.getClass().getName()+"]...");
|
||||||
|
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
throw new ReportException("ReportFactory.getReport: Exception while getting Report type : \n" + e.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.trace("MyReportFactory.getReport : returning class ["+report.getClass().getName()+"]...");
|
||||||
|
return report;
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,46 @@
|
||||||
|
package org.gcube.contentmanager.storageserver.data;
|
||||||
|
|
||||||
|
import java.util.Vector;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import com.mongodb.DBObject;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
public class CubbyHole {
|
||||||
|
|
||||||
|
private Vector<DBObject> requestQueue = new Vector<DBObject>();
|
||||||
|
final static Logger logger=LoggerFactory.getLogger(CubbyHole.class);
|
||||||
|
private boolean available;
|
||||||
|
|
||||||
|
public synchronized DBObject get() {
|
||||||
|
while (requestQueue.size() == 0){
|
||||||
|
try {
|
||||||
|
wait();
|
||||||
|
}
|
||||||
|
catch (InterruptedException e){
|
||||||
|
logger.error("getRequest()", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
DBObject value=requestQueue.remove(0);
|
||||||
|
logger.debug("get element from queue: "+value);
|
||||||
|
available = false;
|
||||||
|
notifyAll();
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
public synchronized void put(DBObject value) {
|
||||||
|
// while (available == true) {
|
||||||
|
// try {
|
||||||
|
// wait();
|
||||||
|
// } catch (InterruptedException e) {
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
logger.debug("put element to queue: "+value);
|
||||||
|
requestQueue.addElement(value);
|
||||||
|
available = true;
|
||||||
|
notifyAll();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,102 @@
|
||||||
|
package org.gcube.contentmanager.storageserver.data;
|
||||||
|
|
||||||
|
|
||||||
|
import java.net.UnknownHostException;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import javax.management.RuntimeErrorException;
|
||||||
|
|
||||||
|
import com.mongodb.BasicDBObject;
|
||||||
|
import com.mongodb.Bytes;
|
||||||
|
import com.mongodb.DB;
|
||||||
|
import com.mongodb.DBCollection;
|
||||||
|
import com.mongodb.DBCursor;
|
||||||
|
import com.mongodb.DBObject;
|
||||||
|
import com.mongodb.MongoClient;
|
||||||
|
import com.mongodb.ServerAddress;
|
||||||
|
import org.bson.types.BSONTimestamp;
|
||||||
|
import org.gcube.contentmanager.storageserver.parse.JsonParser;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
public class ReadingMongoOplog extends Thread{
|
||||||
|
|
||||||
|
final static Logger logger=LoggerFactory.getLogger(ReadingMongoOplog.class);
|
||||||
|
private ServerAddress[] server;
|
||||||
|
private MongoClient mongoClient;
|
||||||
|
private DB local;
|
||||||
|
private DBCollection oplog;
|
||||||
|
private CubbyHole c;
|
||||||
|
private int number;
|
||||||
|
|
||||||
|
public ReadingMongoOplog(List<String> srvs, CubbyHole c, int number){
|
||||||
|
this.c=c;
|
||||||
|
this.number=number;
|
||||||
|
try {
|
||||||
|
if(srvs.size() > 0){
|
||||||
|
server=new ServerAddress[srvs.size()];
|
||||||
|
int i=0;
|
||||||
|
for(String s : srvs){
|
||||||
|
server[i]=new ServerAddress(s);
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
logger.error("MongoDB server not Setted. Please set one or more servers");
|
||||||
|
throw new RuntimeException("MongoDB server not Setted. Please set one or more servers");
|
||||||
|
}
|
||||||
|
} catch (UnknownHostException e) {
|
||||||
|
// TODO Auto-generated catch block
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
init();
|
||||||
|
}
|
||||||
|
|
||||||
|
private void init() {
|
||||||
|
mongoClient = new MongoClient(Arrays.asList(server));//"146.48.123.71"
|
||||||
|
local = mongoClient.getDB("local");
|
||||||
|
oplog = local.getCollection("oplog.rs");
|
||||||
|
}
|
||||||
|
|
||||||
|
public void run() {
|
||||||
|
DBCursor lastCursor = oplog.find().sort(new BasicDBObject("$natural", -1)).limit(1);
|
||||||
|
if (!lastCursor.hasNext()) {
|
||||||
|
logger.error("no oplog!");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
DBObject last = lastCursor.next();
|
||||||
|
BSONTimestamp ts = (BSONTimestamp) last.get("ts");
|
||||||
|
while (true) {
|
||||||
|
logger.debug("starting at ts: " + ts);
|
||||||
|
DBCursor cursor = oplog.find(new BasicDBObject("ts", new BasicDBObject("$gt", ts)));
|
||||||
|
cursor.addOption(Bytes.QUERYOPTION_TAILABLE);
|
||||||
|
cursor.addOption(Bytes.QUERYOPTION_AWAITDATA);
|
||||||
|
while (cursor.hasNext()) {
|
||||||
|
DBObject x = cursor.next();
|
||||||
|
ts = (BSONTimestamp) x.get("ts");
|
||||||
|
if(x.get("o2")!=null){
|
||||||
|
if(x.containsField("o")){
|
||||||
|
// parser.jsonRecordParser(x);
|
||||||
|
c.put(x);
|
||||||
|
logger.info("Producer #" + this.number + " put: " + x);
|
||||||
|
}else{
|
||||||
|
logger.info("operation is not accounted");
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
logger.debug("record discarded: \t"+x);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
Thread.sleep(1000);
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
// TODO Auto-generated catch block
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
|
@ -0,0 +1,103 @@
|
||||||
|
package org.gcube.contentmanager.storageserver.parse;
|
||||||
|
|
||||||
|
import org.bson.types.ObjectId;
|
||||||
|
import org.gcube.contentmanager.storageserver.accounting.Report;
|
||||||
|
import org.gcube.contentmanager.storageserver.accounting.ReportConfig;
|
||||||
|
import org.gcube.contentmanager.storageserver.accounting.ReportException;
|
||||||
|
import org.gcube.contentmanager.storageserver.accounting.ReportFactory;
|
||||||
|
import org.gcube.contentmanager.storageserver.data.CubbyHole;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import com.mongodb.DBObject;
|
||||||
|
|
||||||
|
public class JsonParser extends Thread{
|
||||||
|
|
||||||
|
final static Logger logger=LoggerFactory.getLogger(JsonParser.class);
|
||||||
|
private CubbyHole c;
|
||||||
|
private int number;
|
||||||
|
private Report report;
|
||||||
|
public JsonParser(CubbyHole c, int number){
|
||||||
|
this.c=c;
|
||||||
|
this.number=number;
|
||||||
|
// int the accounting report
|
||||||
|
|
||||||
|
try {
|
||||||
|
init();
|
||||||
|
} catch (ReportException e) {
|
||||||
|
throw new RuntimeException("Accounting report Exception initialization");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void init() throws ReportException{
|
||||||
|
report=new ReportFactory().getReport(ReportConfig.ACCOUNTING_TYPE);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void run() {
|
||||||
|
while(true){
|
||||||
|
DBObject x=c.get();
|
||||||
|
logger.info("Consumer #" + this.number + " got: " + x);
|
||||||
|
DBObject obj=(DBObject)x.get("o");
|
||||||
|
String op=(String) x.get("op");
|
||||||
|
String filename=(String) obj.get("filename");
|
||||||
|
String type=(String) obj.get("type");
|
||||||
|
String name=(String) obj.get("name");
|
||||||
|
String owner=(String) obj.get("owner");
|
||||||
|
ObjectId objectId=(ObjectId)obj.get("_id");
|
||||||
|
String id = objectId.toString();
|
||||||
|
long length=(long) obj.get("length");
|
||||||
|
logger.debug("[recordCheck] operation: "+op+" name: "+name+" type: "+type+" path: "+filename+" length: "+length+" owner: "+owner+ " id: "+id);
|
||||||
|
if((length >0) && (name!=null)){
|
||||||
|
//call to the accounting library
|
||||||
|
String scope=retrieveScopeFromFilename(filename);
|
||||||
|
report.init(owner, scope);
|
||||||
|
report.timeUpdate();
|
||||||
|
String operation=mappingOperationField(op);
|
||||||
|
report.ultimate(owner, null, operation, length+"", filename, id);
|
||||||
|
report.send();
|
||||||
|
logger.debug("[accountingCall] operation: "+op+" name: "+name+" type: "+type+" path: "+filename+" length: "+length+" owner: "+owner);
|
||||||
|
}else{
|
||||||
|
logger.debug("operation is not accounted");
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private String mappingOperationField(String op) {
|
||||||
|
if(op.equals("u")){
|
||||||
|
return "UPDATE";
|
||||||
|
}else if(op.equals("i")){
|
||||||
|
return "INSERT";
|
||||||
|
}else if(op.equals("d")){
|
||||||
|
return "DELETE";
|
||||||
|
}
|
||||||
|
return op;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String retrieveScopeFromFilename(String filename) {
|
||||||
|
String[] split=filename.split("/");
|
||||||
|
if(split.length>0){
|
||||||
|
String scope=null;
|
||||||
|
int i=1;
|
||||||
|
if(split[1].equals("VOLATILE")){
|
||||||
|
i=2;
|
||||||
|
// scope="/"+split[1];
|
||||||
|
// if(split[2] != null && (!split[2].equals("home")) && (!split[2].equals("public"))){
|
||||||
|
// scope=scope+"/"+split[2];
|
||||||
|
// }
|
||||||
|
}
|
||||||
|
scope="/"+split[i];
|
||||||
|
i++;
|
||||||
|
while((!split[i].equals("home")) && (!split[i].equals("public"))){
|
||||||
|
scope=scope+"/"+split[i];
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
logger.info("retieved scope: "+scope);
|
||||||
|
return scope;
|
||||||
|
}else logger.error("Scope bad format: scope not retrieved from string: "+filename);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,27 @@
|
||||||
|
package org.gcube.contentmanager.storageserver.startup;
|
||||||
|
|
||||||
|
import java.util.Arrays;
|
||||||
|
import org.gcube.contentmanager.storageserver.data.CubbyHole;
|
||||||
|
import org.gcube.contentmanager.storageserver.data.ReadingMongoOplog;
|
||||||
|
import org.gcube.contentmanager.storageserver.parse.JsonParser;
|
||||||
|
|
||||||
|
public class Startup {
|
||||||
|
|
||||||
|
public static void main(String[] args) {
|
||||||
|
for (int i=0; i<args.length;i++)
|
||||||
|
System.out.println("param N." +i + ": " + args[i]);
|
||||||
|
|
||||||
|
if(args.length < 1 || args.length > 1){
|
||||||
|
System.out.println("Usage:");
|
||||||
|
System.out.println("\tjava Startup ip \n\n");
|
||||||
|
System.out.println("Example:");
|
||||||
|
System.out.println("\tjava Startup 127.0.0.1 \n\n");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
CubbyHole c = new CubbyHole();
|
||||||
|
ReadingMongoOplog producer=new ReadingMongoOplog( Arrays.asList(args[0]), c, 1 );
|
||||||
|
JsonParser consumer=new JsonParser(c, 1);
|
||||||
|
producer.start();
|
||||||
|
consumer.start();
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue