luca.frosini
/
generate-map-reduce-couchbase
13
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fixed project to publish map reduce 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/private/luca.frosini/GenerateMapReduceCouchbase@152523 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Luca Frosini 2017-08-29 10:02:45 +00:00
parent 26653dae9c
commit 19e5d0b3c7
1 changed files with 80 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

116
src/main/java/org/gcube/dbtest/dbtest.java
View File

@ -15,17 +15,15 @@ import com.couchbase.client.java.CouchbaseCluster;
import com.couchbase.client.java.bucket.BucketManager;
import com.couchbase.client.java.env.CouchbaseEnvironment;
import com.couchbase.client.java.env.DefaultCouchbaseEnvironment;
import com.couchbase.client.java.error.DesignDocumentAlreadyExistsException;
import com.couchbase.client.java.view.DefaultView;
import com.couchbase.client.java.view.DesignDocument;
import com.couchbase.client.java.view.View;
public class dbtest {
protected static final String DEFAULT_DB = "accounting";
protected static final String DEFAULT_DB_SERVICE = "ServiceUsageRecord";
protected static final String DEFAULT_DB_STORAGE = "accounting_storage";
protected static final String DB = "ServiceUsageRecord";
protected static final String URL = "http://couchbase01.d4science.org";
protected static final String PASSWORD = "accounting";
@ -33,56 +31,102 @@ public class dbtest {
protected static final CouchbaseEnvironment ENV = DefaultCouchbaseEnvironment.builder().queryEnabled(true).build();
public static void main(String[] args) throws Exception {
createMapReduceCouchBase();
//createMapReduceCouchBase();
createExtraMapReduceCouchBase();
}
private static void createMapReduceCouchBase() throws Exception {
Cluster cluster = CouchbaseCluster.create(ENV, URL);
Bucket bucket = cluster.openBucket(DEFAULT_DB_SERVICE, PASSWORD);
Bucket bucket = cluster.openBucket(DB, PASSWORD);
BucketManager bucketManager = bucket.bucketManager();
File currentDirectory = new File(".");
File srcDirectory = new File(currentDirectory, "src");
File mainDirectory = new File(srcDirectory, "main");
File resourcesDirectory = new File(mainDirectory, "resources");
File mapReduceDirectory = new File(resourcesDirectory, "MapReduce");
File file = new File(mapReduceDirectory, "ServiceUsageRecord");
File[] filesInDir = file.listFiles();
Arrays.sort(filesInDir);
Map<String, List<View>> views = new HashMap<String, List<View>>();
for (File fs : filesInDir) {
String viewName = fs.getName().split("__")[0];
String mapReduceName = fs.getName();
File mapFile = new File(fs, "map.js");
File reduceFile = new File(fs, "reduce.js");
String mapContent = new String(Files.readAllBytes(Paths.get(mapFile.toURI())));
String reduceContent = new String(Files.readAllBytes(Paths.get(reduceFile.toURI())));
View view = DefaultView.create(mapReduceName, mapContent, reduceContent);
if (!views.containsKey(viewName)){
views.put(viewName, new ArrayList<View>());
}
views.get(viewName).add(view);
}
for (Map.Entry<String, List<View>> entry : views.entrySet()) {
String viewName = entry.getKey();
DesignDocument designDoc = DesignDocument.create(viewName, entry.getValue());
bucketManager.insertDesignDocument(designDoc);
}
cluster.disconnect();
}
private static void createExtraMapReduceCouchBase() throws Exception {
Cluster cluster = CouchbaseCluster.create(ENV, URL);
Bucket bucket = cluster.openBucket(DB, PASSWORD);
BucketManager bucketManager = bucket.bucketManager();
File currentDirectory = new File(".");
File srcDirectory = new File(currentDirectory, "src");
File mainDirectory = new File(srcDirectory, "main");
File resourcesDirectory = new File(mainDirectory, "resources");
File mapReduceDirectory = new File(resourcesDirectory, "MapReduce");
File file = new File(mapReduceDirectory, "ServiceUsageRecord");
System.out.println(file.getAbsolutePath());
File file = new File(mapReduceDirectory, "extraService");
Map<String, List<View>> views = new HashMap<String, List<View>>();
File[] filesInDir = file.listFiles();
Arrays.sort(filesInDir);
Map<String, List<View>> maps = new HashMap<String, List<View>>();
for (File fs : filesInDir) {
String nameMapReduce = fs.getName();
String nameViewCustom = fs.getName().split("__")[0];
if (!fs.getName().equals("Invalid")) {
String stringMap = new String(Files.readAllBytes(Paths.get(fs.toString() + "/map.js")));
String stringReduce = new String(Files.readAllBytes(Paths.get(fs.toString() + "/reduce.js")));
if (!maps.containsKey(nameViewCustom)){
maps.put(nameViewCustom, new ArrayList<View>());
}
for (File f : file.listFiles()){
String viewName = f.getName();
if (!views.containsKey(viewName)){
views.put(viewName, new ArrayList<View>());
}
for(File fs : f.listFiles()){
String mapReduceName = fs.getName();
File mapFile = new File(fs, "map.js");
File reduceFile = new File(fs, "reduce.js");
String mapContent = new String(Files.readAllBytes(Paths.get(mapFile.toURI())));
String reduceContent = new String(Files.readAllBytes(Paths.get(reduceFile.toURI())));
System.out.println("add view:" + nameViewCustom + " and map:" + nameMapReduce);
View view = DefaultView.create(nameMapReduce, stringMap, stringReduce);
maps.get(nameViewCustom).add(view);
View view = DefaultView.create(mapReduceName, mapContent, reduceContent);
views.get(viewName).add(view);
}
}
for (Map.Entry<String, List<View>> entry : maps.entrySet()) {
String key = entry.getKey();
System.out.println("key" + key);
DesignDocument designDoc = DesignDocument.create(key, entry.getValue());
bucketManager.insertDesignDocument(designDoc);
for (Map.Entry<String, List<View>> entry : views.entrySet()) {
String viewName = entry.getKey();
DesignDocument designDoc = DesignDocument.create(viewName, entry.getValue());
try{
bucketManager.insertDesignDocument(designDoc);
}catch (DesignDocumentAlreadyExistsException e) {
System.out.println(viewName + " Already Exists");
}
}
cluster.disconnect();
}
}