added dnet-data-services, includes mdstore service. Imported also cnr-inspector (dnet-core-components) and cnr-enabling-inspector (dnet-information-service)

dnet-core-components/src/main/java/eu/dnetlib/enabling/inspector/AbstractInspectorController.java

@@ -0,0 +1,85 @@
+package eu.dnetlib.enabling.inspector;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+import javax.annotation.PostConstruct;
+import javax.annotation.Resource;
+import javax.servlet.http.HttpServletRequest;
+
+import org.springframework.web.bind.annotation.ModelAttribute;
+
+import com.google.common.collect.Lists;
+
+import eu.dnetlib.miscutils.collections.MappedCollection;
+import eu.dnetlib.miscutils.functional.UnaryFunction;
+
+/**
+ * Common stuff for the inspector controllers.
+ * 
+ * @author marko
+ * 
+ */
+public abstract class AbstractInspectorController { // NOPMD
+	/**
+	 * entry point groups
+	 */
+	@Resource
+	List<EntryPointDescriptorGroup> entryPointsGroups;
+
+	/**
+	 * sort entry point groups according to the group name.
+	 */
+	@PostConstruct
+	public void sortEntryPointGroups() {
+		Collections.sort(entryPointsGroups, new Comparator<EntryPointDescriptorGroup>() {
+
+			@Override
+			public int compare(EntryPointDescriptorGroup o1, EntryPointDescriptorGroup o2) {
+				return o1.getName().compareTo(o2.getName());
+			}
+		});
+	}
+
+	/**
+	 * common base url for all inspector pages.
+	 * 
+	 * @param request
+	 *            http request
+	 * @return base url
+	 */
+	@ModelAttribute("baseUrl")
+	public String baseUrl(final HttpServletRequest request) {
+		return request.getContextPath() + "/mvc/inspector";
+	}
+
+	@ModelAttribute("entryPointGroups")
+	public List<EntryPointDescriptorGroup> entryPointGroups() {
+		return entryPointsGroups;
+	}
+	/**
+	 * Obtain a list of entry points, ordered by groups.
+	 * 
+	 * @return
+	 */
+	@ModelAttribute("entryPoints")
+	public List<EntryPointDescriptorModel> entryPoints(final HttpServletRequest request) {
+		final String currentRelativeUrl = request.getPathInfo().replaceAll("/inspector/(.*\\.do).*", "$1");
+		
+		ArrayList<EntryPointDescriptorModel> all = new ArrayList<EntryPointDescriptorModel>();
+
+		UnaryFunction<EntryPointDescriptorModel, EntryPointDescriptor> mapper = new UnaryFunction<EntryPointDescriptorModel, EntryPointDescriptor>() {
+
+			@Override
+			public EntryPointDescriptorModel evaluate(EntryPointDescriptor arg) {
+				return new EntryPointDescriptorModel(arg.getName(), arg.getRelativeUrl(), currentRelativeUrl.equals(arg.getRelativeUrl()), arg.isHiddenAsDefault());
+			}
+		};
+
+		for (EntryPointDescriptorGroup group : entryPointsGroups)
+			all.addAll(Lists.newArrayList(new MappedCollection<EntryPointDescriptorModel, EntryPointDescriptor>(group.getDescriptors(), mapper)));
+		return all;
+	}
+}

dnet-core-components/src/main/java/eu/dnetlib/enabling/inspector/DashboardController.java

@@ -0,0 +1,19 @@
+package eu.dnetlib.enabling.inspector;
+
+import org.springframework.stereotype.Controller;
+import org.springframework.ui.Model;
+import org.springframework.web.bind.annotation.RequestMapping;
+
+@Controller
+public class DashboardController extends AbstractInspectorController {
+	
+	@RequestMapping("/inspector")
+	public String dashboard(final Model model) {
+		
+		return "inspector/dashboard";
+	}
+	
+
+
+	
+}

dnet-core-components/src/main/java/eu/dnetlib/enabling/inspector/DefaultStaticFilesController.java

@@ -0,0 +1,44 @@
+package eu.dnetlib.enabling.inspector;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.servlet.ServletResponse;
+
+import org.apache.commons.io.IOUtils;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+
+@Controller
+public class DefaultStaticFilesController {
+
+	private final Map<String, String> extToMime = new HashMap<String, String>();
+
+	public DefaultStaticFilesController() {
+		extToMime.put("js", "text/javascript");
+		extToMime.put("css", "text/css");
+		extToMime.put("png", "image/png");
+		extToMime.put("jpg", "image/jpeg");
+	}
+
+	@RequestMapping(value = "/inspector/static.do")
+	public void stat(ServletResponse response, OutputStream out, @RequestParam(value = "src", required = true) final String src) throws Exception {
+		renderStatic(response, out, src);
+	}
+
+	protected void renderStatic(final ServletResponse response, final OutputStream output, final String name) throws IOException {
+		String ext = name.substring(name.lastIndexOf('.') + 1, name.length());
+
+		renderStatic(response, output, getClass().getResourceAsStream("/eu/dnetlib/enabling/views/inspector/" + name), extToMime.get(ext));
+	}
+
+	protected void renderStatic(final ServletResponse response, final OutputStream output, final InputStream input, final String contentType)
+			throws IOException {
+		response.setContentType(contentType);
+		IOUtils.copy(input, output);
+	}
+}

dnet-core-components/src/main/java/eu/dnetlib/enabling/inspector/EntryPointDescriptor.java

@@ -0,0 +1,41 @@
+package eu.dnetlib.enabling.inspector;
+
+/**
+ * Each module can declare beans implementing this interface, defining entry points to inspector controllers defined in
+ * the module.
+ *
+ * <p>
+ * The master template will join all these and create the navigation bar
+ * </p>
+ *
+ * <p>
+ * Entry point descriptors are grouped in InspectorEntryPointGroups, which contain an ordered list of
+ * InspectorEntryPointDescriptors
+ * </p>
+ *
+ * @author marko
+ *
+ */
+public interface EntryPointDescriptor {
+	/**
+	 * entry point url relative to the baseUri.
+	 *
+	 * @return url
+	 */
+	String getRelativeUrl();
+
+	/**
+	 * Get entry point name, as displayed in the web page.
+	 *
+	 * @return entry point name
+	 */
+	String getName();
+	
+	/**
+	 * Return visible visible as default
+	 *
+	 * @return visible
+	 */
+	boolean isHiddenAsDefault();
+	
+}

dnet-core-components/src/main/java/eu/dnetlib/enabling/inspector/EntryPointDescriptorGroup.java

@@ -0,0 +1,26 @@
+package eu.dnetlib.enabling.inspector;
+
+import java.util.Collection;
+
+/**
+ * Entry points are grouped in entry point groups. The master template will probably order groups by name, the order
+ * within a group is decided by implementors of this interface.
+ *
+ * @author marko
+ *
+ */
+public interface EntryPointDescriptorGroup {
+	/**
+	 * group name.
+	 *
+	 * @return name
+	 */
+	String getName();
+
+	/**
+	 * descriptors.
+	 *
+	 * @return descriptors
+	 */
+	Collection<EntryPointDescriptor> getDescriptors();
+}

dnet-core-components/src/main/java/eu/dnetlib/enabling/inspector/EntryPointDescriptorModel.java

@@ -0,0 +1,81 @@
+package eu.dnetlib.enabling.inspector;
+
+/**
+ * Used as MVC model for actual entry point descriptor. It contains also a boolean if it's the current page.
+ * 
+ * @author marko
+ * 
+ */
+public class EntryPointDescriptorModel implements EntryPointDescriptor {
+	
+	/**
+	 * name.
+	 */
+	private String name;
+
+	/**
+	 * relative url.
+	 */
+	private String relativeUrl;
+	
+	private boolean current;
+	
+	/**
+	 * Default value for visibility.
+	 */
+	private boolean hiddenAsDefault = false;
+
+	/**
+	 * Compatibility for other users of EntryPointDescriptorModel.
+	 * 
+	 * @param name
+	 * @param relativeUrl
+	 * @param current
+	 */
+	public EntryPointDescriptorModel(final String name, final String relativeUrl, final boolean current) {
+			this(name, relativeUrl, current, false);
+	}
+	
+	public EntryPointDescriptorModel(final String name, final String relativeUrl, final boolean current, final boolean hiddenAsDefault) {
+		super();
+		this.name = name;
+		this.relativeUrl = relativeUrl;
+		this.current = current;
+		this.hiddenAsDefault = hiddenAsDefault;
+	}
+	
+	@Override
+	public String getName() {
+		return name;
+	}
+
+	public void setName(final String name) {
+		this.name = name;
+	}
+	
+	@Override
+	public String getRelativeUrl() {
+		return relativeUrl;
+	}
+
+	public void setRelativeUrl(final String relativeUrl) {
+		this.relativeUrl = relativeUrl;
+	}
+
+	public boolean isCurrent() {
+		return current;
+	}
+
+	public void setCurrent(final boolean current) {
+		this.current = current;
+	}
+
+	@Override
+	public boolean isHiddenAsDefault() {
+		return hiddenAsDefault;
+	}
+
+	public void setHiddenAsDefault(boolean hiddenAsDefault) {
+		this.hiddenAsDefault = hiddenAsDefault;
+	}
+}

dnet-core-components/src/main/java/eu/dnetlib/enabling/inspector/StaticEntryPointDescriptor.java

@@ -0,0 +1,53 @@
+package eu.dnetlib.enabling.inspector;
+
+/**
+ * Entry point descriptor.
+ *
+ * @author marko
+ *
+ */
+public class StaticEntryPointDescriptor implements EntryPointDescriptor {
+
+	/**
+	 * name.
+	 */
+	private String name;
+
+	/**
+	 * relative url.
+	 */
+	private String relativeUrl;
+	
+	/**
+	 * Default value for visibility.
+	 */
+	private boolean hiddenAsDefault = false;
+	
+	@Override
+	public String getRelativeUrl() {
+		return relativeUrl;
+	}
+
+	public void setRelativeUrl(String relativeUrl) {
+		this.relativeUrl = relativeUrl;
+	}
+
+	@Override
+	public String getName() {
+		return name;
+	}
+
+	public void setName(String name) {
+		this.name = name;
+	}
+
+	public void setHiddenAsDefault(boolean hiddenAsDefault) {
+		this.hiddenAsDefault = hiddenAsDefault;
+	}
+
+	@Override
+	public boolean isHiddenAsDefault() {
+		return hiddenAsDefault;
+	}
+
+}

dnet-core-components/src/main/java/eu/dnetlib/enabling/inspector/StaticEntryPointDescriptorGroup.java

@@ -0,0 +1,42 @@
+package eu.dnetlib.enabling.inspector;
+
+import java.util.Collection;
+
+/**
+ * Declare a spring bean and put a list of entry points for each module defining.
+ *
+ * @author marko
+ *
+ */
+public class StaticEntryPointDescriptorGroup implements EntryPointDescriptorGroup {
+
+	/**
+	 * entry point descriptors.
+	 */
+	private Collection<EntryPointDescriptor> descriptors;
+
+	/**
+	 * group name.
+	 */
+	private String name;
+
+	@Override
+	public Collection<EntryPointDescriptor> getDescriptors() {
+		return descriptors;
+	}
+
+	public void setDescriptors(final Collection<EntryPointDescriptor> descriptors) {
+		this.descriptors = descriptors;
+	}
+
+	@Override
+	public String getName() {
+		return name;
+	}
+
+	public void setName(final String name) {
+		this.name = name;
+	}
+
+
+}

dnet-core-components/src/main/resources/eu/dnetlib/enabling/inspector/webContext-inspector.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns="http://www.springframework.org/schema/beans"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:p="http://www.springframework.org/schema/p"
+	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
+
+	<bean id="templateViewResolver"
+		class="eu.dnetlib.springutils.stringtemplate.StringTemplateViewResolver"
+		p:cache="${inspector.template.cache.enabled}" p:templateGroup-ref="templateGroup" p:contentType="text/html;charset=UTF-8" />
+
+	<bean id="templateGroup" class="eu.dnetlib.springutils.stringtemplate.ClassPathStringTemplateGroup"
+		p:refreshInterval="0" p:package="eu.dnetlib.enabling.views">
+		<constructor-arg type="java.lang.String" value="inspector" />
+	</bean>
+	
+	<bean id="dashboardInspectorGroup"
+		class="eu.dnetlib.enabling.inspector.StaticEntryPointDescriptorGroup"
+		p:name="dashboard">
+		<property name="descriptors">
+			<list>
+				<bean class="eu.dnetlib.enabling.inspector.StaticEntryPointDescriptor"
+					p:name="dashboard" p:relativeUrl="../inspector" />
+			</list>
+		</property>
+	</bean>
+	
+	
+</beans>

dnet-core-components/src/main/resources/eu/dnetlib/enabling/views/inspector/dashboard.st

@@ -0,0 +1,42 @@
+$inspector/master(it={
+  <h2>Inspector Dashboard</h2>
+  
+  <style type="text/css">
+#grouplist {
+  -moz-column-count: 3;
+	-moz-column-gap: 20px;
+	-webkit-column-count: 3;
+	-webkit-column-gap: 20px;
+	column-count: 3;
+	column-gap: 20px;
+}
+
+#grouplist > li  {
+  border: 1px solid #ccc;
+  padding: 8px;
+  margin: 8px;
+}
+
+#grouplist > li  {
+  -moz-column-break-inside: avoid;
+  -webkit-column-break-inside: avoid;
+  break-inside: avoid-column;
+}
+  </style>
+  
+  <p>Installed inspector panels</p>
+  <ul id="grouplist">
+    $entryPointGroups:{
+      <li>
+       <div class="box">
+       <span>$it.name$</span>
+        <ul>
+         $it.descriptors:{
+          <li><a href="$baseUrl$/$it.relativeUrl$">$it.name$</a></li>
+         }$
+        </ul>
+        </div>
+      </li>
+    }$
+  </ul>
+})$

dnet-core-components/src/main/resources/eu/dnetlib/enabling/views/inspector/inspector.css

@@ -0,0 +1,145 @@
+html { overflow: -moz-scrollbars-vertical !important; }
+body {
+  padding: 2em 1em 2em 70px;
+  margin: 0;
+  font-family: sans-serif;
+  color: black;
+  background: white;
+  background-position: top left;
+  background-attachment: fixed;
+  background-repeat: no-repeat;
+}
+:link { color: #00C; background: transparent }
+:visited { color: #609; background: transparent }
+a:active { color: #C00; background: transparent }
+
+a:link img, a:visited img { border-style: none } /* no border on img links */
+
+a img { color: white; }        /* trick to hide the border in Netscape 4 */
+@media all {                   /* hide the next rule from Netscape 4 */
+  a img { color: inherit; }    /* undo the color change above */
+}
+
+th, td { /* ns 4 */
+  font-family: sans-serif;
+}
+
+h1, h2, h3, h4, h5, h6 { text-align: left }
+#content { margin-top: 8px; }
+
+/* background should be transparent, but WebTV has a bug */
+h1, h2, h3 { color: #005A9C; background: white }
+h1 { font: 170% sans-serif }
+h2 { font: 140% sans-serif }
+h3 { font: 120% sans-serif }
+h4 { font: bold 100% sans-serif }
+h5 { font: italic 100% sans-serif }
+h6 { font: small-caps 100% sans-serif }
+
+/* specific */
+
+#nav {
+  position: absolute;
+  top: 0px;
+  right: 0px;
+}
+
+#nav li {
+  display: inline;
+}
+
+#menuConfig {
+  display: none;
+  position: absolute;
+  background-color: white;
+  top: 220px;
+  left: 0;
+  right: 0;
+  padding: 8px;
+  
+  box-shadow: 10px 10px 20px #888;
+  -webkit-box-shadow: 10px 10px 20px #888;
+  -moz-box-shadow: 10px 10px 20px #888;
+  
+  margin-left: auto;
+  margin-right: auto;
+  width: 600px;
+ 
+  border: 1px solid #ccc;
+}
+
+#menuConfig ul {
+  -moz-column-count: 3;
+	-moz-column-gap: 20px;
+	-webkit-column-count: 3;
+	-webkit-column-gap: 20px;
+	column-count: 3;
+	column-gap: 20px;
+}
+
+#menuConfig ul li {
+  padding: 4px;
+} 
+
+
+      #nav ul {   
+        margin: 0px;
+        padding: 0px;
+        list-style: none;
+      }
+
+      #nav ul li {
+        background-color: #005A9C;
+      }
+  
+      #nav ul li,
+      #nav ul li a {
+        float: left;
+        display: block;
+      }
+  
+      #nav ul li a {
+        margin: 6px;
+        padding: 5px;
+        font-size: 14px;
+        color: #fff;
+        text-decoration: none;
+      }
+      #nav ul li a:hover,
+      #nav ul li a.selected {
+        background-color: #fff;
+        color: #272727;
+      }
+
+      #nav .menuPerspective {
+     	background-color: #ccdddd;
+		color: #003366;
+      }
+      
+      #nav .menuPerspective.selected {
+     	background-color: #ccffaa;
+      }
+
+/* common css */
+
+pre.profile {
+  border: 1px solid #ccccff;
+  padding: 4px;
+  background-color: #ffffe0;
+}
+
+textarea.profile {
+  width: 100%;
+  height: 60em;
+  
+  border: 1px solid #b0b0ff;
+  padding: 4px;
+}
+
+span.disabled {
+ color: red;
+}
+
+span.enabled {
+ color: green;
+}

dnet-core-components/src/main/resources/eu/dnetlib/enabling/views/inspector/inspector.js

@@ -0,0 +1,115 @@
+var defaultHiddenItems=",";
+
+$(function () {
+    $("#nav ul li a").corner();
+    $('[autofocus]').autofocus();
+});
+
+function setDefaultHiddenItems(list) {
+	defaultHiddenItems=list;
+}
+
+function currentPerspective() {
+	p = $.jStorage.get('SELECTED_PERSPECTIVE', '_NOT_FOUND_');
+	if (p=='_NOT_FOUND_') {
+		$.jStorage.set('SELECTED_PERSPECTIVE', 'P1');
+		return 'P1';
+	}
+	return p;
+}
+
+function removeFromPerspective(item) {
+	curr = currentPerspective();
+	list = retrieveHiddenList(curr);
+	if(list.indexOf(","+item+",") == -1) {
+		list += item + ",";
+		$.jStorage.set(curr, list);
+	}
+}
+
+function retrieveHiddenList(p) {
+	list = $.jStorage.get(p, '_NOT_FOUND_');
+	if (list=='_NOT_FOUND_') {
+		$.jStorage.set(p, defaultHiddenItems);
+		return defaultHiddenItems;
+	}
+	return list;
+}
+
+function addToPerspective(item) {
+	curr = currentPerspective();
+	list = retrieveHiddenList(curr);
+	if(list.indexOf(","+item+",") > -1) {
+		list = list.replace(","+item+",", ",");
+		$.jStorage.set(curr, list);
+	}
+}
+
+$(document).ready(function () {
+    updateMenu(currentPerspective());
+});
+
+function perspectiveItemName(el) {
+	return el.name.replace("perspective_check_", "");
+}
+
+function configPerspectives() {	
+	if ($('#menuConfig').is(':visible')) {
+		$('#menuConfig').hide();
+	} else {
+		updatePerspectiveCheckboxes();
+		$('#menuConfig').show();
+	}
+}
+
+function updatePerspectiveCheckboxes() {
+	var blacklist = retrieveHiddenList(currentPerspective());
+	
+	$('#menuConfig input').each(function (ix, el) {
+		var name = perspectiveItemName(el);
+		if(blacklist.indexOf(","+name+",") > -1)
+			el.checked=false;
+		else
+			el.checked=true;
+	});
+}
+
+function changePerspectiveItem(el) {
+	var name = perspectiveItemName(el);
+
+	//Uncomment for debug
+	//console.log("changing " + name);
+	
+	if(!el.checked) {
+		removeFromPerspective(name);
+	} else {
+		addToPerspective(name);
+	}
+	updateMenu(currentPerspective());
+}
+
+function resetJStorage(p) {
+	if (confirm("Reset Local Storage?")) {
+        $.jStorage.flush();
+        window.location.reload();
+    }
+}
+
+function updateMenu(p) {
+    list = retrieveHiddenList(p);     
+    $(".menuItem").each(function () {
+        if ((new RegExp("," + $(this).text() + ",")).test(list)) {
+            $(this).hide();
+        } else {
+            $(this).show();
+        }
+    });
+    $(".menuPerspective").removeClass("selected");
+    $("#menu_" + p).addClass("selected");
+}
+
+function changePerspective(p) {
+    $.jStorage.set("SELECTED_PERSPECTIVE", p);
+    updatePerspectiveCheckboxes();
+    updateMenu(p);
+}

dnet-core-components/src/main/resources/eu/dnetlib/enabling/views/inspector/jquery-ui.min.js

@@ -0,0 +1,141 @@
+/*!
+ * jQuery UI 1.8.5
+ *
+ * Copyright 2010, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI
+ */
+(function(c,j){function k(a){return!c(a).parents().andSelf().filter(function(){return c.curCSS(this,"visibility")==="hidden"||c.expr.filters.hidden(this)}).length}c.ui=c.ui||{};if(!c.ui.version){c.extend(c.ui,{version:"1.8.5",keyCode:{ALT:18,BACKSPACE:8,CAPS_LOCK:20,COMMA:188,COMMAND:91,COMMAND_LEFT:91,COMMAND_RIGHT:93,CONTROL:17,DELETE:46,DOWN:40,END:35,ENTER:13,ESCAPE:27,HOME:36,INSERT:45,LEFT:37,MENU:93,NUMPAD_ADD:107,NUMPAD_DECIMAL:110,NUMPAD_DIVIDE:111,NUMPAD_ENTER:108,NUMPAD_MULTIPLY:106,
+NUMPAD_SUBTRACT:109,PAGE_DOWN:34,PAGE_UP:33,PERIOD:190,RIGHT:39,SHIFT:16,SPACE:32,TAB:9,UP:38,WINDOWS:91}});c.fn.extend({_focus:c.fn.focus,focus:function(a,b){return typeof a==="number"?this.each(function(){var d=this;setTimeout(function(){c(d).focus();b&&b.call(d)},a)}):this._focus.apply(this,arguments)},scrollParent:function(){var a;a=c.browser.msie&&/(static|relative)/.test(this.css("position"))||/absolute/.test(this.css("position"))?this.parents().filter(function(){return/(relative|absolute|fixed)/.test(c.curCSS(this,
+"position",1))&&/(auto|scroll)/.test(c.curCSS(this,"overflow",1)+c.curCSS(this,"overflow-y",1)+c.curCSS(this,"overflow-x",1))}).eq(0):this.parents().filter(function(){return/(auto|scroll)/.test(c.curCSS(this,"overflow",1)+c.curCSS(this,"overflow-y",1)+c.curCSS(this,"overflow-x",1))}).eq(0);return/fixed/.test(this.css("position"))||!a.length?c(document):a},zIndex:function(a){if(a!==j)return this.css("zIndex",a);if(this.length){a=c(this[0]);for(var b;a.length&&a[0]!==document;){b=a.css("position");
+if(b==="absolute"||b==="relative"||b==="fixed"){b=parseInt(a.css("zIndex"));if(!isNaN(b)&&b!=0)return b}a=a.parent()}}return 0},disableSelection:function(){return this.bind("mousedown.ui-disableSelection selectstart.ui-disableSelection",function(a){a.preventDefault()})},enableSelection:function(){return this.unbind(".ui-disableSelection")}});c.each(["Width","Height"],function(a,b){function d(f,g,l,m){c.each(e,function(){g-=parseFloat(c.curCSS(f,"padding"+this,true))||0;if(l)g-=parseFloat(c.curCSS(f,
+"border"+this+"Width",true))||0;if(m)g-=parseFloat(c.curCSS(f,"margin"+this,true))||0});return g}var e=b==="Width"?["Left","Right"]:["Top","Bottom"],h=b.toLowerCase(),i={innerWidth:c.fn.innerWidth,innerHeight:c.fn.innerHeight,outerWidth:c.fn.outerWidth,outerHeight:c.fn.outerHeight};c.fn["inner"+b]=function(f){if(f===j)return i["inner"+b].call(this);return this.each(function(){c.style(this,h,d(this,f)+"px")})};c.fn["outer"+b]=function(f,g){if(typeof f!=="number")return i["outer"+b].call(this,f);return this.each(function(){c.style(this,
+h,d(this,f,true,g)+"px")})}});c.extend(c.expr[":"],{data:function(a,b,d){return!!c.data(a,d[3])},focusable:function(a){var b=a.nodeName.toLowerCase(),d=c.attr(a,"tabindex");if("area"===b){b=a.parentNode;d=b.name;if(!a.href||!d||b.nodeName.toLowerCase()!=="map")return false;a=c("img[usemap=#"+d+"]")[0];return!!a&&k(a)}return(/input|select|textarea|button|object/.test(b)?!a.disabled:"a"==b?a.href||!isNaN(d):!isNaN(d))&&k(a)},tabbable:function(a){var b=c.attr(a,"tabindex");return(isNaN(b)||b>=0)&&c(a).is(":focusable")}});
+c(function(){var a=document.createElement("div"),b=document.body;c.extend(a.style,{minHeight:"100px",height:"auto",padding:0,borderWidth:0});c.support.minHeight=b.appendChild(a).offsetHeight===100;b.removeChild(a).style.display="none"});c.extend(c.ui,{plugin:{add:function(a,b,d){a=c.ui[a].prototype;for(var e in d){a.plugins[e]=a.plugins[e]||[];a.plugins[e].push([b,d[e]])}},call:function(a,b,d){if((b=a.plugins[b])&&a.element[0].parentNode)for(var e=0;e<b.length;e++)a.options[b[e][0]]&&b[e][1].apply(a.element,
+d)}},contains:function(a,b){return document.compareDocumentPosition?a.compareDocumentPosition(b)&16:a!==b&&a.contains(b)},hasScroll:function(a,b){if(c(a).css("overflow")==="hidden")return false;b=b&&b==="left"?"scrollLeft":"scrollTop";var d=false;if(a[b]>0)return true;a[b]=1;d=a[b]>0;a[b]=0;return d},isOverAxis:function(a,b,d){return a>b&&a<b+d},isOver:function(a,b,d,e,h,i){return c.ui.isOverAxis(a,d,h)&&c.ui.isOverAxis(b,e,i)}})}})(jQuery);
+;/*!
+ * jQuery UI Widget 1.8.5
+ *
+ * Copyright 2010, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Widget
+ */
+(function(b,j){if(b.cleanData){var k=b.cleanData;b.cleanData=function(a){for(var c=0,d;(d=a[c])!=null;c++)b(d).triggerHandler("remove");k(a)}}else{var l=b.fn.remove;b.fn.remove=function(a,c){return this.each(function(){if(!c)if(!a||b.filter(a,[this]).length)b("*",this).add([this]).each(function(){b(this).triggerHandler("remove")});return l.call(b(this),a,c)})}}b.widget=function(a,c,d){var e=a.split(".")[0],f;a=a.split(".")[1];f=e+"-"+a;if(!d){d=c;c=b.Widget}b.expr[":"][f]=function(h){return!!b.data(h,
+a)};b[e]=b[e]||{};b[e][a]=function(h,g){arguments.length&&this._createWidget(h,g)};c=new c;c.options=b.extend(true,{},c.options);b[e][a].prototype=b.extend(true,c,{namespace:e,widgetName:a,widgetEventPrefix:b[e][a].prototype.widgetEventPrefix||a,widgetBaseClass:f},d);b.widget.bridge(a,b[e][a])};b.widget.bridge=function(a,c){b.fn[a]=function(d){var e=typeof d==="string",f=Array.prototype.slice.call(arguments,1),h=this;d=!e&&f.length?b.extend.apply(null,[true,d].concat(f)):d;if(e&&d.substring(0,1)===
+"_")return h;e?this.each(function(){var g=b.data(this,a);if(!g)throw"cannot call methods on "+a+" prior to initialization; attempted to call method '"+d+"'";if(!b.isFunction(g[d]))throw"no such method '"+d+"' for "+a+" widget instance";var i=g[d].apply(g,f);if(i!==g&&i!==j){h=i;return false}}):this.each(function(){var g=b.data(this,a);g?g.option(d||{})._init():b.data(this,a,new c(d,this))});return h}};b.Widget=function(a,c){arguments.length&&this._createWidget(a,c)};b.Widget.prototype={widgetName:"widget",
+widgetEventPrefix:"",options:{disabled:false},_createWidget:function(a,c){b.data(c,this.widgetName,this);this.element=b(c);this.options=b.extend(true,{},this.options,b.metadata&&b.metadata.get(c)[this.widgetName],a);var d=this;this.element.bind("remove."+this.widgetName,function(){d.destroy()});this._create();this._init()},_create:function(){},_init:function(){},destroy:function(){this.element.unbind("."+this.widgetName).removeData(this.widgetName);this.widget().unbind("."+this.widgetName).removeAttr("aria-disabled").removeClass(this.widgetBaseClass+
+"-disabled ui-state-disabled")},widget:function(){return this.element},option:function(a,c){var d=a,e=this;if(arguments.length===0)return b.extend({},e.options);if(typeof a==="string"){if(c===j)return this.options[a];d={};d[a]=c}b.each(d,function(f,h){e._setOption(f,h)});return e},_setOption:function(a,c){this.options[a]=c;if(a==="disabled")this.widget()[c?"addClass":"removeClass"](this.widgetBaseClass+"-disabled ui-state-disabled").attr("aria-disabled",c);return this},enable:function(){return this._setOption("disabled",
+false)},disable:function(){return this._setOption("disabled",true)},_trigger:function(a,c,d){var e=this.options[a];c=b.Event(c);c.type=(a===this.widgetEventPrefix?a:this.widgetEventPrefix+a).toLowerCase();d=d||{};if(c.originalEvent){a=b.event.props.length;for(var f;a;){f=b.event.props[--a];c[f]=c.originalEvent[f]}}this.element.trigger(c,d);return!(b.isFunction(e)&&e.call(this.element[0],c,d)===false||c.isDefaultPrevented())}}})(jQuery);
+;/*!
+ * jQuery UI Mouse 1.8.5
+ *
+ * Copyright 2010, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Mouse
+ *
+ * Depends:
+ *	jquery.ui.widget.js
+ */
+(function(c){c.widget("ui.mouse",{options:{cancel:":input,option",distance:1,delay:0},_mouseInit:function(){var a=this;this.element.bind("mousedown."+this.widgetName,function(b){return a._mouseDown(b)}).bind("click."+this.widgetName,function(b){if(a._preventClickEvent){a._preventClickEvent=false;b.stopImmediatePropagation();return false}});this.started=false},_mouseDestroy:function(){this.element.unbind("."+this.widgetName)},_mouseDown:function(a){a.originalEvent=a.originalEvent||{};if(!a.originalEvent.mouseHandled){this._mouseStarted&&
+this._mouseUp(a);this._mouseDownEvent=a;var b=this,e=a.which==1,f=typeof this.options.cancel=="string"?c(a.target).parents().add(a.target).filter(this.options.cancel).length:false;if(!e||f||!this._mouseCapture(a))return true;this.mouseDelayMet=!this.options.delay;if(!this.mouseDelayMet)this._mouseDelayTimer=setTimeout(function(){b.mouseDelayMet=true},this.options.delay);if(this._mouseDistanceMet(a)&&this._mouseDelayMet(a)){this._mouseStarted=this._mouseStart(a)!==false;if(!this._mouseStarted){a.preventDefault();
+return true}}this._mouseMoveDelegate=function(d){return b._mouseMove(d)};this._mouseUpDelegate=function(d){return b._mouseUp(d)};c(document).bind("mousemove."+this.widgetName,this._mouseMoveDelegate).bind("mouseup."+this.widgetName,this._mouseUpDelegate);c.browser.safari||a.preventDefault();return a.originalEvent.mouseHandled=true}},_mouseMove:function(a){if(c.browser.msie&&!a.button)return this._mouseUp(a);if(this._mouseStarted){this._mouseDrag(a);return a.preventDefault()}if(this._mouseDistanceMet(a)&&
+this._mouseDelayMet(a))(this._mouseStarted=this._mouseStart(this._mouseDownEvent,a)!==false)?this._mouseDrag(a):this._mouseUp(a);return!this._mouseStarted},_mouseUp:function(a){c(document).unbind("mousemove."+this.widgetName,this._mouseMoveDelegate).unbind("mouseup."+this.widgetName,this._mouseUpDelegate);if(this._mouseStarted){this._mouseStarted=false;this._preventClickEvent=a.target==this._mouseDownEvent.target;this._mouseStop(a)}return false},_mouseDistanceMet:function(a){return Math.max(Math.abs(this._mouseDownEvent.pageX-
+a.pageX),Math.abs(this._mouseDownEvent.pageY-a.pageY))>=this.options.distance},_mouseDelayMet:function(){return this.mouseDelayMet},_mouseStart:function(){},_mouseDrag:function(){},_mouseStop:function(){},_mouseCapture:function(){return true}})})(jQuery);
+;/*
+ * jQuery UI Position 1.8.5
+ *
+ * Copyright 2010, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Position
+ */
+(function(c){c.ui=c.ui||{};var n=/left|center|right/,o=/top|center|bottom/,t=c.fn.position,u=c.fn.offset;c.fn.position=function(b){if(!b||!b.of)return t.apply(this,arguments);b=c.extend({},b);var a=c(b.of),d=a[0],g=(b.collision||"flip").split(" "),e=b.offset?b.offset.split(" "):[0,0],h,k,j;if(d.nodeType===9){h=a.width();k=a.height();j={top:0,left:0}}else if(d.scrollTo&&d.document){h=a.width();k=a.height();j={top:a.scrollTop(),left:a.scrollLeft()}}else if(d.preventDefault){b.at="left top";h=k=0;j=
+{top:b.of.pageY,left:b.of.pageX}}else{h=a.outerWidth();k=a.outerHeight();j=a.offset()}c.each(["my","at"],function(){var f=(b[this]||"").split(" ");if(f.length===1)f=n.test(f[0])?f.concat(["center"]):o.test(f[0])?["center"].concat(f):["center","center"];f[0]=n.test(f[0])?f[0]:"center";f[1]=o.test(f[1])?f[1]:"center";b[this]=f});if(g.length===1)g[1]=g[0];e[0]=parseInt(e[0],10)||0;if(e.length===1)e[1]=e[0];e[1]=parseInt(e[1],10)||0;if(b.at[0]==="right")j.left+=h;else if(b.at[0]==="center")j.left+=h/
+2;if(b.at[1]==="bottom")j.top+=k;else if(b.at[1]==="center")j.top+=k/2;j.left+=e[0];j.top+=e[1];return this.each(function(){var f=c(this),l=f.outerWidth(),m=f.outerHeight(),p=parseInt(c.curCSS(this,"marginLeft",true))||0,q=parseInt(c.curCSS(this,"marginTop",true))||0,v=l+p+parseInt(c.curCSS(this,"marginRight",true))||0,w=m+q+parseInt(c.curCSS(this,"marginBottom",true))||0,i=c.extend({},j),r;if(b.my[0]==="right")i.left-=l;else if(b.my[0]==="center")i.left-=l/2;if(b.my[1]==="bottom")i.top-=m;else if(b.my[1]===
+"center")i.top-=m/2;i.left=parseInt(i.left);i.top=parseInt(i.top);r={left:i.left-p,top:i.top-q};c.each(["left","top"],function(s,x){c.ui.position[g[s]]&&c.ui.position[g[s]][x](i,{targetWidth:h,targetHeight:k,elemWidth:l,elemHeight:m,collisionPosition:r,collisionWidth:v,collisionHeight:w,offset:e,my:b.my,at:b.at})});c.fn.bgiframe&&f.bgiframe();f.offset(c.extend(i,{using:b.using}))})};c.ui.position={fit:{left:function(b,a){var d=c(window);d=a.collisionPosition.left+a.collisionWidth-d.width()-d.scrollLeft();
+b.left=d>0?b.left-d:Math.max(b.left-a.collisionPosition.left,b.left)},top:function(b,a){var d=c(window);d=a.collisionPosition.top+a.collisionHeight-d.height()-d.scrollTop();b.top=d>0?b.top-d:Math.max(b.top-a.collisionPosition.top,b.top)}},flip:{left:function(b,a){if(a.at[0]!=="center"){var d=c(window);d=a.collisionPosition.left+a.collisionWidth-d.width()-d.scrollLeft();var g=a.my[0]==="left"?-a.elemWidth:a.my[0]==="right"?a.elemWidth:0,e=a.at[0]==="left"?a.targetWidth:-a.targetWidth,h=-2*a.offset[0];
+b.left+=a.collisionPosition.left<0?g+e+h:d>0?g+e+h:0}},top:function(b,a){if(a.at[1]!=="center"){var d=c(window);d=a.collisionPosition.top+a.collisionHeight-d.height()-d.scrollTop();var g=a.my[1]==="top"?-a.elemHeight:a.my[1]==="bottom"?a.elemHeight:0,e=a.at[1]==="top"?a.targetHeight:-a.targetHeight,h=-2*a.offset[1];b.top+=a.collisionPosition.top<0?g+e+h:d>0?g+e+h:0}}}};if(!c.offset.setOffset){c.offset.setOffset=function(b,a){if(/static/.test(c.curCSS(b,"position")))b.style.position="relative";var d=
+c(b),g=d.offset(),e=parseInt(c.curCSS(b,"top",true),10)||0,h=parseInt(c.curCSS(b,"left",true),10)||0;g={top:a.top-g.top+e,left:a.left-g.left+h};"using"in a?a.using.call(b,g):d.css(g)};c.fn.offset=function(b){var a=this[0];if(!a||!a.ownerDocument)return null;if(b)return this.each(function(){c.offset.setOffset(this,b)});return u.call(this)}}})(jQuery);
+;/*
+ * jQuery UI Draggable 1.8.5
+ *
+ * Copyright 2010, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Draggables
+ *
+ * Depends:
+ *	jquery.ui.core.js
+ *	jquery.ui.mouse.js
+ *	jquery.ui.widget.js
+ */
+(function(d){d.widget("ui.draggable",d.ui.mouse,{widgetEventPrefix:"drag",options:{addClasses:true,appendTo:"parent",axis:false,connectToSortable:false,containment:false,cursor:"auto",cursorAt:false,grid:false,handle:false,helper:"original",iframeFix:false,opacity:false,refreshPositions:false,revert:false,revertDuration:500,scope:"default",scroll:true,scrollSensitivity:20,scrollSpeed:20,snap:false,snapMode:"both",snapTolerance:20,stack:false,zIndex:false},_create:function(){if(this.options.helper==
+"original"&&!/^(?:r|a|f)/.test(this.element.css("position")))this.element[0].style.position="relative";this.options.addClasses&&this.element.addClass("ui-draggable");this.options.disabled&&this.element.addClass("ui-draggable-disabled");this._mouseInit()},destroy:function(){if(this.element.data("draggable")){this.element.removeData("draggable").unbind(".draggable").removeClass("ui-draggable ui-draggable-dragging ui-draggable-disabled");this._mouseDestroy();return this}},_mouseCapture:function(a){var b=
+this.options;if(this.helper||b.disabled||d(a.target).is(".ui-resizable-handle"))return false;this.handle=this._getHandle(a);if(!this.handle)return false;return true},_mouseStart:function(a){var b=this.options;this.helper=this._createHelper(a);this._cacheHelperProportions();if(d.ui.ddmanager)d.ui.ddmanager.current=this;this._cacheMargins();this.cssPosition=this.helper.css("position");this.scrollParent=this.helper.scrollParent();this.offset=this.positionAbs=this.element.offset();this.offset={top:this.offset.top-
+this.margins.top,left:this.offset.left-this.margins.left};d.extend(this.offset,{click:{left:a.pageX-this.offset.left,top:a.pageY-this.offset.top},parent:this._getParentOffset(),relative:this._getRelativeOffset()});this.originalPosition=this.position=this._generatePosition(a);this.originalPageX=a.pageX;this.originalPageY=a.pageY;b.cursorAt&&this._adjustOffsetFromHelper(b.cursorAt);b.containment&&this._setContainment();if(this._trigger("start",a)===false){this._clear();return false}this._cacheHelperProportions();
+d.ui.ddmanager&&!b.dropBehaviour&&d.ui.ddmanager.prepareOffsets(this,a);this.helper.addClass("ui-draggable-dragging");this._mouseDrag(a,true);return true},_mouseDrag:function(a,b){this.position=this._generatePosition(a);this.positionAbs=this._convertPositionTo("absolute");if(!b){b=this._uiHash();if(this._trigger("drag",a,b)===false){this._mouseUp({});return false}this.position=b.position}if(!this.options.axis||this.options.axis!="y")this.helper[0].style.left=this.position.left+"px";if(!this.options.axis||
+this.options.axis!="x")this.helper[0].style.top=this.position.top+"px";d.ui.ddmanager&&d.ui.ddmanager.drag(this,a);return false},_mouseStop:function(a){var b=false;if(d.ui.ddmanager&&!this.options.dropBehaviour)b=d.ui.ddmanager.drop(this,a);if(this.dropped){b=this.dropped;this.dropped=false}if(!this.element[0]||!this.element[0].parentNode)return false;if(this.options.revert=="invalid"&&!b||this.options.revert=="valid"&&b||this.options.revert===true||d.isFunction(this.options.revert)&&this.options.revert.call(this.element,
+b)){var c=this;d(this.helper).animate(this.originalPosition,parseInt(this.options.revertDuration,10),function(){c._trigger("stop",a)!==false&&c._clear()})}else this._trigger("stop",a)!==false&&this._clear();return false},cancel:function(){this.helper.is(".ui-draggable-dragging")?this._mouseUp({}):this._clear();return this},_getHandle:function(a){var b=!this.options.handle||!d(this.options.handle,this.element).length?true:false;d(this.options.handle,this.element).find("*").andSelf().each(function(){if(this==
+a.target)b=true});return b},_createHelper:function(a){var b=this.options;a=d.isFunction(b.helper)?d(b.helper.apply(this.element[0],[a])):b.helper=="clone"?this.element.clone():this.element;a.parents("body").length||a.appendTo(b.appendTo=="parent"?this.element[0].parentNode:b.appendTo);a[0]!=this.element[0]&&!/(fixed|absolute)/.test(a.css("position"))&&a.css("position","absolute");return a},_adjustOffsetFromHelper:function(a){if(typeof a=="string")a=a.split(" ");if(d.isArray(a))a={left:+a[0],top:+a[1]||
+0};if("left"in a)this.offset.click.left=a.left+this.margins.left;if("right"in a)this.offset.click.left=this.helperProportions.width-a.right+this.margins.left;if("top"in a)this.offset.click.top=a.top+this.margins.top;if("bottom"in a)this.offset.click.top=this.helperProportions.height-a.bottom+this.margins.top},_getParentOffset:function(){this.offsetParent=this.helper.offsetParent();var a=this.offsetParent.offset();if(this.cssPosition=="absolute"&&this.scrollParent[0]!=document&&d.ui.contains(this.scrollParent[0],
+this.offsetParent[0])){a.left+=this.scrollParent.scrollLeft();a.top+=this.scrollParent.scrollTop()}if(this.offsetParent[0]==document.body||this.offsetParent[0].tagName&&this.offsetParent[0].tagName.toLowerCase()=="html"&&d.browser.msie)a={top:0,left:0};return{top:a.top+(parseInt(this.offsetParent.css("borderTopWidth"),10)||0),left:a.left+(parseInt(this.offsetParent.css("borderLeftWidth"),10)||0)}},_getRelativeOffset:function(){if(this.cssPosition=="relative"){var a=this.element.position();return{top:a.top-
+(parseInt(this.helper.css("top"),10)||0)+this.scrollParent.scrollTop(),left:a.left-(parseInt(this.helper.css("left"),10)||0)+this.scrollParent.scrollLeft()}}else return{top:0,left:0}},_cacheMargins:function(){this.margins={left:parseInt(this.element.css("marginLeft"),10)||0,top:parseInt(this.element.css("marginTop"),10)||0}},_cacheHelperProportions:function(){this.helperProportions={width:this.helper.outerWidth(),height:this.helper.outerHeight()}},_setContainment:function(){var a=this.options;if(a.containment==
+"parent")a.containment=this.helper[0].parentNode;if(a.containment=="document"||a.containment=="window")this.containment=[0-this.offset.relative.left-this.offset.parent.left,0-this.offset.relative.top-this.offset.parent.top,d(a.containment=="document"?document:window).width()-this.helperProportions.width-this.margins.left,(d(a.containment=="document"?document:window).height()||document.body.parentNode.scrollHeight)-this.helperProportions.height-this.margins.top];if(!/^(document|window|parent)$/.test(a.containment)&&
+a.containment.constructor!=Array){var b=d(a.containment)[0];if(b){a=d(a.containment).offset();var c=d(b).css("overflow")!="hidden";this.containment=[a.left+(parseInt(d(b).css("borderLeftWidth"),10)||0)+(parseInt(d(b).css("paddingLeft"),10)||0)-this.margins.left,a.top+(parseInt(d(b).css("borderTopWidth"),10)||0)+(parseInt(d(b).css("paddingTop"),10)||0)-this.margins.top,a.left+(c?Math.max(b.scrollWidth,b.offsetWidth):b.offsetWidth)-(parseInt(d(b).css("borderLeftWidth"),10)||0)-(parseInt(d(b).css("paddingRight"),
+10)||0)-this.helperProportions.width-this.margins.left,a.top+(c?Math.max(b.scrollHeight,b.offsetHeight):b.offsetHeight)-(parseInt(d(b).css("borderTopWidth"),10)||0)-(parseInt(d(b).css("paddingBottom"),10)||0)-this.helperProportions.height-this.margins.top]}}else if(a.containment.constructor==Array)this.containment=a.containment},_convertPositionTo:function(a,b){if(!b)b=this.position;a=a=="absolute"?1:-1;var c=this.cssPosition=="absolute"&&!(this.scrollParent[0]!=document&&d.ui.contains(this.scrollParent[0],
+this.offsetParent[0]))?this.offsetParent:this.scrollParent,f=/(html|body)/i.test(c[0].tagName);return{top:b.top+this.offset.relative.top*a+this.offset.parent.top*a-(d.browser.safari&&d.browser.version<526&&this.cssPosition=="fixed"?0:(this.cssPosition=="fixed"?-this.scrollParent.scrollTop():f?0:c.scrollTop())*a),left:b.left+this.offset.relative.left*a+this.offset.parent.left*a-(d.browser.safari&&d.browser.version<526&&this.cssPosition=="fixed"?0:(this.cssPosition=="fixed"?-this.scrollParent.scrollLeft():
+f?0:c.scrollLeft())*a)}},_generatePosition:function(a){var b=this.options,c=this.cssPosition=="absolute"&&!(this.scrollParent[0]!=document&&d.ui.contains(this.scrollParent[0],this.offsetParent[0]))?this.offsetParent:this.scrollParent,f=/(html|body)/i.test(c[0].tagName),e=a.pageX,g=a.pageY;if(this.originalPosition){if(this.containment){if(a.pageX-this.offset.click.left<this.containment[0])e=this.containment[0]+this.offset.click.left;if(a.pageY-this.offset.click.top<this.containment[1])g=this.containment[1]+
+this.offset.click.top;if(a.pageX-this.offset.click.left>this.containment[2])e=this.containment[2]+this.offset.click.left;if(a.pageY-this.offset.click.top>this.containment[3])g=this.containment[3]+this.offset.click.top}if(b.grid){g=this.originalPageY+Math.round((g-this.originalPageY)/b.grid[1])*b.grid[1];g=this.containment?!(g-this.offset.click.top<this.containment[1]||g-this.offset.click.top>this.containment[3])?g:!(g-this.offset.click.top<this.containment[1])?g-b.grid[1]:g+b.grid[1]:g;e=this.originalPageX+
+Math.round((e-this.originalPageX)/b.grid[0])*b.grid[0];e=this.containment?!(e-this.offset.click.left<this.containment[0]||e-this.offset.click.left>this.containment[2])?e:!(e-this.offset.click.left<this.containment[0])?e-b.grid[0]:e+b.grid[0]:e}}return{top:g-this.offset.click.top-this.offset.relative.top-this.offset.parent.top+(d.browser.safari&&d.browser.version<526&&this.cssPosition=="fixed"?0:this.cssPosition=="fixed"?-this.scrollParent.scrollTop():f?0:c.scrollTop()),left:e-this.offset.click.left-
+this.offset.relative.left-this.offset.parent.left+(d.browser.safari&&d.browser.version<526&&this.cssPosition=="fixed"?0:this.cssPosition=="fixed"?-this.scrollParent.scrollLeft():f?0:c.scrollLeft())}},_clear:function(){this.helper.removeClass("ui-draggable-dragging");this.helper[0]!=this.element[0]&&!this.cancelHelperRemoval&&this.helper.remove();this.helper=null;this.cancelHelperRemoval=false},_trigger:function(a,b,c){c=c||this._uiHash();d.ui.plugin.call(this,a,[b,c]);if(a=="drag")this.positionAbs=
+this._convertPositionTo("absolute");return d.Widget.prototype._trigger.call(this,a,b,c)},plugins:{},_uiHash:function(){return{helper:this.helper,position:this.position,originalPosition:this.originalPosition,offset:this.positionAbs}}});d.extend(d.ui.draggable,{version:"1.8.5"});d.ui.plugin.add("draggable","connectToSortable",{start:function(a,b){var c=d(this).data("draggable"),f=c.options,e=d.extend({},b,{item:c.element});c.sortables=[];d(f.connectToSortable).each(function(){var g=d.data(this,"sortable");
+if(g&&!g.options.disabled){c.sortables.push({instance:g,shouldRevert:g.options.revert});g._refreshItems();g._trigger("activate",a,e)}})},stop:function(a,b){var c=d(this).data("draggable"),f=d.extend({},b,{item:c.element});d.each(c.sortables,function(){if(this.instance.isOver){this.instance.isOver=0;c.cancelHelperRemoval=true;this.instance.cancelHelperRemoval=false;if(this.shouldRevert)this.instance.options.revert=true;this.instance._mouseStop(a);this.instance.options.helper=this.instance.options._helper;
+c.options.helper=="original"&&this.instance.currentItem.css({top:"auto",left:"auto"})}else{this.instance.cancelHelperRemoval=false;this.instance._trigger("deactivate",a,f)}})},drag:function(a,b){var c=d(this).data("draggable"),f=this;d.each(c.sortables,function(){this.instance.positionAbs=c.positionAbs;this.instance.helperProportions=c.helperProportions;this.instance.offset.click=c.offset.click;if(this.instance._intersectsWith(this.instance.containerCache)){if(!this.instance.isOver){this.instance.isOver=
+1;this.instance.currentItem=d(f).clone().appendTo(this.instance.element).data("sortable-item",true);this.instance.options._helper=this.instance.options.helper;this.instance.options.helper=function(){return b.helper[0]};a.target=this.instance.currentItem[0];this.instance._mouseCapture(a,true);this.instance._mouseStart(a,true,true);this.instance.offset.click.top=c.offset.click.top;this.instance.offset.click.left=c.offset.click.left;this.instance.offset.parent.left-=c.offset.parent.left-this.instance.offset.parent.left;
+this.instance.offset.parent.top-=c.offset.parent.top-this.instance.offset.parent.top;c._trigger("toSortable",a);c.dropped=this.instance.element;c.currentItem=c.element;this.instance.fromOutside=c}this.instance.currentItem&&this.instance._mouseDrag(a)}else if(this.instance.isOver){this.instance.isOver=0;this.instance.cancelHelperRemoval=true;this.instance.options.revert=false;this.instance._trigger("out",a,this.instance._uiHash(this.instance));this.instance._mouseStop(a,true);this.instance.options.helper=
+this.instance.options._helper;this.instance.currentItem.remove();this.instance.placeholder&&this.instance.placeholder.remove();c._trigger("fromSortable",a);c.dropped=false}})}});d.ui.plugin.add("draggable","cursor",{start:function(){var a=d("body"),b=d(this).data("draggable").options;if(a.css("cursor"))b._cursor=a.css("cursor");a.css("cursor",b.cursor)},stop:function(){var a=d(this).data("draggable").options;a._cursor&&d("body").css("cursor",a._cursor)}});d.ui.plugin.add("draggable","iframeFix",{start:function(){var a=
+d(this).data("draggable").options;d(a.iframeFix===true?"iframe":a.iframeFix).each(function(){d('<div class="ui-draggable-iframeFix" style="background: #fff;"></div>').css({width:this.offsetWidth+"px",height:this.offsetHeight+"px",position:"absolute",opacity:"0.001",zIndex:1E3}).css(d(this).offset()).appendTo("body")})},stop:function(){d("div.ui-draggable-iframeFix").each(function(){this.parentNode.removeChild(this)})}});d.ui.plugin.add("draggable","opacity",{start:function(a,b){a=d(b.helper);b=d(this).data("draggable").options;
+if(a.css("opacity"))b._opacity=a.css("opacity");a.css("opacity",b.opacity)},stop:function(a,b){a=d(this).data("draggable").options;a._opacity&&d(b.helper).css("opacity",a._opacity)}});d.ui.plugin.add("draggable","scroll",{start:function(){var a=d(this).data("draggable");if(a.scrollParent[0]!=document&&a.scrollParent[0].tagName!="HTML")a.overflowOffset=a.scrollParent.offset()},drag:function(a){var b=d(this).data("draggable"),c=b.options,f=false;if(b.scrollParent[0]!=document&&b.scrollParent[0].tagName!=
+"HTML"){if(!c.axis||c.axis!="x")if(b.overflowOffset.top+b.scrollParent[0].offsetHeight-a.pageY<c.scrollSensitivity)b.scrollParent[0].scrollTop=f=b.scrollParent[0].scrollTop+c.scrollSpeed;else if(a.pageY-b.overflowOffset.top<c.scrollSensitivity)b.scrollParent[0].scrollTop=f=b.scrollParent[0].scrollTop-c.scrollSpeed;if(!c.axis||c.axis!="y")if(b.overflowOffset.left+b.scrollParent[0].offsetWidth-a.pageX<c.scrollSensitivity)b.scrollParent[0].scrollLeft=f=b.scrollParent[0].scrollLeft+c.scrollSpeed;else if(a.pageX-
+b.overflowOffset.left<c.scrollSensitivity)b.scrollParent[0].scrollLeft=f=b.scrollParent[0].scrollLeft-c.scrollSpeed}else{if(!c.axis||c.axis!="x")if(a.pageY-d(document).scrollTop()<c.scrollSensitivity)f=d(document).scrollTop(d(document).scrollTop()-c.scrollSpeed);else if(d(window).height()-(a.pageY-d(document).scrollTop())<c.scrollSensitivity)f=d(document).scrollTop(d(document).scrollTop()+c.scrollSpeed);if(!c.axis||c.axis!="y")if(a.pageX-d(document).scrollLeft()<c.scrollSensitivity)f=d(document).scrollLeft(d(document).scrollLeft()-
+c.scrollSpeed);else if(d(window).width()-(a.pageX-d(document).scrollLeft())<c.scrollSensitivity)f=d(document).scrollLeft(d(document).scrollLeft()+c.scrollSpeed)}f!==false&&d.ui.ddmanager&&!c.dropBehaviour&&d.ui.ddmanager.prepareOffsets(b,a)}});d.ui.plugin.add("draggable","snap",{start:function(){var a=d(this).data("draggable"),b=a.options;a.snapElements=[];d(b.snap.constructor!=String?b.snap.items||":data(draggable)":b.snap).each(function(){var c=d(this),f=c.offset();this!=a.element[0]&&a.snapElements.push({item:this,
+width:c.outerWidth(),height:c.outerHeight(),top:f.top,left:f.left})})},drag:function(a,b){for(var c=d(this).data("draggable"),f=c.options,e=f.snapTolerance,g=b.offset.left,n=g+c.helperProportions.width,m=b.offset.top,o=m+c.helperProportions.height,h=c.snapElements.length-1;h>=0;h--){var i=c.snapElements[h].left,k=i+c.snapElements[h].width,j=c.snapElements[h].top,l=j+c.snapElements[h].height;if(i-e<g&&g<k+e&&j-e<m&&m<l+e||i-e<g&&g<k+e&&j-e<o&&o<l+e||i-e<n&&n<k+e&&j-e<m&&m<l+e||i-e<n&&n<k+e&&j-e<o&&
+o<l+e){if(f.snapMode!="inner"){var p=Math.abs(j-o)<=e,q=Math.abs(l-m)<=e,r=Math.abs(i-n)<=e,s=Math.abs(k-g)<=e;if(p)b.position.top=c._convertPositionTo("relative",{top:j-c.helperProportions.height,left:0}).top-c.margins.top;if(q)b.position.top=c._convertPositionTo("relative",{top:l,left:0}).top-c.margins.top;if(r)b.position.left=c._convertPositionTo("relative",{top:0,left:i-c.helperProportions.width}).left-c.margins.left;if(s)b.position.left=c._convertPositionTo("relative",{top:0,left:k}).left-c.margins.left}var t=
+p||q||r||s;if(f.snapMode!="outer"){p=Math.abs(j-m)<=e;q=Math.abs(l-o)<=e;r=Math.abs(i-g)<=e;s=Math.abs(k-n)<=e;if(p)b.position.top=c._convertPositionTo("relative",{top:j,left:0}).top-c.margins.top;if(q)b.position.top=c._convertPositionTo("relative",{top:l-c.helperProportions.height,left:0}).top-c.margins.top;if(r)b.position.left=c._convertPositionTo("relative",{top:0,left:i}).left-c.margins.left;if(s)b.position.left=c._convertPositionTo("relative",{top:0,left:k-c.helperProportions.width}).left-c.margins.left}if(!c.snapElements[h].snapping&&
+(p||q||r||s||t))c.options.snap.snap&&c.options.snap.snap.call(c.element,a,d.extend(c._uiHash(),{snapItem:c.snapElements[h].item}));c.snapElements[h].snapping=p||q||r||s||t}else{c.snapElements[h].snapping&&c.options.snap.release&&c.options.snap.release.call(c.element,a,d.extend(c._uiHash(),{snapItem:c.snapElements[h].item}));c.snapElements[h].snapping=false}}}});d.ui.plugin.add("draggable","stack",{start:function(){var a=d(this).data("draggable").options;a=d.makeArray(d(a.stack)).sort(function(c,f){return(parseInt(d(c).css("zIndex"),
+10)||0)-(parseInt(d(f).css("zIndex"),10)||0)});if(a.length){var b=parseInt(a[0].style.zIndex)||0;d(a).each(function(c){this.style.zIndex=b+c});this[0].style.zIndex=b+a.length}}});d.ui.plugin.add("draggable","zIndex",{start:function(a,b){a=d(b.helper);b=d(this).data("draggable").options;if(a.css("zIndex"))b._zIndex=a.css("zIndex");a.css("zIndex",b.zIndex)},stop:function(a,b){a=d(this).data("draggable").options;a._zIndex&&d(b.helper).css("zIndex",a._zIndex)}})})(jQuery);
+;/*
+ * jQuery UI Droppable 1.8.5
+ *
+ * Copyright 2010, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Droppables
+ *
+ * Depends:
+ *	jquery.ui.core.js
+ *	jquery.ui.widget.js
+ *	jquery.ui.mouse.js
+ *	jquery.ui.draggable.js
+ */
+(function(d){d.widget("ui.droppable",{widgetEventPrefix:"drop",options:{accept:"*",activeClass:false,addClasses:true,greedy:false,hoverClass:false,scope:"default",tolerance:"intersect"},_create:function(){var a=this.options,b=a.accept;this.isover=0;this.isout=1;this.accept=d.isFunction(b)?b:function(c){return c.is(b)};this.proportions={width:this.element[0].offsetWidth,height:this.element[0].offsetHeight};d.ui.ddmanager.droppables[a.scope]=d.ui.ddmanager.droppables[a.scope]||[];d.ui.ddmanager.droppables[a.scope].push(this);
+a.addClasses&&this.element.addClass("ui-droppable")},destroy:function(){for(var a=d.ui.ddmanager.droppables[this.options.scope],b=0;b<a.length;b++)a[b]==this&&a.splice(b,1);this.element.removeClass("ui-droppable ui-droppable-disabled").removeData("droppable").unbind(".droppable");return this},_setOption:function(a,b){if(a=="accept")this.accept=d.isFunction(b)?b:function(c){return c.is(b)};d.Widget.prototype._setOption.apply(this,arguments)},_activate:function(a){var b=d.ui.ddmanager.current;this.options.activeClass&&
+this.element.addClass(this.options.activeClass);b&&this._trigger("activate",a,this.ui(b))},_deactivate:function(a){var b=d.ui.ddmanager.current;this.options.activeClass&&this.element.removeClass(this.options.activeClass);b&&this._trigger("deactivate",a,this.ui(b))},_over:function(a){var b=d.ui.ddmanager.current;if(!(!b||(b.currentItem||b.element)[0]==this.element[0]))if(this.accept.call(this.element[0],b.currentItem||b.element)){this.options.hoverClass&&this.element.addClass(this.options.hoverClass);
+this._trigger("over",a,this.ui(b))}},_out:function(a){var b=d.ui.ddmanager.current;if(!(!b||(b.currentItem||b.element)[0]==this.element[0]))if(this.accept.call(this.element[0],b.currentItem||b.element)){this.options.hoverClass&&this.element.removeClass(this.options.hoverClass);this._trigger("out",a,this.ui(b))}},_drop:function(a,b){var c=b||d.ui.ddmanager.current;if(!c||(c.currentItem||c.element)[0]==this.element[0])return false;var e=false;this.element.find(":data(droppable)").not(".ui-draggable-dragging").each(function(){var g=
+d.data(this,"droppable");if(g.options.greedy&&!g.options.disabled&&g.options.scope==c.options.scope&&g.accept.call(g.element[0],c.currentItem||c.element)&&d.ui.intersect(c,d.extend(g,{offset:g.element.offset()}),g.options.tolerance)){e=true;return false}});if(e)return false;if(this.accept.call(this.element[0],c.currentItem||c.element)){this.options.activeClass&&this.element.removeClass(this.options.activeClass);this.options.hoverClass&&this.element.removeClass(this.options.hoverClass);this._trigger("drop",
+a,this.ui(c));return this.element}return false},ui:function(a){return{draggable:a.currentItem||a.element,helper:a.helper,position:a.position,offset:a.positionAbs}}});d.extend(d.ui.droppable,{version:"1.8.5"});d.ui.intersect=function(a,b,c){if(!b.offset)return false;var e=(a.positionAbs||a.position.absolute).left,g=e+a.helperProportions.width,f=(a.positionAbs||a.position.absolute).top,h=f+a.helperProportions.height,i=b.offset.left,k=i+b.proportions.width,j=b.offset.top,l=j+b.proportions.height;
+switch(c){case "fit":return i<=e&&g<=k&&j<=f&&h<=l;case "intersect":return i<e+a.helperProportions.width/2&&g-a.helperProportions.width/2<k&&j<f+a.helperProportions.height/2&&h-a.helperProportions.height/2<l;case "pointer":return d.ui.isOver((a.positionAbs||a.position.absolute).top+(a.clickOffset||a.offset.click).top,(a.positionAbs||a.position.absolute).left+(a.clickOffset||a.offset.click).left,j,i,b.proportions.height,b.proportions.width);case "touch":return(f>=j&&f<=l||h>=j&&h<=l||f<j&&h>l)&&(e>=
+i&&e<=k||g>=i&&g<=k||e<i&&g>k);default:return false}};d.ui.ddmanager={current:null,droppables:{"default":[]},prepareOffsets:function(a,b){var c=d.ui.ddmanager.droppables[a.options.scope]||[],e=b?b.type:null,g=(a.currentItem||a.element).find(":data(droppable)").andSelf(),f=0;a:for(;f<c.length;f++)if(!(c[f].options.disabled||a&&!c[f].accept.call(c[f].element[0],a.currentItem||a.element))){for(var h=0;h<g.length;h++)if(g[h]==c[f].element[0]){c[f].proportions.height=0;continue a}c[f].visible=c[f].element.css("display")!=
+"none";if(c[f].visible){c[f].offset=c[f].element.offset();c[f].proportions={width:c[f].element[0].offsetWidth,height:c[f].element[0].offsetHeight};e=="mousedown"&&c[f]._activate.call(c[f],b)}}},drop:function(a,b){var c=false;d.each(d.ui.ddmanager.droppables[a.options.scope]||[],function(){if(this.options){if(!this.options.disabled&&this.visible&&d.ui.intersect(a,this,this.options.tolerance))c=c||this._drop.call(this,b);if(!this.options.disabled&&this.visible&&this.accept.call(this.element[0],a.currentItem||
+a.element)){this.isout=1;this.isover=0;this._deactivate.call(this,b)}}});return c},drag:function(a,b){a.options.refreshPositions&&d.ui.ddmanager.prepareOffsets(a,b);d.each(d.ui.ddmanager.droppables[a.options.scope]||[],function(){if(!(this.options.disabled||this.greedyChild||!this.visible)){var c=d.ui.intersect(a,this,this.options.tolerance);if(c=!c&&this.isover==1?"isout":c&&this.isover==0?"isover":null){var e;if(this.options.greedy){var g=this.element.parents(":data(droppable):eq(0)");if(g.length){e=
+d.data(g[0],"droppable");e.greedyChild=c=="isover"?1:0}}if(e&&c=="isover"){e.isover=0;e.isout=1;e._out.call(e,b)}this[c]=1;this[c=="isout"?"isover":"isout"]=0;this[c=="isover"?"_over":"_out"].call(this,b);if(e&&c=="isout"){e.isout=0;e.isover=1;e._over.call(e,b)}}}})}}})(jQuery);
+;

dnet-core-components/src/main/resources/eu/dnetlib/enabling/views/inspector/jstorage.js

@@ -0,0 +1,362 @@
+/*
+ * ----------------------------- JSTORAGE -------------------------------------
+ * Simple local storage wrapper to save data on the browser side, supporting
+ * all major browsers - IE6+, Firefox2+, Safari4+, Chrome4+ and Opera 10.5+
+ *
+ * Copyright (c) 2010 Andris Reinman, andris.reinman@gmail.com
+ * Project homepage: www.jstorage.info
+ *
+ * Licensed under MIT-style license:
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+/**
+ * $.jStorage
+ * 
+ * USAGE:
+ *
+ * jStorage requires Prototype, MooTools or jQuery! If jQuery is used, then
+ * jQuery-JSON (http://code.google.com/p/jquery-json/) is also needed.
+ * (jQuery-JSON needs to be loaded BEFORE jStorage!)
+ *
+ * Methods:
+ *
+ * -set(key, value)
+ * $.jStorage.set(key, value) -> saves a value
+ *
+ * -get(key[, default])
+ * value = $.jStorage.get(key [, default]) ->
+ *    retrieves value if key exists, or default if it doesn't
+ *
+ * -deleteKey(key)
+ * $.jStorage.deleteKey(key) -> removes a key from the storage
+ *
+ * -flush()
+ * $.jStorage.flush() -> clears the cache
+ * 
+ * -storageObj()
+ * $.jStorage.storageObj() -> returns a read-ony copy of the actual storage
+ * 
+ * -storageSize()
+ * $.jStorage.storageSize() -> returns the size of the storage in bytes
+ *
+ * -index()
+ * $.jStorage.index() -> returns the used keys as an array
+ * 
+ * <value> can be any JSON-able value, including objects and arrays.
+ *
+ **/
+
+(function($){
+    if(!$ || !($.toJSON || Object.toJSON || window.JSON)){
+        throw new Error("jQuery, MooTools or Prototype needs to be loaded before jStorage!");
+    }
+    
+    var
+        /* This is the object, that holds the cached values */ 
+        _storage = {},
+
+        /* Actual browser storage (localStorage or globalStorage['domain']) */
+        _storage_service = {jStorage:"{}"},
+
+        /* DOM element for older IE versions, holds userData behavior */
+        _storage_elm = null,
+        
+        /* How much space does the storage take */
+        _storage_size = 0,
+
+        /* function to encode objects to JSON strings */
+        json_encode = $.toJSON || Object.toJSON || (window.JSON && (JSON.encode || JSON.stringify)),
+
+        /* function to decode objects from JSON strings */
+        json_decode = $.evalJSON || (window.JSON && (JSON.decode || JSON.parse)) || function(str){
+            return String(str).evalJSON();
+        },
+        
+        /* which backend is currently used */
+        _backend = false;
+        
+        /**
+         * XML encoding and decoding as XML nodes can't be JSON'ized
+         * XML nodes are encoded and decoded if the node is the value to be saved
+         * but not if it's as a property of another object
+         * Eg. -
+         *   $.jStorage.set("key", xmlNode);        // IS OK
+         *   $.jStorage.set("key", {xml: xmlNode}); // NOT OK
+         */
+        _XMLService = {
+            
+            /**
+             * Validates a XML node to be XML
+             * based on jQuery.isXML function
+             */
+            isXML: function(elm){
+                var documentElement = (elm ? elm.ownerDocument || elm : 0).documentElement;
+                return documentElement ? documentElement.nodeName !== "HTML" : false;
+            },
+            
+            /**
+             * Encodes a XML node to string
+             * based on http://www.mercurytide.co.uk/news/article/issues-when-working-ajax/
+             */
+            encode: function(xmlNode) {
+                if(!this.isXML(xmlNode)){
+                    return false;
+                }
+                try{ // Mozilla, Webkit, Opera
+                    return new XMLSerializer().serializeToString(xmlNode);
+                }catch(E1) {
+                    try {  // IE
+                        return xmlNode.xml;
+                    }catch(E2){}
+                }
+                return false;
+            },
+            
+            /**
+             * Decodes a XML node from string
+             * loosely based on http://outwestmedia.com/jquery-plugins/xmldom/
+             */
+            decode: function(xmlString){
+                var dom_parser = ("DOMParser" in window && (new DOMParser()).parseFromString) ||
+                        (window.ActiveXObject && function(_xmlString) {
+                    var xml_doc = new ActiveXObject('Microsoft.XMLDOM');
+                    xml_doc.async = 'false';
+                    xml_doc.loadXML(_xmlString);
+                    return xml_doc;
+                }),
+                resultXML;
+                if(!dom_parser){
+                    return false;
+                }
+                resultXML = dom_parser.call("DOMParser" in window && (new DOMParser()) || window, xmlString, 'text/xml');
+                return this.isXML(resultXML)?resultXML:false;
+            }
+        };
+
+    ////////////////////////// PRIVATE METHODS ////////////////////////
+
+    /**
+     * Initialization function. Detects if the browser supports DOM Storage
+     * or userData behavior and behaves accordingly.
+     * @returns undefined
+     */
+    function _init(){
+        /* Check if browser supports localStorage */
+        if(window.localStorage){
+            try {
+                _storage_service = window.localStorage;
+                _backend = "localStorage";
+            } catch(E3) {/* Firefox fails when touching localStorage and cookies are disabled */}
+        }
+        /* Check if browser supports globalStorage */
+        else if(window.globalStorage){
+            try {
+                _storage_service = window.globalStorage[window.location.hostname];
+                _backend = "globalStorage";
+            } catch(E4) {/* Firefox fails when touching localStorage and cookies are disabled */}
+        }
+        /* Check if browser supports userData behavior */
+        else {
+            _storage_elm = document.createElement('link');
+            if(_storage_elm.addBehavior){
+
+                /* Use a DOM element to act as userData storage */
+                _storage_elm.style.behavior = 'url(#default#userData)';
+
+                /* userData element needs to be inserted into the DOM! */
+                document.getElementsByTagName('head')[0].appendChild(_storage_elm);
+
+                _storage_elm.load("jStorage");
+                var data = "{}";
+                try{
+                    data = _storage_elm.getAttribute("jStorage");
+                }catch(E5){}
+                _storage_service.jStorage = data;
+                _backend = "userDataBehavior";
+            }else{
+                _storage_elm = null;
+                return;
+            }
+        }
+
+        /* if jStorage string is retrieved, then decode it */
+        if(_storage_service.jStorage){
+            try{
+                _storage = json_decode(String(_storage_service.jStorage));
+            }catch(E6){_storage_service.jStorage = "{}";}
+        }else{
+            _storage_service.jStorage = "{}";
+        }
+        _storage_size = _storage_service.jStorage?String(_storage_service.jStorage).length:0;
+    }
+
+    /**
+     * This functions provides the "save" mechanism to store the jStorage object
+     * @returns undefined
+     */
+    function _save(){
+        try{
+            _storage_service.jStorage = json_encode(_storage);
+            // If userData is used as the storage engine, additional
+            if(_storage_elm) {
+                _storage_elm.setAttribute("jStorage",_storage_service.jStorage);
+                _storage_elm.save("jStorage");
+            }
+            _storage_size = _storage_service.jStorage?String(_storage_service.jStorage).length:0;
+        }catch(E7){/* probably cache is full, nothing is saved this way*/}
+    }
+
+    /**
+     * Function checks if a key is set and is string or numberic
+     */
+    function _checkKey(key){
+        if(!key || (typeof key != "string" && typeof key != "number")){
+            throw new TypeError('Key name must be string or numeric');
+        }
+        return true;
+    }
+
+    ////////////////////////// PUBLIC INTERFACE /////////////////////////
+
+    $.jStorage = {
+        /* Version number */
+        version: "0.1.4.1",
+
+        /**
+         * Sets a key's value.
+         * 
+         * @param {String} key - Key to set. If this value is not set or not
+         *              a string an exception is raised.
+         * @param value - Value to set. This can be any value that is JSON
+         *              compatible (Numbers, Strings, Objects etc.).
+         * @returns the used value
+         */
+        set: function(key, value){
+            _checkKey(key);
+            if(_XMLService.isXML(value)){
+                value = {_is_xml:true,xml:_XMLService.encode(value)};
+            }
+            _storage[key] = value;
+            _save();
+            return value;
+        },
+        
+        /**
+         * Looks up a key in cache
+         * 
+         * @param {String} key - Key to look up.
+         * @param {mixed} def - Default value to return, if key didn't exist.
+         * @returns the key value, default value or <null>
+         */
+        get: function(key, def){
+            _checkKey(key);
+            if(key in _storage){
+                if(typeof _storage[key] == "object" &&
+                        _storage[key]._is_xml &&
+                            _storage[key]._is_xml){
+                    return _XMLService.decode(_storage[key].xml);
+                }else{
+                    return _storage[key];
+                }
+            }
+            return typeof(def) == 'undefined' ? null : def;
+        },
+        
+        /**
+         * Deletes a key from cache.
+         * 
+         * @param {String} key - Key to delete.
+         * @returns true if key existed or false if it didn't
+         */
+        deleteKey: function(key){
+            _checkKey(key);
+            if(key in _storage){
+                delete _storage[key];
+                _save();
+                return true;
+            }
+            return false;
+        },
+
+        /**
+         * Deletes everything in cache.
+         * 
+         * @returns true
+         */
+        flush: function(){
+            _storage = {};
+            _save();
+            /*
+             * Just to be sure - andris9/jStorage#3
+             */
+            try{
+                window.localStorage.clear();
+            }catch(E8){}
+            return true;
+        },
+        
+        /**
+         * Returns a read-only copy of _storage
+         * 
+         * @returns Object
+        */
+        storageObj: function(){
+            function F() {}
+            F.prototype = _storage;
+            return new F();
+        },
+        
+        /**
+         * Returns an index of all used keys as an array
+         * ['key1', 'key2',..'keyN']
+         * 
+         * @returns Array
+        */
+        index: function(){
+            var index = [], i;
+            for(i in _storage){
+                if(_storage.hasOwnProperty(i)){
+                    index.push(i);
+                }
+            }
+            return index;
+        },
+        
+        /**
+         * How much space in bytes does the storage take?
+         * 
+         * @returns Number
+         */
+        storageSize: function(){
+            return _storage_size;
+        },
+        
+        /**
+         * Which backend is currently in use?
+         * 
+         * @returns String
+         */
+        currentBackend: function(){
+            return _backend;
+        }
+    };
+
+    // Initialize jStorage
+    _init();
+
+})(window.jQuery || window.$);

dnet-core-components/src/main/resources/eu/dnetlib/enabling/views/inspector/master.st

@@ -0,0 +1,47 @@
+<html>
+	<head>
+		<title>$if(title)$$title$$else$No title$endif$</title>
+		<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+		<script type="text/javascript" src="$baseUrl$/static.do?src=jquery.js"></script>
+		<script type="text/javascript" src="$baseUrl$/static.do?src=jquery-ui.min.js"></script>
+		<script type="text/javascript" src="$baseUrl$/static.do?src=jstorage.js"></script>
+		<script type="text/javascript" src="$baseUrl$/static.do?src=inspector.js"></script>
+		<link rel="stylesheet" href="$baseUrl$/static.do?src=inspector.css">
+		$header$
+		
+		<script>
+			setDefaultHiddenItems(",$entryPoints:{$if(it.hiddenAsDefault)$$it.name$,$endif$}$");
+		</script>
+	</head>
+ 
+	<body style="background-image: url($baseUrl$/static.do?src=inspector-logo.png);">
+		<div id="nav">
+			<ul>
+				$entryPoints:{<li class="menuItem"><a $if(it.current)$class="selected"$endif$ href="$baseUrl$/$it.relativeUrl$">$it.name$</a></li> }$
+				<li>
+					<a id="menu_P1" class="menuPerspective" href="javascript:changePerspective('P1')" title="perspective P1">P1</a>
+				</li>
+				<li>
+					<a id="menu_P2" class="menuPerspective" href="javascript:changePerspective('P2')" title="perspective P2">P2</a>
+				</li>
+				<li>
+					<a id="menu_P3" class="menuPerspective" href="javascript:changePerspective('P3')" title="perspective P3">P3</a>
+				</li>
+				<li>
+					<a href="javascript:configPerspectives()" class="menuPerspective" title="perspective configuration">+</a>
+				</li>
+			</ul>
+		</div>
+		<div id="menuConfig">
+			<div>
+				<a href="#" onclick="configPerspectives(); return 0;">close</a> | 
+				<a href="#" onclick="resetJStorage(); return 0;">reset all</a>
+			</div>
+			<ul>
+			$entryPoints:{<li class="menuAction"><input name="perspective_check_$it.name$" type="checkbox" onchange="changePerspectiveItem(this)" ></input>$it.name$</li>}$
+			</ul>
+		</div>
+		<div id="content">$it$</div>
+	</body>
+	
+</html>

dnet-data-services/pom.xml

@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>eu.dnetlib</groupId>
+		<artifactId>dnet-core</artifactId>
+		<version>1.0.0-SNAPSHOT</version>
+	</parent>
+	<artifactId>dnet-data-services</artifactId>
+	<groupId>eu.dnetlib</groupId>
+	<packaging>jar</packaging>
+	<version>1.0.0-SNAPSHOT</version>
+
+	<dependencies>
+		<dependency>
+			<groupId>eu.dnetlib</groupId>
+			<artifactId>dnet-core-components</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>eu.dnetlib</groupId>
+			<artifactId>dnet-core-services</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+
+
+		<dependency>
+			<groupId>com.ximpleware</groupId>
+			<artifactId>vtd-xml</artifactId>
+		</dependency>
+
+	</dependencies>
+
+</project>

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/BulkRecordMapperFactory.java

@@ -0,0 +1,133 @@
+package eu.dnetlib.data.mdstore.modular;
+
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.util.List;
+
+import javax.xml.namespace.QName;
+import javax.xml.stream.XMLEventFactory;
+import javax.xml.stream.XMLEventReader;
+import javax.xml.stream.XMLEventWriter;
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLOutputFactory;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.events.Namespace;
+import javax.xml.stream.events.StartElement;
+import javax.xml.stream.events.XMLEvent;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+
+import eu.dnetlib.miscutils.collections.Pair;
+import eu.dnetlib.miscutils.factory.Factory;
+
+public class BulkRecordMapperFactory implements Factory<Function<String, Pair<String, String>>> {
+	
+	private static final Log log = LogFactory.getLog(BulkRecordMapperFactory.class); // NOPMD by marko on 11/24/08 5:02 PM
+	
+	protected static final String MD_RECORD = "mdRecord";
+	
+	protected static final String MD_ID = "mdId";
+	
+	protected static final String RECORD = "record";
+
+	protected ThreadLocal<XMLInputFactory> inputFactory = new ThreadLocal<XMLInputFactory>() {
+		@Override
+		protected XMLInputFactory initialValue() {
+			return XMLInputFactory.newInstance();
+		}
+	};
+
+	protected ThreadLocal<XMLOutputFactory> outputFactory = new ThreadLocal<XMLOutputFactory>() {
+		@Override
+		protected XMLOutputFactory initialValue() {
+			return XMLOutputFactory.newInstance();
+		}
+	};
+
+	protected ThreadLocal<XMLEventFactory> eventFactory = new ThreadLocal<XMLEventFactory>() {
+		@Override
+		protected XMLEventFactory initialValue() {
+			return XMLEventFactory.newInstance();
+		}
+	};
+	
+	@Override
+	public Function<String, Pair<String, String>> newInstance() {
+		return new Function<String, Pair<String, String>>() {
+			private String mdId = null;
+			private String record = null;
+			@Override
+			public Pair<String, String> apply(String embeddedRecord) {
+				try {
+					final XMLEventReader parser = inputFactory.get().createXMLEventReader(new StringReader(embeddedRecord));
+
+					while (parser.hasNext()) {
+						final XMLEvent event = parser.nextEvent();
+						if (event != null && event.isStartElement()) {
+							final String localName = event.asStartElement().getName().getLocalPart();
+
+							if (MD_RECORD.equals(localName)) {
+								mdId = event.asStartElement().getAttributeByName(new QName(MD_ID)).getValue();
+
+							} else if (RECORD.equals(localName)) {
+								record = getRecord(embeddedRecord, parser);
+							} 
+						}
+					}
+				} catch (final XMLStreamException e) {
+					log.error("error parsing record: " + embeddedRecord);
+				}				
+				return new Pair<String, String>(mdId, record);
+			}
+		};
+	}
+	
+	/**
+	 * Copy the /indexRecord/result element and children, preserving namespace declarations etc.
+	 * 
+	 * @param indexDocument
+	 * @param results
+	 * @param parser
+	 * @throws XMLStreamException
+	 */
+	protected String getRecord(final String record, final XMLEventReader parser) throws XMLStreamException {
+		StringWriter results = new StringWriter();
+		final XMLEventWriter writer = outputFactory.get().createXMLEventWriter(results);
+
+		// TODO: newRecord should copy all the namespace prefixes setup in parents
+		// fortunately the only parent of the result element is the 'indexrecord', so it should be easy to get
+		// the namespaces declared on the root element (and fast)
+
+		final List<Namespace> namespaces = Lists.newArrayList(
+				eventFactory.get().createNamespace("dri", "http://www.driver-repository.eu/namespace/dri"),
+				eventFactory.get().createNamespace("dr", "http://www.driver-repository.eu/namespace/dr"),
+				eventFactory.get().createNamespace("xsi", "http://www.w3.org/2001/XMLSchema-instance"),
+				eventFactory.get().createNamespace("dc", "http://purl.org/dc/elements/1.1/"));
+
+		StartElement newRecord = eventFactory.get().createStartElement("", null, RECORD, null, namespaces.iterator());
+
+		// new root record
+		writer.add(newRecord);
+
+		// copy the rest as it is
+		while (parser.hasNext()) {
+			final XMLEvent resultEvent = parser.nextEvent();
+
+			// TODO: replace with depth tracking instead of close tag tracking. 
+			if (resultEvent.isEndElement() && resultEvent.asEndElement().getName().getLocalPart().equals(RECORD)) {
+				writer.add(resultEvent);
+				break;
+			}
+
+			writer.add(resultEvent);
+		}
+		writer.close();
+
+		return results.toString();
+	}
+	
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/MDFormatDescription.java

@@ -0,0 +1,36 @@
+package eu.dnetlib.data.mdstore.modular;
+
+/**
+ * Created by sandro on 11/29/16.
+ */
+public class MDFormatDescription {
+
+    private String name;
+
+    private String xpath;
+
+    public MDFormatDescription() {
+
+    }
+
+    public MDFormatDescription(String name, String xpath) {
+        this.name = name;
+        this.xpath = xpath;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getXpath() {
+        return xpath;
+    }
+
+    public void setXpath(String xpath) {
+        this.xpath = xpath;
+    }
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/MDStoreConstants.java

@@ -0,0 +1,12 @@
+package eu.dnetlib.data.mdstore.modular;
+
+public class MDStoreConstants {
+
+	public static final String ID =  "id";
+	public static final String OBJIDENTIFIER =  "objIdentifier";
+	public static final String TIMESTAMP = "timestamp";
+	public static final String ORIGINALID = "originalId";
+	public static final String BODY = "body";
+
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/MDStoreDescription.java

@@ -0,0 +1,80 @@
+package eu.dnetlib.data.mdstore.modular;
+
+
+public class MDStoreDescription {
+
+	private String id;
+
+	private String format;
+
+	private String layout;
+
+	private String interpretation;
+
+	private int size;
+
+	private boolean indexed;
+
+	public MDStoreDescription(final String id, final String format, final String layout, final String interpretation, final int size, final boolean indexed) {
+		super();
+		this.id = id;
+		this.format = format;
+		this.layout = layout;
+		this.interpretation = interpretation;
+		this.size = size;
+		this.indexed = indexed;
+	}
+
+    public MDStoreDescription() {
+
+    }
+
+    public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getFormat() {
+		return format;
+	}
+
+	public void setFormat(final String format) {
+		this.format = format;
+	}
+
+	public String getLayout() {
+		return layout;
+	}
+
+	public void setLayout(final String layout) {
+		this.layout = layout;
+	}
+
+	public String getInterpretation() {
+		return interpretation;
+	}
+
+	public void setInterpretation(final String interpretation) {
+		this.interpretation = interpretation;
+	}
+
+	public int getSize() {
+		return size;
+	}
+
+	public void setSize(final int size) {
+		this.size = size;
+	}
+
+	public boolean isIndexed() {
+		return indexed;
+	}
+
+	public void setIndexed(final boolean indexed) {
+		this.indexed = indexed;
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/MDStoreFeeder.java

@@ -0,0 +1,154 @@
+package eu.dnetlib.data.mdstore.modular;
+
+import com.google.common.collect.Maps;
+import eu.dnetlib.data.mdstore.MDStoreServiceException;
+import eu.dnetlib.data.mdstore.modular.action.DoneCallback;
+import eu.dnetlib.data.mdstore.modular.action.FailedCallback;
+import eu.dnetlib.data.mdstore.modular.connector.MDStore;
+import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;
+import eu.dnetlib.miscutils.datetime.DateUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Required;
+
+import java.util.List;
+import java.util.Map;
+
+public class MDStoreFeeder {
+
+	private static final Log log = LogFactory.getLog(MDStoreFeeder.class);
+
+	private MDStoreDao dao;
+
+	private ResultSetClientFactory resultSetClientFactory;
+
+	private UniqueServiceLocator serviceLocator;
+
+	private boolean syncFeed = true;
+
+	public void feed(final String mdId,
+                     final String rsEpr,
+                     final String storingType,
+                     final boolean sync,
+                     final List<MDFormatDescription> mdformats,
+                     final DoneCallback doneCallback,
+                     final FailedCallback failCallback) throws MDStoreServiceException {
+		log.info("Start feeding mdstore " + mdId);
+		log.debug("Start feeding mdstore " + mdId + " with epr " + rsEpr);
+
+		String transactionId = null;
+
+		try {
+			final boolean refresh = "REFRESH".equals(storingType);
+
+			final MDStore mdstore = dao.startTransaction(mdId, refresh);
+			transactionId = mdstore.getId();
+
+			final Iterable<String> records = resultSetClientFactory.getClient(rsEpr);
+
+			if (refresh) {
+				mdstore.truncate();
+			}
+            int writeOps;
+
+            if (mdformats == null) {
+                writeOps = mdstore.feed(records, refresh);
+            } else {
+                writeOps = mdstore.feed(records, refresh, mdformats);
+            }
+
+			dao.commit(mdstore.getId(), mdId);
+
+			int size = dao.refreshSize(mdId);
+
+			touch(mdId, size);
+
+			log.info("Finished feeding mdstore " + mdId + " - new size: " + size);
+
+			doneCallback.call(buildParams(size, writeOps));
+		} catch (Throwable e) {
+			if (transactionId != null) {
+				dao.invalidTransaction(transactionId, mdId);
+			}
+			log.error("Error feeding mdstore: " + mdId);
+			failCallback.call(e);
+		}
+	}
+
+	private Map<String, String> buildParams(final int size, final int storeCount) {
+		Map<String, String> params = Maps.newHashMap();
+		params.put("mdstoreSize", String.valueOf(size));
+		params.put("writeOps", String.valueOf(storeCount));
+		return params;
+	}
+
+	/**
+	 * Sets the last modified date in the profile.
+	 *
+	 * @param mdId
+	 */
+	public void touch(final String mdId, final int size) {
+		try {
+			final String now = DateUtils.now_ISO8601();
+
+			final String mdstoreXUpdate = "for $x in //RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '" + mdId + "']"
+					+ "return update value $x//LAST_STORAGE_DATE with '" + now + "'";
+
+			serviceLocator.getService(ISRegistryService.class).executeXUpdate(mdstoreXUpdate);
+
+			touchSize(mdId, size);
+		} catch (final Exception e) {
+			throw new IllegalStateException(e);
+		}
+	}
+
+	public void touchSize(final String mdId, final int size) {
+		try {
+			final String mdstoreNumberXUpdate = "for $x in //RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '" + mdId + "']"
+					+ "return update value $x//NUMBER_OF_RECORDS with '" + size + "'";
+
+			serviceLocator.getService(ISRegistryService.class).executeXUpdate(mdstoreNumberXUpdate);
+		} catch (final Exception e) {
+			throw new IllegalStateException(e);
+		}
+	}
+
+	public MDStoreDao getDao() {
+		return dao;
+	}
+
+	@Required
+	public void setDao(final MDStoreDao dao) {
+		this.dao = dao;
+	}
+
+	public ResultSetClientFactory getResultSetClientFactory() {
+		return resultSetClientFactory;
+	}
+
+	@Required
+	public void setResultSetClientFactory(final ResultSetClientFactory resultSetClientFactory) {
+		this.resultSetClientFactory = resultSetClientFactory;
+	}
+
+	public boolean isSyncFeed() {
+		return syncFeed;
+	}
+
+	public void setSyncFeed(final boolean syncFeed) {
+		this.syncFeed = syncFeed;
+	}
+
+	public UniqueServiceLocator getServiceLocator() {
+		return serviceLocator;
+	}
+
+	@Required
+	public void setServiceLocator(final UniqueServiceLocator serviceLocator) {
+		this.serviceLocator = serviceLocator;
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/MDStoreProfileCreator.java

@@ -0,0 +1,80 @@
+package eu.dnetlib.data.mdstore.modular;
+
+import javax.xml.ws.Endpoint;
+
+import org.antlr.stringtemplate.StringTemplate;
+import org.springframework.beans.factory.annotation.Required;
+
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.soap.EndpointReferenceBuilder;
+
+public class MDStoreProfileCreator {
+	/**
+	 * service locator.
+	 */
+	private UniqueServiceLocator serviceLocator;
+
+	/**
+	 * mdstore ds template.
+	 */
+	private StringTemplate mdstoreDsTemplate;
+
+	/**
+	 * service endpoint.
+	 */
+	private Endpoint endpoint;
+
+	/**
+	 * endpoint builder.
+	 */
+	private EndpointReferenceBuilder<Endpoint> eprBuilder;
+
+	public String registerProfile(String format, String interpretation, String layout) throws ISRegistryException {
+		// XXX: mini hack
+		StringTemplate template = new StringTemplate(mdstoreDsTemplate.getTemplate());
+		template.setAttribute("serviceUri", eprBuilder.getAddress(endpoint));
+		template.setAttribute("format", format);
+		template.setAttribute("interpretation", interpretation);
+		template.setAttribute("layout", layout);
+
+		return serviceLocator.getService(ISRegistryService.class).registerProfile(template.toString());
+	}
+
+	public StringTemplate getMdstoreDsTemplate() {
+		return mdstoreDsTemplate;
+	}
+
+	@Required
+	public void setMdstoreDsTemplate(StringTemplate mdstoreDsTemplate) {
+		this.mdstoreDsTemplate = mdstoreDsTemplate;
+	}
+
+	public Endpoint getEndpoint() {
+		return endpoint;
+	}
+
+	@Required
+	public void setEndpoint(Endpoint endpoint) {
+		this.endpoint = endpoint;
+	}
+
+	public EndpointReferenceBuilder<Endpoint> getEprBuilder() {
+		return eprBuilder;
+	}
+
+	@Required
+	public void setEprBuilder(EndpointReferenceBuilder<Endpoint> eprBuilder) {
+		this.eprBuilder = eprBuilder;
+	}
+
+	public UniqueServiceLocator getServiceLocator() {
+		return serviceLocator;
+	}
+
+	@Required
+	public void setServiceLocator(UniqueServiceLocator serviceLocator) {
+		this.serviceLocator = serviceLocator;
+	}
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/MDStoreRetriever.java

@@ -0,0 +1,77 @@
+package eu.dnetlib.data.mdstore.modular;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import eu.dnetlib.data.mdstore.DocumentNotFoundException;
+import eu.dnetlib.data.mdstore.MDStoreServiceException;
+import eu.dnetlib.data.mdstore.modular.connector.MDStore;
+import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;
+import eu.dnetlib.enabling.resultset.ResultSetFactory;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Required;
+
+import javax.xml.ws.wsaddressing.W3CEndpointReference;
+import java.util.List;
+import java.util.Map;
+
+public class MDStoreRetriever {
+
+	/** Logger */
+	private static final Log log = LogFactory.getLog(MDStoreRetriever.class);
+
+	private MDStoreDao dao;
+
+	private ResultSetFactory resultSetFactory;
+
+	public W3CEndpointReference deliver(final String mdId, final String from, final String until, final String recordFilter) throws MDStoreServiceException {
+		final MDStore mdStore = dao.readMDStore(mdId);
+		return getResultSetFactory().createResultSet(mdStore.deliver(from, until, recordFilter));
+	}
+
+	public Iterable<String> deliver(final String format, final String layout, final String interpretation) throws MDStoreServiceException {
+		try {
+			return Iterables.concat(Iterables.transform(dao.listMDStores(format, layout, interpretation), new Function<String, Iterable<String>>() {
+
+				@Override
+				public Iterable<String> apply(final String mdId) {
+					log.debug("bulk deliver of mdId: " + mdId);
+					try {
+						return dao.readMDStore(mdId).iterate();
+					} catch (MDStoreServiceException e) {
+						throw new RuntimeException(e);
+					}
+				}
+			}));
+		} catch (RuntimeException e) {
+			throw new MDStoreServiceException(e);
+		}
+	}
+
+	public String deliverRecord(final String mdId, final String recordId) throws MDStoreServiceException, DocumentNotFoundException {
+		return dao.getMDStore(mdId).getRecord(recordId);
+	}
+
+	public MDStoreDao getDao() {
+		return dao;
+	}
+
+	@Required
+	public void setDao(final MDStoreDao dao) {
+		this.dao = dao;
+	}
+
+    public ResultSetFactory getResultSetFactory() {
+        return resultSetFactory;
+    }
+
+	@Required
+	public void setResultSetFactory(final ResultSetFactory resultSetFactory) {
+		this.resultSetFactory = resultSetFactory;
+	}
+
+    public List<String> getMDStoreRecords(final String mdId, int pageSize, final int offset, final Map<String, String> queryParam) throws MDStoreServiceException {
+        MDStore mdStore = dao.getMDStore(mdId);
+        return mdStore.deliver(mdId, pageSize, offset, queryParam);
+    }
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/MDStoreUtils.java

@@ -0,0 +1,81 @@
+package eu.dnetlib.data.mdstore.modular;
+
+import com.google.common.collect.Lists;
+import eu.dnetlib.data.mdstore.MDStoreServiceException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpDocumentNotFoundException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.w3c.dom.Document;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.xpath.XPath;
+import javax.xml.xpath.XPathConstants;
+import javax.xml.xpath.XPathExpression;
+import javax.xml.xpath.XPathFactory;
+import java.io.ByteArrayInputStream;
+import java.util.List;
+
+public class MDStoreUtils {
+
+    /**
+     * service locator.
+     */
+    @Autowired
+    private UniqueServiceLocator serviceLocator;
+
+    public List<MDFormatDescription> getField(final String format, final String layout) throws MDStoreServiceException {
+
+        String xquery = "for $x in collection('/db/DRIVER/MDFormatDSResources/MDFormatDSResourceType')/RESOURCE_PROFILE/BODY[CONFIGURATION/NAME='" + format + "'] return $x/STATUS/LAYOUTS/LAYOUT[@name='" + layout
+                + "']/FIELDS";
+
+        try {
+            String result = serviceLocator.getService(ISLookUpService.class).getResourceProfileByQuery(xquery);
+            final List<MDFormatDescription> mdformat = Lists.newArrayList();
+            DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
+            DocumentBuilder builder;
+            builder = factory.newDocumentBuilder();
+            Document doc = builder.parse(new ByteArrayInputStream(result.getBytes()));
+            XPathFactory xPathFactory = XPathFactory.newInstance();
+            XPath myXpath = xPathFactory.newXPath();
+            XPathExpression expression = myXpath.compile("//FIELD");
+            Object values = expression.evaluate(doc, XPathConstants.NODESET);
+            NodeList v = (NodeList) values;
+            for (int i = 0; i < v.getLength(); i++) {
+                Node currentItem = v.item(i);
+                NamedNodeMap attributes = currentItem.getAttributes();
+                String name = null;
+                String xpath = null;
+                if (attributes.getNamedItem("name") != null) {
+                    name = attributes.getNamedItem("name").getNodeValue();
+                }
+                if (attributes.getNamedItem("xpath") != null) {
+                    xpath = attributes.getNamedItem("xpath").getNodeValue();
+                }
+                if (attributes.getNamedItem("value") != null) {
+                    xpath = attributes.getNamedItem("value").getNodeValue();
+                }
+
+                MDFormatDescription currentMdFormat = new MDFormatDescription();
+                currentMdFormat.setName(name);
+                currentMdFormat.setXpath(xpath);
+                mdformat.add(currentMdFormat);
+            }
+            return mdformat;
+
+        } catch (ISLookUpDocumentNotFoundException e1) {
+            return null;
+        } catch (ISLookUpException e1) {
+            return null;
+        } catch (Exception e) {
+            throw new MDStoreServiceException("Error on retrieving field from mdformat", e);
+        }
+
+    }
+}
+

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/ModularMDStoreService.java

@@ -0,0 +1,123 @@
+package eu.dnetlib.data.mdstore.modular;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+import eu.dnetlib.data.mdstore.DocumentNotFoundException;
+import eu.dnetlib.data.mdstore.MDStoreService;
+import eu.dnetlib.data.mdstore.MDStoreServiceException;
+import eu.dnetlib.enabling.resultset.IterableResultSetFactory;
+import eu.dnetlib.enabling.tools.AbstractBaseService;
+import eu.dnetlib.enabling.tools.blackboard.NotificationHandler;
+import org.springframework.beans.factory.annotation.Required;
+
+import javax.xml.ws.wsaddressing.W3CEndpointReference;
+import java.util.List;
+import java.util.Map;
+
+public class ModularMDStoreService extends AbstractBaseService implements MDStoreService {
+
+	/**
+	 * notification handler.
+	 */
+	private NotificationHandler notificationHandler;
+
+	private MDStoreFeeder feeder;
+
+	private MDStoreRetriever retriever;
+
+	private IterableResultSetFactory iterableResultSetFactory;
+
+
+    public List<String> getMDStoreRecords(final String mdId, final int pageSize, final int offset, final Map<String, String> queryParam) throws MDStoreServiceException {
+
+        return retriever.getMDStoreRecords(mdId, pageSize, offset, queryParam);
+
+    }
+
+	@Override
+	public W3CEndpointReference deliverMDRecords(final String mdId, final String from, final String until, final String recordFilter)
+			throws MDStoreServiceException {
+		return retriever.deliver(mdId, from, until, recordFilter);
+	}
+
+	@Override
+	public W3CEndpointReference bulkDeliverMDRecords(final String format, final String layout, final String interpretation) throws MDStoreServiceException {
+		return getIterableResultSetFactory().createIterableResultSet(retriever.deliver(format, layout, interpretation));
+	}
+
+	@Override
+	public String deliverRecord(final String mdId, final String recordId) throws MDStoreServiceException, DocumentNotFoundException {
+		return retriever.deliverRecord(mdId, recordId);
+	}
+
+	@Override
+	public List<String> getListOfMDStores() throws MDStoreServiceException {
+		return (Lists.transform(retriever.getDao().listMDStores(), new Function<MDStoreDescription, String>() {
+
+			@Override
+			public String apply(final MDStoreDescription input) {
+				return input.getId();
+			}
+		}));
+	}
+
+	@Override
+	public List<String> listMDStores(final String format, final String layout, final String interpretation) throws MDStoreServiceException {
+		return retriever.getDao().listMDStores(format, layout, interpretation);
+	}
+
+	@Override
+	public void notify(final String subscriptionId, final String topic, final String isId, final String message) {
+		getNotificationHandler().notified(subscriptionId, topic, isId, message);
+	}
+
+	@Override
+	public boolean storeMDRecordsFromRS(final String mdId, final String rsId, final String storingType) throws MDStoreServiceException {
+		throw new MDStoreServiceException("not implemented, use the Blackboard asynchronous equivalent");
+	}
+
+	@Override
+	public int size(final String mdId) throws MDStoreServiceException {
+		return this.getRetriever().getDao().getCachedSize(mdId);
+	}
+
+	@Override
+	public int size(final String format, final String layout, final String interpretation) throws MDStoreServiceException {
+		return this.getRetriever().getDao().getSumOfSizes(format, layout, interpretation);
+	}
+
+	public NotificationHandler getNotificationHandler() {
+		return notificationHandler;
+	}
+
+	@Required
+	public void setNotificationHandler(final NotificationHandler notificationHandler) {
+		this.notificationHandler = notificationHandler;
+	}
+
+	public MDStoreFeeder getFeeder() {
+		return feeder;
+	}
+
+	public void setFeeder(final MDStoreFeeder feeder) {
+		this.feeder = feeder;
+	}
+
+	public MDStoreRetriever getRetriever() {
+		return retriever;
+	}
+
+	public void setRetriever(final MDStoreRetriever retriever) {
+		this.retriever = retriever;
+	}
+
+	public IterableResultSetFactory getIterableResultSetFactory() {
+		return iterableResultSetFactory;
+	}
+
+	@Required
+	public void setIterableResultSetFactory(final IterableResultSetFactory iterableResultSetFactory) {
+		this.iterableResultSetFactory = iterableResultSetFactory;
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/RecordParser.java

@@ -0,0 +1,24 @@
+package eu.dnetlib.data.mdstore.modular;
+
+import java.util.Map;
+
+import eu.dnetlib.miscutils.datetime.DateUtils;
+
+/**
+ * Parses a mdrecord and extracts the minimum information (like id, date etc) which is necessary for the mdstoring
+ * process.
+ * 
+ * @author marko
+ * 
+ */
+public interface RecordParser {
+
+	Map<String, String> parseRecord(String record);
+
+	void setTimestamp(long ts);
+
+	default long getTimestamp() {
+		return DateUtils.now();
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/RecordParserFactory.java

@@ -0,0 +1,30 @@
+package eu.dnetlib.data.mdstore.modular;
+
+import org.springframework.beans.factory.annotation.Required;
+
+import eu.dnetlib.miscutils.factory.Factory;
+
+public class RecordParserFactory implements Factory<RecordParser> {
+	
+	private Class<? extends RecordParser> parserType;
+
+	@Override
+	public RecordParser newInstance() {
+		try {
+			return getParserType().newInstance();
+		} catch (Throwable e) {
+			throw new RuntimeException(e);
+		} 	
+	}
+
+	@Required
+	public void setParserType(Class<? extends RecordParser> parserType) {
+		this.parserType = parserType;
+	}
+
+	public Class<? extends RecordParser> getParserType() {
+		return parserType;
+	}
+
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/SimpleRecordParser.java

@@ -0,0 +1,66 @@
+package eu.dnetlib.data.mdstore.modular;
+
+import java.io.StringReader;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.xml.xpath.XPath;
+import javax.xml.xpath.XPathFactory;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.xml.sax.InputSource;
+
+import static eu.dnetlib.data.mdstore.modular.MDStoreConstants.*;
+
+/**
+ * Terrible implementation of a record parser.
+ * 
+ * @author marko
+ *
+ */
+public class SimpleRecordParser implements RecordParser {
+	static final Log log = LogFactory.getLog(SimpleRecordParser.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	private long ts;
+
+	@Override
+	public Map<String, String> parseRecord(String record) {
+		Map<String, String> props = new HashMap<String, String>();
+		props.put(TIMESTAMP, String.valueOf(getTimestamp()));
+
+		try {
+//			DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
+			XPath xpath = XPathFactory.newInstance().newXPath();
+
+//			Document doc = builder.parse(new InputSource(new StringReader(record)));
+			InputSource doc = new InputSource(new StringReader(record));
+			
+			props.put(ID, xpath.evaluate("//*[local-name()='objIdentifier']", doc));
+			props.put("originalId", xpath.evaluate("//*[local-name()='efgEntity']/*/*[local-name()='identifier']", doc));
+			
+//			String date = xpath.evaluate("//*[local-name()='dateOfCollection'][1]", doc);
+//			props.put("date", new Date(date).getTime());	
+				
+		} catch (Exception e) {
+			log.warn("got exception while parsing document", e);
+			log.warn("record is:");
+			log.warn(record);
+			log.warn("------------");
+		}
+		return props;
+
+	}
+
+	@Override
+	public void setTimestamp(final long ts) {
+		this.ts = ts;
+		log.debug("RecordParser date set to "+ts);
+	}
+
+	@Override
+	public long getTimestamp() {
+		return ts;
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/StreamingRecordParser.java

@@ -0,0 +1,98 @@
+package eu.dnetlib.data.mdstore.modular;
+
+import java.io.ByteArrayInputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Stack;
+
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamConstants;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import static eu.dnetlib.data.mdstore.modular.MDStoreConstants.*;
+
+/**
+ * This method outperforms SimpleRecordParser by a vast amount, especially since we are just getting stuff in the
+ * header.
+ * 
+ * @author marko
+ * 
+ */
+public class StreamingRecordParser implements RecordParser {
+
+	private static final Log log = LogFactory.getLog(StreamingRecordParser.class);
+	private long ts;
+
+	@Override
+	public Map<String, String> parseRecord(String record) {
+
+		try {
+			XMLInputFactory factory = XMLInputFactory.newInstance();
+			XMLStreamReader parser = factory.createXMLStreamReader(new ByteArrayInputStream(record.getBytes()));
+
+			HashMap<String, String> res = new HashMap<String, String>();
+			res.put(TIMESTAMP, String.valueOf(getTimestamp()));
+
+			Stack<String> elementStack = new Stack<String>();
+			elementStack.push("/");
+
+			while (parser.hasNext()) {
+				int event = parser.next();
+
+				if (event == XMLStreamConstants.END_ELEMENT) {
+					elementStack.pop();
+				} else if (event == XMLStreamConstants.START_ELEMENT) {
+					final String localName = parser.getLocalName();
+					elementStack.push(localName);
+
+					if (OBJIDENTIFIER.equals(localName)) {
+						parser.next();
+
+						res.put(ID, parser.getText().trim());
+
+					} else if ("identifier".equals(localName) && "efgEntity".equals(grandParent(elementStack))) {
+						if (!res.containsKey("originalId")) {
+							parser.next();
+//							log.info("ZZZZZZ OK: found identifier at right depth " + elementStack);
+							res.put("originalId", parser.getText().trim());
+						}
+					}
+
+					else if ("identifier".equals(localName)) {
+
+//						log.info("ZZZZZZ: found identifier not at right depth " + elementStack + " grand parent " + grandParent(elementStack));
+					}
+
+					if (res.containsKey(ID) && res.containsKey("originalId"))
+						return res;
+				}
+			}
+			return res;
+		} catch (XMLStreamException e) {
+			throw new IllegalStateException(e);
+		}
+
+	}
+
+	private String grandParent(Stack<String> elementStack) {
+		if (elementStack.size() <= 3)
+			return "";
+		return elementStack.get(elementStack.size() - 3);
+	}
+
+	@Override
+	public void setTimestamp(final long ts) {
+		this.ts = ts;
+		log.debug("RecordParser date set to "+ts);
+	}
+
+	@Override
+	public long getTimestamp() {
+		return ts;
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/action/AbstractMDStoreAction.java

@@ -0,0 +1,125 @@
+package eu.dnetlib.data.mdstore.modular.action;
+
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+
+import javax.annotation.Resource;
+
+import org.antlr.stringtemplate.StringTemplate;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.core.io.ClassPathResource;
+
+import eu.dnetlib.data.mdstore.MDStoreServiceException;
+import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardServerAction;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;
+
+/**
+ * The Class AbstractMDStoreAction.
+ */
+public abstract class AbstractMDStoreAction implements BlackboardServerAction<MDStoreActions> {
+
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	/** The dao. */
+	private MDStoreDao dao;
+
+	/** The executor. */
+	private final Executor executor = Executors.newCachedThreadPool();
+
+	/** Logger */
+	private static final Log log = LogFactory.getLog(AbstractMDStoreAction.class);
+
+	private static final ClassPathResource mdstoreServiceStatusTemplate = new ClassPathResource(
+			"/eu/dnetlib/data/mdstore/modular/mdstoreServiceStatusTemplate.xml.st");
+
+	/**
+	 * Execute async.
+	 * 
+	 * @param handler
+	 *            the handler
+	 * @param job
+	 *            the job
+	 * @throws MDStoreServiceException
+	 *             the MD store service exception
+	 */
+	protected abstract void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) throws MDStoreServiceException;
+
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see
+	 * eu.dnetlib.enabling.tools.blackboard.BlackboardServerAction#execute(eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler,
+	 * eu.dnetlib.enabling.tools.blackboard.BlackboardJob)
+	 */
+	@Override
+	public void execute(final BlackboardServerHandler handler, final BlackboardJob job) {
+		executor.execute(new Runnable() {
+
+			@Override
+			public void run() {
+				try {
+					handler.ongoing(job);
+					executeAsync(handler, job);
+				} catch (MDStoreServiceException e) {
+					handler.failed(job, e);
+				}
+			}
+		});
+	}
+
+	protected void completeWithSuccess(final BlackboardServerHandler handler, final BlackboardJob job) {
+		// Don't change this synchronization rule
+		synchronized (this) {
+			updateMDStoreServiceProfile(job);
+			handler.done(job);
+		}
+	}
+
+	protected void completeWithFail(final BlackboardServerHandler handler, final BlackboardJob job, final Throwable e) {
+		// Don't change this synchronization rule
+		synchronized (this) {
+			updateMDStoreServiceProfile(job);
+			handler.failed(job, e);
+		}
+	}
+
+	private void updateMDStoreServiceProfile(final BlackboardJob job) {
+		final String id = job.getServiceId();
+
+		log.info("Updating mdstore service profile status, id: " + id);
+		try {
+			final StringTemplate st = new StringTemplate(IOUtils.toString(mdstoreServiceStatusTemplate.getInputStream()));
+			st.setAttribute("status", dao.getDBStatus());
+			serviceLocator.getService(ISRegistryService.class).updateProfileNode(id, "//STATUS", st.toString());
+		} catch (Exception e) {
+			log.error("Error upadating profile " + id, e);
+		}
+	}
+
+	/**
+	 * Gets the dao.
+	 * 
+	 * @return the dao
+	 */
+	public MDStoreDao getDao() {
+		return dao;
+	}
+
+	/**
+	 * Sets the dao.
+	 * 
+	 * @param dao
+	 *            the new dao
+	 */
+	public void setDao(final MDStoreDao dao) {
+		this.dao = dao;
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/action/CreateAction.java

@@ -0,0 +1,42 @@
+package eu.dnetlib.data.mdstore.modular.action;
+
+import eu.dnetlib.data.mdstore.MDStoreServiceException;
+import eu.dnetlib.data.mdstore.modular.MDStoreProfileCreator;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;
+import org.springframework.beans.factory.annotation.Required;
+
+public class CreateAction extends AbstractMDStoreAction {
+
+	private MDStoreProfileCreator profileCreator;
+
+	@Override
+	public void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) throws MDStoreServiceException {
+		final String format = job.getParameters().get("format");
+		final String interpretation = job.getParameters().get("interpretation");
+		final String layout = job.getParameters().get("layout");
+
+		try {
+			String mdId = profileCreator.registerProfile(format, interpretation, layout);
+
+			getDao().createMDStore(mdId, format, interpretation, layout);
+
+			job.getParameters().put("id", mdId);
+
+			completeWithSuccess(handler, job);
+		} catch (ISRegistryException e) {
+			throw new MDStoreServiceException(e);
+		}
+	}
+
+	public MDStoreProfileCreator getProfileCreator() {
+		return profileCreator;
+	}
+
+	@Required
+	public void setProfileCreator(final MDStoreProfileCreator profileCreator) {
+		this.profileCreator = profileCreator;
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/action/DeleteAction.java

@@ -0,0 +1,35 @@
+package eu.dnetlib.data.mdstore.modular.action;
+
+import eu.dnetlib.data.mdstore.MDStoreServiceException;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;
+import org.springframework.beans.factory.annotation.Required;
+
+public class DeleteAction extends AbstractMDStoreAction {
+
+	private UniqueServiceLocator serviceLocator;
+
+	@Override
+	public void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) throws MDStoreServiceException {
+		final String currentId = job.getParameters().get("id");
+		try {
+			serviceLocator.getService(ISRegistryService.class).deleteProfile(currentId);
+			getDao().deleteMDStore(currentId);
+			completeWithSuccess(handler, job);
+		} catch (Exception e) {
+			throw new MDStoreServiceException("Error deleting mdstore with id " + currentId, e);
+		}
+	}
+
+	public UniqueServiceLocator getServiceLocator() {
+		return serviceLocator;
+	}
+
+	@Required
+	public void setServiceLocator(final UniqueServiceLocator serviceLocator) {
+		this.serviceLocator = serviceLocator;
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/action/DoneCallback.java

@@ -0,0 +1,8 @@
+package eu.dnetlib.data.mdstore.modular.action;
+
+import java.util.Map;
+
+public interface DoneCallback {
+
+	void call(Map<String, String> params);
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/action/DummyPlugin.java

@@ -0,0 +1,23 @@
+package eu.dnetlib.data.mdstore.modular.action;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;
+
+public class DummyPlugin implements MDStorePlugin {
+
+	private static final Log log = LogFactory.getLog(DummyPlugin.class);
+
+	@Override
+	public void run(final MDStoreDao dao, final Map<String, String> params, final DoneCallback done) {
+		log.info("Nothing to do, I'm a dummy plugin. Bye.");
+		HashMap<String, String> res = new HashMap<String, String>();
+		res.put("ret", "cheers!");
+		done.call(res);
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/action/FailedCallback.java

@@ -0,0 +1,6 @@
+package eu.dnetlib.data.mdstore.modular.action;
+
+public interface FailedCallback {
+
+	void call(Throwable e);
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/action/FeedAction.java

@@ -0,0 +1,70 @@
+package eu.dnetlib.data.mdstore.modular.action;
+
+import java.util.List;
+
+import eu.dnetlib.data.mdstore.MDStoreServiceException;
+import eu.dnetlib.data.mdstore.modular.MDFormatDescription;
+import eu.dnetlib.data.mdstore.modular.MDStoreFeeder;
+import eu.dnetlib.data.mdstore.modular.MDStoreUtils;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Required;
+
+public class FeedAction extends AbstractMDStoreAction {
+
+	private static final Log log = LogFactory.getLog(FeedAction.class);
+    private static final String layoutIndex = "store";
+
+	private MDStoreFeeder feeder;
+
+    @Autowired
+    private MDStoreUtils mdstoreUtils;
+
+
+	@Override
+	public void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) throws MDStoreServiceException {
+
+		final String mdId = job.getParameters().get("mdId");
+		if ((mdId == null) || mdId.isEmpty()) throw new MDStoreServiceException("Blackboard param (mdId) is empty");
+
+		final String epr = job.getParameters().get("epr");
+		if ((epr == null) || epr.isEmpty()) throw new MDStoreServiceException("Blackboard param (mdId) is empty");
+
+		String storingType = job.getParameters().get("storingType");
+		if ((storingType == null) || storingType.isEmpty()) {
+			storingType = "REFRESH";
+		}
+        String format = feeder.getDao().getMDStore(mdId).getFormat();
+        List<MDFormatDescription> mdformats = mdstoreUtils.getField(format, layoutIndex);
+
+        if (mdformats != null) {
+            for (MDFormatDescription desc : mdformats) {
+                log.info("name: " + desc.getName());
+                log.info("xpath: " + desc.getXpath());
+            }
+        }
+
+
+        feeder.feed(mdId, epr, storingType, true, mdformats, params -> {
+	        job.getParameters().put("mdstoreSize", "" + params.get("mdstoreSize"));
+	        job.getParameters().put("writeOps", "" + params.get("writeOps"));
+	        completeWithSuccess(handler, job);
+        }, e -> {
+	        log.error("Error feeding mdstore: " + mdId, e);
+	        completeWithFail(handler, job, e);
+        });
+	}
+
+	public MDStoreFeeder getFeeder() {
+		return feeder;
+	}
+
+	@Required
+	public void setFeeder(final MDStoreFeeder feeder) {
+		this.feeder = feeder;
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/action/MDStoreActions.java

@@ -0,0 +1,11 @@
+/**
+ *
+ */
+package eu.dnetlib.data.mdstore.modular.action;
+
+public enum MDStoreActions {
+	CREATE,
+	DELETE,
+	FEED,
+	RUN_PLUGIN
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/action/MDStorePlugin.java

@@ -0,0 +1,12 @@
+package eu.dnetlib.data.mdstore.modular.action;
+
+import java.util.Map;
+
+import eu.dnetlib.data.mdstore.MDStoreServiceException;
+import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;
+
+public interface MDStorePlugin {
+
+	public void run(MDStoreDao dao, Map<String, String> params, DoneCallback doneCallback) throws MDStoreServiceException;
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/action/MDStorePluginEnumerator.java

@@ -0,0 +1,33 @@
+package eu.dnetlib.data.mdstore.modular.action;
+
+import java.util.Map;
+
+import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.BeanFactory;
+import org.springframework.beans.factory.BeanFactoryAware;
+import org.springframework.beans.factory.ListableBeanFactory;
+
+public class MDStorePluginEnumerator implements BeanFactoryAware {
+
+	// private static final Log log = LogFactory.getLog(MDStorePluginEnumerator.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	/**
+	 * bean factory.
+	 */
+	private ListableBeanFactory beanFactory;
+
+	@Override
+	public void setBeanFactory(final BeanFactory beanFactory) throws BeansException {
+		this.beanFactory = (ListableBeanFactory) beanFactory;
+	}
+
+	/**
+	 * Get all beans implementing the MDStorePlugin interface.
+	 *
+	 * @return
+	 */
+	public Map<String, MDStorePlugin> getAll() {
+		return beanFactory.getBeansOfType(MDStorePlugin.class);
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/action/PluginAction.java

@@ -0,0 +1,41 @@
+package eu.dnetlib.data.mdstore.modular.action;
+
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import eu.dnetlib.data.mdstore.MDStoreServiceException;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;
+
+public class PluginAction extends AbstractMDStoreAction {
+
+	private static final Log log = LogFactory.getLog(PluginAction.class);
+
+	@Autowired
+	private MDStorePluginEnumerator pluginEnumerator;
+
+	@Override
+	protected void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) throws MDStoreServiceException {
+
+		final String name = job.getParameters().get("plugin.name");
+
+		final Map<String, MDStorePlugin> plugins = pluginEnumerator.getAll();
+
+		if ((plugins == null) || plugins.isEmpty() || !plugins.containsKey(name)) throw new MDStoreServiceException("Unable to find plugin: " + name);
+
+		log.info("running MDStore plugin: " + name);
+
+		plugins.get(name).run(getDao(), job.getParameters(), new DoneCallback() {
+
+			@Override
+			public void call(final Map<String, String> params) {
+				job.getParameters().putAll(params);
+				handler.done(job);
+			}
+		});
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/connector/MDStore.java

@@ -0,0 +1,40 @@
+package eu.dnetlib.data.mdstore.modular.connector;
+
+import java.util.List;
+import java.util.Map;
+
+import eu.dnetlib.data.mdstore.DocumentNotFoundException;
+import eu.dnetlib.data.mdstore.MDStoreServiceException;
+import eu.dnetlib.data.mdstore.modular.MDFormatDescription;
+import eu.dnetlib.enabling.resultset.ResultSetListener;
+
+public interface MDStore {
+
+	String getId();
+
+	String getFormat();
+
+	String getInterpretation();
+
+	String getLayout();
+
+	void truncate();
+
+	int feed(Iterable<String> records, boolean incremental);
+
+    int feed(Iterable<String> records, boolean incremental, List<MDFormatDescription> mdformats);
+
+	ResultSetListener deliver(String from, String until, String recordFilter) throws MDStoreServiceException;
+
+	ResultSetListener deliverIds(String from, String until, String recordFilter) throws MDStoreServiceException;
+
+	Iterable<String> iterate();
+
+	int getSize();
+
+	void deleteRecord(String recordId);
+
+	String getRecord(String recordId) throws DocumentNotFoundException;
+
+    List<String> deliver(String mdId, int pageSize, int offset, Map<String, String> queryParam);
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/connector/MDStoreDBStatus.java

@@ -0,0 +1,49 @@
+package eu.dnetlib.data.mdstore.modular.connector;
+
+public class MDStoreDBStatus {
+
+	private int handledDatastructures;
+	private int usedDiskSpace;
+	private String date;
+
+	public MDStoreDBStatus() {}
+
+	public MDStoreDBStatus(final int handledDatastructures, final int usedDiskSpace, final String date) {
+		this.handledDatastructures = handledDatastructures;
+		this.usedDiskSpace = usedDiskSpace;
+		this.date = date;
+	}
+
+	public int getHandledDatastructures() {
+		return handledDatastructures;
+	}
+
+	public void setHandledDatastructures(final int handledDatastructures) {
+		this.handledDatastructures = handledDatastructures;
+	}
+
+	public int getUsedDiskSpace() {
+		return usedDiskSpace;
+	}
+
+	public void setUsedDiskSpace(final int usedDiskSpace) {
+		this.usedDiskSpace = usedDiskSpace;
+	}
+
+	public String getDate() {
+		return date;
+	}
+
+	public void setDate(final String date) {
+		this.date = date;
+	}
+
+	@Override
+	public String toString() {
+		return "MDStoreDBStatus{" +
+				"handledDatastructures=" + handledDatastructures +
+				", usedDiskSpace=" + usedDiskSpace +
+				", date='" + date + '\'' +
+				'}';
+	}
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/connector/MDStoreDao.java

@@ -0,0 +1,39 @@
+package eu.dnetlib.data.mdstore.modular.connector;
+
+import java.util.List;
+
+import eu.dnetlib.data.mdstore.MDStoreServiceException;
+import eu.dnetlib.data.mdstore.modular.MDStoreDescription;
+
+public interface MDStoreDao {
+
+	MDStore getMDStore(final String mdId) throws MDStoreServiceException;
+
+	MDStore readMDStore(final String mdId) throws MDStoreServiceException;
+
+	MDStore startTransaction(final String mdId, final boolean refresh) throws MDStoreServiceException;
+
+	void commit(final String transactionId, final String mdId) throws MDStoreServiceException;
+
+	List<MDStoreDescription> listMDStores() throws MDStoreServiceException;
+
+	List<String> listMDStores(final String format, final String layout, final String interpretation) throws MDStoreServiceException;
+
+	void createMDStore(final String mdId, final String format, final String interpretation, final String layout) throws MDStoreServiceException;
+
+	void deleteMDStore(final String id) throws MDStoreServiceException;
+
+	int getCachedSize(final String id) throws MDStoreServiceException;
+
+	void refreshSizes() throws MDStoreServiceException;
+
+	int refreshSize(final String id) throws MDStoreServiceException;
+
+	int getSumOfSizes(final String format, final String layout, final String interpretation) throws MDStoreServiceException;
+
+	void startGarbage() throws MDStoreServiceException;
+
+	void invalidTransaction(final String transactionId, final String mdId) throws MDStoreServiceException;
+
+	MDStoreDBStatus getDBStatus();
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/connector/MDStoreExpiredInfo.java

@@ -0,0 +1,39 @@
+package eu.dnetlib.data.mdstore.modular.connector;
+
+import java.util.Date;
+
+public class MDStoreExpiredInfo {
+
+	private String id;
+	private Date lastRead;
+
+	/**
+	 * @return the lastRead
+	 */
+	public Date getLastRead() {
+		return lastRead;
+	}
+
+	/**
+	 * @param lastRead
+	 *            the lastRead to set
+	 */
+	public void setLastRead(final Date lastRead) {
+		this.lastRead = lastRead;
+	}
+
+	/**
+	 * @return the id
+	 */
+	public String getId() {
+		return id;
+	}
+
+	/**
+	 * @param id
+	 *            the id to set
+	 */
+	public void setId(final String id) {
+		this.id = id;
+	}
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/connector/MDStoreManagerInfo.java

@@ -0,0 +1,59 @@
+package eu.dnetlib.data.mdstore.modular.connector;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class MDStoreManagerInfo {
+
+	private String mdId;
+	private String currentId;
+	private List<MDStoreExpiredInfo> stillUsed;
+	private List<MDStoreTransactionInfo> transactions;
+
+	public String getMdId() {
+		return mdId;
+	}
+
+	public void setMdId(final String mdId) {
+		this.mdId = mdId;
+	}
+
+	public String getCurrentId() {
+		return currentId;
+	}
+
+	public void setCurrentId(final String currentId) {
+		this.currentId = currentId;
+	}
+
+	public List<MDStoreExpiredInfo> getStillUsed() {
+		return stillUsed;
+	}
+
+	public void setStillUsed(final List<MDStoreExpiredInfo> stillUsed) {
+		this.stillUsed = stillUsed;
+	}
+
+	public List<MDStoreTransactionInfo> getTransactions() {
+		return transactions;
+	}
+
+	public void setTransactions(final List<MDStoreTransactionInfo> transactions) {
+		this.transactions = transactions;
+	}
+
+	public void addExpiredItem(final MDStoreExpiredInfo info) {
+		if (stillUsed == null) {
+			stillUsed = new ArrayList<MDStoreExpiredInfo>();
+		}
+		stillUsed.add(info);
+	}
+
+	public void addTransactionInfo(final MDStoreTransactionInfo info) {
+		if (transactions == null) {
+			transactions = new ArrayList<MDStoreTransactionInfo>();
+		}
+		transactions.add(info);
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/connector/MDStoreResults.java

@@ -0,0 +1,28 @@
+package eu.dnetlib.data.mdstore.modular.connector;
+
+public class MDStoreResults {
+	private Iterable<String> records;
+	private int size;
+
+	public MDStoreResults(Iterable<String> records, int size) {
+		super();
+		this.records = records;
+		this.size = size;
+	}
+
+	public int getSize() {
+		return size;
+	}
+
+	public void setSize(int size) {
+		this.size = size;
+	}
+
+	public Iterable<String> getRecords() {
+		return records;
+	}
+
+	public void setRecords(Iterable<String> records) {
+		this.records = records;
+	}
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/connector/MDStoreTransactionInfo.java

@@ -0,0 +1,72 @@
+package eu.dnetlib.data.mdstore.modular.connector;
+
+import java.util.Date;
+
+public class MDStoreTransactionInfo {
+
+	private String id;
+	private Boolean refresh;
+	private Date date;
+	private long size;
+
+	/**
+	 * @return the date
+	 */
+	public Date getDate() {
+		return date;
+	}
+
+	/**
+	 * @param date
+	 *            the date to set
+	 */
+	public void setDate(final Date date) {
+		this.date = date;
+	}
+
+	/**
+	 * @return the id
+	 */
+	public String getId() {
+		return id;
+	}
+
+	/**
+	 * @param id
+	 *            the id to set
+	 */
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	/**
+	 * @return the refresh
+	 */
+	public Boolean getRefresh() {
+		return refresh;
+	}
+
+	/**
+	 * @param refresh
+	 *            the refresh to set
+	 */
+	public void setRefresh(final Boolean refresh) {
+		this.refresh = refresh;
+	}
+
+	/**
+	 * @return the size
+	 */
+	public long getSize() {
+		return size;
+	}
+
+	/**
+	 * @param size
+	 *            the size to set
+	 */
+	public void setSize(final long size) {
+		this.size = size;
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/connector/MDStoreTransactionManager.java

@@ -0,0 +1,122 @@
+package eu.dnetlib.data.mdstore.modular.connector;
+
+import eu.dnetlib.data.mdstore.MDStoreServiceException;
+
+public interface MDStoreTransactionManager {
+
+	/**
+	 * Verify consistency.
+	 *
+	 * @throws MDStoreServiceException
+	 *             the MD store service exception
+	 */
+	public abstract void verifyConsistency() throws MDStoreServiceException;
+
+	/**
+	 * Creates the md store.
+	 *
+	 * @param mdId
+	 *            the md id
+	 * @throws MDStoreServiceException
+	 *             the MD store service exception
+	 */
+	public abstract void createMDStore(String mdId) throws MDStoreServiceException;
+
+	/**
+	 * Drop md store.
+	 *
+	 * @param mdId
+	 *            the md id
+	 * @throws MDStoreServiceException
+	 *             the MD store service exception
+	 */
+	public abstract void dropMDStore(String mdId) throws MDStoreServiceException;
+
+	/**
+	 * Gets the MD store collection.
+	 *
+	 * @param mdId
+	 *            the md id
+	 * @return the MD store collection
+	 * @throws MDStoreServiceException
+	 *             the MD store service exception
+	 */
+	public abstract String getMDStoreCollection(String mdId) throws MDStoreServiceException;
+
+	/**
+	 * Start a new transaction for writing in the mdstore this will create a temporary mdstore in which save the content. and after finished
+	 * switch it to the existing one
+	 *
+	 * @param mdId
+	 * @param refresh
+	 * @throws MDStoreServiceException
+	 *             if the mdStore Id doesn't exists in the metadata
+	 */
+	public abstract String startTransaction(String mdId, boolean refresh) throws MDStoreServiceException;
+
+	/**
+	 * Commit the transaction.
+	 *
+	 * @param transactionId
+	 *            the transaction id
+	 * @param mdstoreId
+	 *            the mdstore id
+	 * @param currentMDStore
+	 *            the current md store
+	 * @return true, if successful
+	 * @throws MDStoreServiceException
+	 *             the MD store service exception
+	 */
+	public abstract boolean commit(String transactionId, String mdstoreId, MDStore currentMDStore) throws MDStoreServiceException;
+
+	/**
+	 * Book a a current mdstore, so the transaction manager can't delete during a possible commit, until the resultset associate has not
+	 * expired
+	 *
+	 * @param mdStoreId
+	 * @return
+	 * @throws MDStoreServiceException
+	 */
+	public abstract String readMdStore(String mdStoreId) throws MDStoreServiceException;
+
+	/**
+	 * Return a JSON about the transaction assigned to a particular mdstore
+	 *
+	 * @param mdStoreId
+	 * @return
+	 * @throws MDStoreServiceException
+	 */
+	public abstract MDStoreManagerInfo getInfoForCurrentMdStore(String mdStoreId) throws MDStoreServiceException;
+
+	/**
+	 * Manually drop an old collection assigned to a particular mdStore
+	 *
+	 * @param mdId
+	 *            : the id of the mdStore
+	 * @param idToDrop
+	 *            : The id of the collection to be dropped
+	 * @return
+	 * @throws MDStoreServiceException
+	 */
+	public abstract Boolean dropUsed(String mdId, String idToDrop) throws MDStoreServiceException;
+
+	/**
+	 * Manually drop an old collection assigned to a particular mdStore
+	 *
+	 * @param mdId
+	 *            : the id of the mdStore
+	 * @param idToDrop
+	 *            : The id of the collection to be dropped
+	 * @return
+	 * @throws MDStoreServiceException
+	 */
+	public abstract Boolean dropTransaction(String mdId, String idToDrop) throws MDStoreServiceException;
+
+	/**
+	 * Start the garbage collection of the old mdstore not used
+	 *
+	 * @throws MDStoreServiceException
+	 */
+	public abstract void garbage() throws MDStoreServiceException;
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/inspector/MDStoreInspector.java

@@ -0,0 +1,213 @@
+package eu.dnetlib.data.mdstore.modular.inspector;
+
+import static eu.dnetlib.miscutils.collections.MappedCollection.listMap;
+
+import java.io.StringReader;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Resource;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.Document;
+import org.dom4j.DocumentException;
+import org.dom4j.io.SAXReader;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Controller;
+import org.springframework.ui.Model;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+
+import eu.dnetlib.data.mdstore.MDStoreServiceException;
+import eu.dnetlib.data.mdstore.modular.MDStoreDescription;
+import eu.dnetlib.data.mdstore.modular.MDStoreFeeder;
+import eu.dnetlib.data.mdstore.modular.connector.MDStore;
+import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;
+import eu.dnetlib.data.mdstore.modular.connector.MDStoreManagerInfo;
+import eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager;
+import eu.dnetlib.data.mdstore.modular.mongodb.MongoMDStore;
+import eu.dnetlib.enabling.inspector.AbstractInspectorController;
+import eu.dnetlib.enabling.resultset.ResultSetListener;
+import eu.dnetlib.miscutils.functional.xml.TryIndentXmlString;
+
+@Controller
+public class MDStoreInspector extends AbstractInspectorController {
+
+	private static final Log log = LogFactory.getLog(MDStoreInspector.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	@Resource(name = "mongodbMDStoreDao")
+	private MDStoreDao dao;
+
+	@Autowired
+	private MDStoreTransactionManager transactionManager;
+
+	@Resource
+	private MDStoreFeeder feeder;
+
+	@RequestMapping(value = "/inspector/mdstores.do")
+	public void mdstores(final Model model) throws MDStoreServiceException {
+		model.addAttribute("mdstores", dao.listMDStores());
+	}
+
+	@RequestMapping(value = "/inspector/mdstore.do", method = RequestMethod.GET)
+	public void mdstore(final Model model,
+			@RequestParam("id") final String id,
+			@RequestParam(value = "start", required = false) final Integer startParam,
+			@RequestParam(value = "regex", required = false) final String regex) throws MDStoreServiceException {
+		int pageSize = 10;
+		int start = 0;
+
+		if (startParam != null) {
+			start = startParam;
+		}
+
+		MongoMDStore mdstore = (MongoMDStore) dao.getMDStore(id);
+
+		ResultSetListener rs = mdstore.deliver(null, null, regex);
+		Function<String, Map<String, String>> function = new Function<String, Map<String, String>>() {
+
+			private SAXReader reader = new SAXReader();
+
+			@Override
+			public Map<String, String> apply(final String input) {
+				try {
+
+					Document doc = reader.read(new StringReader(input));
+					final HashMap<String, String> result = new HashMap<String, String>();
+					result.put("docId", doc.valueOf("//*[local-name()='objIdentifier']"));
+					result.put("body", input.replaceAll("<", "&lt;").replaceAll(">", "&gt;"));
+					result.put("mdId", id);
+					return result;
+				} catch (DocumentException e) {
+					throw new RuntimeException(e);
+				}
+
+			}
+		};
+		List<Map<String, String>> page = Lists.transform(listMap(rs.getResult(1 + start, start + pageSize), new TryIndentXmlString()), function);
+
+		model.addAttribute("id", id);
+		model.addAttribute("start", start);
+		model.addAttribute("regex", regex);
+		model.addAttribute("nextPage", start + pageSize);
+		model.addAttribute("prevPage", Math.max(0, start - pageSize));
+		model.addAttribute("size", rs.getSize());
+		model.addAttribute("page", page);
+	}
+
+	@RequestMapping(value = "/inspector/mdstore.do", method = RequestMethod.POST)
+	public String bulkReplace(final Model model,
+			@RequestParam("id") final String id,
+			@RequestParam("regex") final String regex,
+			@RequestParam("replace") final String replace,
+			@RequestParam(value = "checkReplace", required = false) final Boolean checkReplace) throws MDStoreServiceException {
+
+		log.debug("regex: " + regex);
+		log.debug("replace: " + replace);
+
+		MongoMDStore mdstore = (MongoMDStore) dao.getMDStore(id);
+
+		boolean replaceEnable = checkReplace != null && checkReplace == true;
+
+		if (replaceEnable) {
+			mdstore.replace(regex, replace);
+		} else {
+			model.addAttribute("regex", regex);
+		}
+
+		return "redirect:mdstore.do?id=" + id;
+	}
+
+	@RequestMapping(value = "/inspector/mdstoreEditResult.do")
+	public void mdstoreEditRecord(final Model model, @RequestParam("mdId") final String mdId, @RequestParam("docId") final String docId)
+			throws MDStoreServiceException {
+		MDStore mdstore = dao.getMDStore(mdId);
+		String record = mdstore.getRecord(docId);
+		log.debug("Displaying record for editing :" + record);
+		model.addAttribute("mdId", mdId);
+		TryIndentXmlString tryIndent = new TryIndentXmlString();
+		String escaped = tryIndent.evaluate(record);
+		model.addAttribute("body", escaped.replace("&", "&amp;"));
+	}
+
+	@RequestMapping(value = "/inspector/mdstoreSaveRecord.do")
+	public String mdstoreSaveRecord(final Model model, @RequestParam("mdId") final String mdId, @RequestParam("body") final String body)
+			throws MDStoreServiceException {
+		MDStore mdstore = dao.getMDStore(mdId);
+
+		mdstore.feed(Lists.newArrayList(body), true);
+
+		return "redirect:mdstore.do?id=" + mdId;
+
+	}
+
+	@RequestMapping(value = "/inspector/mdstoreDeleteRecord.do")
+	public String mdstoreDeleteRecord(final Model model, @RequestParam("mdId") final String mdId, @RequestParam("docId") final String docId)
+			throws MDStoreServiceException {
+		MDStore mdstore = dao.getMDStore(mdId);
+		log.info("deleting record " + docId);
+		mdstore.deleteRecord(docId);
+		return "redirect:mdstore.do?id=" + mdId;
+	}
+
+	@RequestMapping(value = "/inspector/mdstoresRefreshSizes.do")
+	public String mdstoresRefreshSizes(final Model model) throws MDStoreServiceException {
+
+		for (MDStoreDescription mdstore : dao.listMDStores()) {
+			feeder.touchSize(mdstore.getId(), dao.getMDStore(mdstore.getId()).getSize());
+		}
+
+		return "redirect:mdstores.do";
+	}
+
+	@RequestMapping(value = "/inspector/ensure.do")
+	public String mdstoreEnsureIndex(final Model model, @RequestParam("id") final String mdId) throws MDStoreServiceException {
+
+		MongoMDStore mdStore = (MongoMDStore) dao.getMDStore(mdId);
+
+		log.info("manual ensureIndex for mdId: " + mdId);
+		mdStore.ensureIndices();
+
+		return "redirect:mdstores.do";
+	}
+
+	@RequestMapping(value = "/inspector/infoTransaction.do")
+	public void mdstoreInfoTransaction(final Model model, @RequestParam("id") final String id) throws MDStoreServiceException {
+		MDStoreManagerInfo info = transactionManager.getInfoForCurrentMdStore(id);
+		model.addAttribute("info", info);
+	}
+
+	@RequestMapping(value = "/inspector/dropUsedCollection.do")
+	public String dropUsedCollection(final Model model, @RequestParam("mdId") final String mdId, @RequestParam("id") final String id)
+			throws MDStoreServiceException {
+		transactionManager.dropUsed(mdId, id);
+		return "redirect:mdstore.do?id=" + mdId;
+	}
+
+	@RequestMapping(value = "/inspector/invalidTransactionCollection.do")
+	public String invalidTransactionCollection(final Model model, @RequestParam("mdId") final String mdId, @RequestParam("id") final String id)
+			throws MDStoreServiceException {
+		transactionManager.dropTransaction(mdId, id);
+		return "redirect:mdstore.do?id=" + mdId;
+	}
+
+	@RequestMapping(value = "/inspector/refreshSizes.do")
+	public String refreshSizes(final Model model) throws MDStoreServiceException {
+
+		dao.refreshSizes();
+		return "redirect:mdstores.do";
+	}
+
+	@RequestMapping(value = "/inspector/doGarbage.do")
+	public String doGarbage(final Model model) throws MDStoreServiceException {
+
+		dao.startGarbage();
+		return "redirect:mdstores.do";
+	}
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/mongodb/MDStoreDaoImpl.java

@@ -0,0 +1,327 @@
+package eu.dnetlib.data.mdstore.modular.mongodb;
+
+import java.util.List;
+
+import com.google.common.collect.Lists;
+import com.google.common.primitives.Ints;
+import com.mongodb.BasicDBObject;
+import com.mongodb.BasicDBObjectBuilder;
+import com.mongodb.DBObject;
+import com.mongodb.client.AggregateIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.model.CreateCollectionOptions;
+import eu.dnetlib.data.mdstore.MDStoreServiceException;
+import eu.dnetlib.data.mdstore.modular.MDStoreDescription;
+import eu.dnetlib.data.mdstore.modular.RecordParser;
+import eu.dnetlib.data.mdstore.modular.RecordParserFactory;
+import eu.dnetlib.data.mdstore.modular.connector.MDStore;
+import eu.dnetlib.data.mdstore.modular.connector.MDStoreDBStatus;
+import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;
+import eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager;
+import eu.dnetlib.data.mdstore.modular.mongodb.utils.MDStoreUtils;
+import eu.dnetlib.miscutils.collections.FilteredCollection;
+import eu.dnetlib.miscutils.collections.MappedCollection;
+import eu.dnetlib.miscutils.datetime.DateUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.bson.Document;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Required;
+
+/**
+ * Factory bean for MongoMDStore instances.
+ *
+ * @author marko
+ */
+public class MDStoreDaoImpl implements MDStoreDao {
+
+	public static final String MD_ID = "mdId";
+	public static final String FORMAT = "format";
+	public static final String INTERPRETATION = "interpretation";
+	public static final String LAYOUT = "layout";
+	public static final String SIZE = "size";
+	public static final String METADATA_NAME = "metadata";
+	private static final Log log = LogFactory.getLog(MDStoreDaoImpl.class);
+	private MongoDatabase db;
+
+	private RecordParserFactory recordParserFactory;
+
+	private boolean discardRecords = true;
+
+	@Autowired
+	private MDStoreTransactionManager transactionManager;
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	public void createMDStore(final String mdId, final String format, final String interpretation, final String layout) throws MDStoreServiceException {
+		transactionManager.createMDStore(mdId);
+		final String internalId = transactionManager.getMDStoreCollection(mdId);
+
+		if (!Lists.newArrayList(getDb().listCollectionNames()).contains(internalId)) {
+			log.info(String.format("creating collection %s", internalId));
+			getDb().createCollection(internalId, new CreateCollectionOptions());
+		}
+		final MongoCollection<DBObject> coll = getDb().getCollection(METADATA_NAME, DBObject.class);
+		final BasicDBObject obj = new BasicDBObject();
+		obj.put(MD_ID, mdId);
+		obj.put(FORMAT, format);
+		obj.put(INTERPRETATION, interpretation);
+		obj.put(LAYOUT, layout);
+		obj.put(SIZE, 0);
+		coll.insertOne(obj);
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	public void deleteMDStore(final String mdId) throws MDStoreServiceException {
+		final MongoCollection<DBObject> metadata = getDb().getCollection(METADATA_NAME, DBObject.class);
+		if (metadata == null) throw new MDStoreServiceException("cannot find metadata collection");
+		transactionManager.dropMDStore(mdId);
+		metadata.deleteOne(new BasicDBObject(MD_ID, mdId));
+		log.info("deleted mdId: " + mdId);
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	public MDStore getMDStore(final String mdId) throws MDStoreServiceException {
+		final String internalId = transactionManager.getMDStoreCollection(mdId);
+		return new MongoMDStore(mdId, getDb().getCollection(internalId, DBObject.class), getRecordParser(), isDiscardRecords(), getDb());
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	public MDStore readMDStore(final String mdId) throws MDStoreServiceException {
+		final String internalId = transactionManager.readMdStore(mdId);
+		return new MongoMDStore(mdId, getDb().getCollection(internalId, DBObject.class), getRecordParser(), isDiscardRecords(), getDb());
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	public MDStore startTransaction(final String mdId, final boolean refresh) throws MDStoreServiceException {
+		final String transactionId = transactionManager.startTransaction(mdId, refresh);
+		return new MongoMDStore(transactionId, getDb().getCollection(transactionId, DBObject.class), getRecordParser(), isDiscardRecords(),
+				getDb());
+	}
+
+	private RecordParser getRecordParser() {
+		final RecordParser parser = getRecordParserFactory().newInstance();
+		parser.setTimestamp(DateUtils.now());
+		return parser;
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	public List<MDStoreDescription> listMDStores() {
+		return MappedCollection.listMap(getDb().getCollection(METADATA_NAME, DBObject.class).find(), input -> {
+
+			final String mdId = (String) input.get(MD_ID);
+			log.debug("Getting info for " + mdId);
+			final String format = (String) input.get(FORMAT);
+			final String layout = (String) input.get(LAYOUT);
+			final String interpretation = (String) input.get(INTERPRETATION);
+			MongoMDStore currentMDStore = null;
+			final MDStoreDescription description = new MDStoreDescription();
+			try {
+				currentMDStore = (MongoMDStore) getMDStore(mdId);
+			} catch (final MDStoreServiceException e) {
+				log.error("Error on retrieving mdstore for getting info mdId " + mdId);
+			}
+
+			int size = 0;
+			if (input.containsField(SIZE)) {
+				log.debug("Size retrieved from metadata for mdId :" + mdId);
+				size = (Integer) input.get(SIZE);
+			} else {
+				if (currentMDStore != null) {
+					log.debug("Size not Found in metadata for mdId :" + mdId + " calling getCount ");
+					size = currentMDStore.getSize();
+					input.put("size", size);
+					getDb().getCollection(METADATA_NAME, DBObject.class).findOneAndReplace(new BasicDBObject(MD_ID, mdId), input);
+				}
+			}
+			if (currentMDStore != null) {
+				description.setIndexed(currentMDStore.isIndexed());
+			}
+			description.setId(mdId);
+			description.setFormat(format);
+			description.setLayout(layout);
+			description.setInterpretation(interpretation);
+			description.setSize(size);
+			return description;
+		});
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	public List<String> listMDStores(final String format, final String layout, final String interpretation) {
+		return MappedCollection.listMap(
+				FilteredCollection.listFilter(getDb().getCollection(METADATA_NAME, DBObject.class).find(), MDStoreUtils.dboFilter(format, layout, interpretation)),
+				MDStoreUtils.mdId());
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	public int getCachedSize(final String id) throws MDStoreServiceException {
+		log.debug("retrieve cached size for mdstore: " + id);
+		final DBObject desc = getDb().getCollection(METADATA_NAME, DBObject.class).find(new BasicDBObject(MD_ID, id)).first();
+		if (!desc.containsField(SIZE)) {
+			desc.put(SIZE, getMDStore(id).getSize());
+		}
+
+		final Object oSize = desc.get(SIZE);
+		return (Integer) oSize;
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	public void refreshSizes() throws MDStoreServiceException {
+		for (final MDStoreDescription mdStoreId : listMDStores()) {
+			refreshSize(mdStoreId.getId());
+		}
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	public int refreshSize(final String mdStoreId) throws MDStoreServiceException {
+		final int size = (int) getDb().getCollection(transactionManager.getMDStoreCollection(mdStoreId)).count();
+		final MongoCollection<DBObject> metadata = getDb().getCollection(METADATA_NAME, DBObject.class);
+		metadata.updateOne(new BasicDBObject(MD_ID, mdStoreId), new BasicDBObject("$set", new BasicDBObject(SIZE, size)));
+		return size;
+	}
+
+	@Override
+	public int getSumOfSizes(final String format, final String layout, final String interpretation) throws MDStoreServiceException {
+		final MongoCollection<DBObject> metadata = getDb().getCollection(METADATA_NAME, DBObject.class);
+		BasicDBObject matchObj = (BasicDBObject) BasicDBObjectBuilder.start("$match",
+				BasicDBObjectBuilder.start("format", format).add("layout", layout).add("interpretation", interpretation).get()).get();
+		BasicDBObject groupObj = (BasicDBObject) BasicDBObjectBuilder.start("$group",
+				BasicDBObjectBuilder.start("_id", "").add("total", new BasicDBObject("$sum", "$" + SIZE)).get()).get();
+		BasicDBObject projectObj = new BasicDBObject("$project", new BasicDBObject("_id", 0).append("total", 1));
+		List<BasicDBObject> pipeline = Lists.newArrayList(matchObj, groupObj, projectObj);
+		AggregateIterable<DBObject> output = metadata.aggregate(pipeline, DBObject.class);
+		DBObject result = output.first();
+		if (result == null || !result.containsField("total")) {
+			log.debug("No total found");
+			return 0;
+		} else return (Integer) result.get("total");
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	public void commit(final String transactionId, final String mdId) throws MDStoreServiceException {
+		transactionManager.commit(transactionId, mdId, getMDStore(mdId));
+	}
+
+	/**
+	 * Getter for property 'db'.
+	 *
+	 * @return Value for property 'db'.
+	 */
+	public MongoDatabase getDb() {
+		return db;
+	}
+
+	/**
+	 * Setter for property 'db'.
+	 *
+	 * @param db
+	 *            Value to set for property 'db'.
+	 */
+	@Required
+	public void setDb(final MongoDatabase db) {
+		this.db = db;
+	}
+
+	/**
+	 * Getter for property 'recordParser'.
+	 *
+	 * @return Value for property 'recordParser'.
+	 */
+	public RecordParserFactory getRecordParserFactory() {
+		return recordParserFactory;
+	}
+
+
+	@Required
+	public void setRecordParserFactory(final RecordParserFactory recordParserFactory) {
+		this.recordParserFactory = recordParserFactory;
+	}
+
+	/**
+	 * Getter for property 'discardRecords'.
+	 *
+	 * @return Value for property 'discardRecords'.
+	 */
+	public boolean isDiscardRecords() {
+		return discardRecords;
+	}
+
+	/**
+	 * Setter for property 'discardRecords'.
+	 *
+	 * @param discardRecords
+	 *            Value to set for property 'discardRecords'.
+	 */
+	public void setDiscardRecords(final boolean discardRecords) {
+		this.discardRecords = discardRecords;
+	}
+
+	@Override
+	public MDStoreDBStatus getDBStatus() {
+		final int handledDatastructures = Ints.saturatedCast(getDb().getCollection(METADATA_NAME).count());
+		//final int usedDiskSpace = Ints.saturatedCast(getDb().getStats().getLong("storageSize") / (1024 * 1024)); // in MB
+		//{ dbStats: 1, scale: 1 }
+		BasicDBObject statsQuery = new BasicDBObject("dbStats", 1);
+		statsQuery.put("scale", 1024 * 1024); //storageSize in MB
+		final Document statsRes = getDb().runCommand(statsQuery);
+		log.debug("DBStatus --  " + statsRes.toJson());
+		int usedDiskSpace;
+		//trying to handle different versions of the mongo server: old version returns storage size as long, new version as double
+		//TODO: simplify this when dev, beta, production are aligned with our local, latest, mongo version
+		String usedDiskSpaceStr = statsRes.get("storageSize").toString();
+		try {
+			Long usedDiskSpaceLong = Long.parseLong(usedDiskSpaceStr);
+			usedDiskSpace = Ints.saturatedCast(usedDiskSpaceLong);
+		} catch (NumberFormatException nfe) {
+			Double usedDiskSpaceDbl = Double.parseDouble(usedDiskSpaceStr);
+			usedDiskSpace = usedDiskSpaceDbl.intValue();
+		}
+		final String date = DateUtils.now_ISO8601();
+		return new MDStoreDBStatus(handledDatastructures, usedDiskSpace, date);
+	}
+
+	@Override
+	public void startGarbage() throws MDStoreServiceException {
+		this.transactionManager.garbage();
+	}
+
+	@Override
+	public void invalidTransaction(final String transactionId, final String mdId) throws MDStoreServiceException {
+		transactionManager.dropTransaction(mdId, transactionId);
+
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/mongodb/MDStoreTransactionManagerImpl.java

@@ -0,0 +1,742 @@
+package eu.dnetlib.data.mdstore.modular.mongodb;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+import com.google.common.collect.Lists;
+import com.mongodb.BasicDBList;
+import com.mongodb.BasicDBObject;
+import com.mongodb.DBObject;
+import com.mongodb.WriteConcern;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.MongoIterable;
+import com.mongodb.client.model.Filters;
+import com.mongodb.client.model.IndexOptions;
+import com.mongodb.client.model.UpdateOptions;
+import eu.dnetlib.data.mdstore.MDStoreServiceException;
+import eu.dnetlib.data.mdstore.modular.connector.*;
+import eu.dnetlib.data.mdstore.modular.mongodb.utils.MDStoreUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.bson.conversions.Bson;
+import org.joda.time.DateTime;
+import org.joda.time.Days;
+import org.springframework.beans.factory.annotation.Required;
+
+/**
+ * The Class MDStoreTransactionManager.
+ */
+public class MDStoreTransactionManagerImpl implements MDStoreTransactionManager {
+
+	/** The Constant log. */
+	private static final Log log = LogFactory.getLog(MDStoreTransactionManagerImpl.class);
+
+	/**
+	 * The table name.
+	 */
+	private static String TABLE_NAME = "metadataManager";
+
+	/** The max number of concurrent transactions per mdstore. */
+	private int maxTransactions = 1;
+
+	/**
+	 * The db.
+	 */
+	private MongoDatabase db;
+
+	/**
+	 * The manager table.
+	 */
+	private MongoCollection<DBObject> managerTable;
+
+	/** The expired days. */
+	private int expiredDays;
+
+	private final IndexOptions options = new IndexOptions().background(true);
+
+	/**
+	 * Bootstrap manager.
+	 */
+	private void bootstrapManager() {
+		log.debug("Bootstrap Manager start");
+		final MongoCollection<DBObject> metadataColl = db.getCollection("metadata", DBObject.class);
+		final FindIterable<DBObject> values = metadataColl.find();
+		this.setManagerTable(db.getCollection(TABLE_NAME, DBObject.class));
+		for (DBObject object : values) {
+			final String id = (String) object.get("mdId");
+			String newId = id;
+			if (id.contains("_")) {
+				newId = StringUtils.substringBefore(id, "_");
+			}
+			final BasicDBObject input = new BasicDBObject();
+			input.put("mdId", id);
+			input.put("currentId", newId);
+			input.put("expiring", new String[] {});
+			input.put("transactions", new String[] {});
+			getManagerTable().insertOne(input);
+			log.debug(String.format("Added %s to Metadata Manager data structure", id));
+
+		}
+		final BasicDBObject ensureIndex = new BasicDBObject();
+		ensureIndex.put("mdId", 1);
+		log.debug("Create index in MetadaManager ");
+		this.getManagerTable().createIndex(ensureIndex, options);
+	}
+
+	/**
+	 * Gets the DBObject describing an mdstore. null if there is no mdstore with the given id.
+	 *
+	 * @param mdstoreID the mdStore identifier
+	 * @return DBObject or null
+	 */
+	private DBObject getMDStoreAsDBObject(final String mdstoreID) throws MDStoreServiceException {
+		final BasicDBObject query = new BasicDBObject();
+		query.put("mdId", mdstoreID);
+		final FindIterable<DBObject> it = this.getManagerTable().find(query);
+		return it.first();
+	}
+
+	/**
+	 * Verify consistency.
+	 *
+	 * @throws MDStoreServiceException
+	 *             the MD store service exception
+	 */
+	@Override
+	public void verifyConsistency() throws MDStoreServiceException {
+		if (this.getManagerTable() == null) {
+			if (!Lists.newArrayList(db.listCollectionNames()).contains(TABLE_NAME))
+				bootstrapManager();
+			else {
+				this.setManagerTable(db.getCollection(TABLE_NAME, DBObject.class));
+			}
+		}
+		if (this.getManagerTable() == null) throw new MDStoreServiceException("Something bad happen, unable to create managerTable");
+	}
+
+	/**
+	 * {@inheritDoc}
+	 *
+	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#createMDStore(java.lang.String)
+	 */
+	@Override
+	public void createMDStore(final String mdId) throws MDStoreServiceException {
+		log.debug("Creating new mdstore");
+		verifyConsistency();
+		String newId = mdId;
+		if (mdId.contains("_")) {
+			newId = StringUtils.substringBefore(mdId, "_");
+		}
+		final BasicDBObject instance = new BasicDBObject();
+		instance.put("mdId", mdId);
+		instance.put("currentId", newId);
+		instance.put("expiring", new String[] {});
+		getManagerTable().insertOne(instance);
+	}
+
+	/**
+	 * {@inheritDoc}
+	 *
+	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#dropMDStore(java.lang.String)
+	 */
+	@Override
+	public void dropMDStore(final String mdId) throws MDStoreServiceException {
+		verifyConsistency();
+		log.debug("Droping MDStore: " + mdId);
+		final BasicDBObject query = new BasicDBObject();
+		query.put("mdId", mdId);
+		final DBObject dropped = this.getManagerTable().findOneAndDelete(query);
+		garbage();
+		final String collectionName = (String) dropped.get("currentId");
+		this.db.getCollection(collectionName).drop();
+		this.db.getCollection("discarded-" + collectionName).drop();
+	}
+
+	/**
+	 * {@inheritDoc}
+	 *
+	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#getMDStoreCollection(java.lang.String)
+	 */
+	@Override
+	public String getMDStoreCollection(final String mdId) throws MDStoreServiceException {
+		verifyConsistency();
+		DBObject mdstoreInfo = getMDStoreAsDBObject(mdId);
+		if (mdstoreInfo != null)
+			return (String) mdstoreInfo.get("currentId");
+		else return null;
+	}
+
+	/**
+	 * {@inheritDoc}
+	 *
+	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#startTransaction(java.lang.String, boolean)
+	 */
+	@Override
+	public String startTransaction(final String mdId, final boolean refresh) throws MDStoreServiceException {
+		verifyConsistency();
+		log.info("Start transaction for metadata store " + mdId);
+		final DBObject mdstoreInfo = getMDStoreAsDBObject(mdId);
+		if (mdstoreInfo == null) throw new MDStoreServiceException("Error, unable to find Mdstore with Id " + mdId);
+		String idCreation = StringUtils.substringBefore(mdId, "_");
+		idCreation = idCreation + "::" + System.currentTimeMillis();
+
+		BasicDBList values;
+		if (mdstoreInfo.containsField("transactions")) {
+			values = (BasicDBList) mdstoreInfo.get("transactions");
+			if (values.size() > getMaxTransactions())
+				throw new MDStoreServiceException("Cannot create more than " + getMaxTransactions() + " transactions, found: " + values.size() + ", mdId:"
+						+ mdId);
+		} else {
+			values = new BasicDBList();
+		}
+		final BasicDBObject transactionMetadata = new BasicDBObject();
+		transactionMetadata.put("id", idCreation);
+		transactionMetadata.put("refresh", refresh);
+		transactionMetadata.put("date", new Date());
+		values.add(transactionMetadata);
+		mdstoreInfo.put("transactions", values);
+		this.getManagerTable().findOneAndReplace(new BasicDBObject("_id", mdstoreInfo.get("_id")), mdstoreInfo);
+		return idCreation;
+	}
+
+	/**
+	 * {@inheritDoc}
+	 *
+	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#commit(java.lang.String, java.lang.String,
+	 *      eu.dnetlib.data.mdstore.modular.connector.MDStore)
+	 */
+	@Override
+	public boolean commit(final String transactionId, final String mdstoreId, final MDStore current) throws MDStoreServiceException {
+		verifyConsistency();
+		final DBObject mdstoreInfo = getMDStoreAsDBObject(mdstoreId);
+		if (mdstoreInfo == null) throw new MDStoreServiceException("Error, unable to find Mdstore with Id " + mdstoreId);
+		final BasicDBList transactions = (BasicDBList) mdstoreInfo.get("transactions");
+		final DBObject transaction = findTransaction(transactions, transactionId);
+		if (transaction == null) throw new MDStoreServiceException("Error, unable to find transaction with Id " + transactionId);
+		final boolean refresh = (Boolean) transaction.get("refresh");
+		transactions.remove(transaction);
+		final String oldId = (String) mdstoreInfo.get("currentId");
+		if (refresh) {
+			mdstoreInfo.put("currentId", transactionId);
+			final BasicDBList stillUsed = (BasicDBList) mdstoreInfo.get("expiring");
+			if (stillUsed.size() == 0) {
+				db.getCollection(oldId).drop();
+				db.getCollection("discarded-" + oldId).drop();
+			}
+			log.debug("Replaced collection ");
+		} else {
+			log.debug("commit incremental ");
+			updateIncremental(transactionId, oldId);
+			db.getCollection(transactionId).drop();
+			db.getCollection("discarded-" + transactionId).drop();
+		}
+		this.getManagerTable().findOneAndReplace(new BasicDBObject("_id", mdstoreInfo.get("_id")), mdstoreInfo);
+
+		log.info("Committed transaction for metadata store " + mdstoreId);
+		return true;
+	}
+
+	/**
+	 * Find transaction.
+	 *
+	 * @param transactions
+	 *            the transactions
+	 * @param transactionId
+	 *            the transaction id
+	 * @return the DB object
+	 */
+	private DBObject findTransaction(final BasicDBList transactions, final String transactionId) {
+		if (transactions.size() == 0) return null;
+		for (Object tx : transactions) {
+			final BasicDBObject transaction = (BasicDBObject) tx;
+			final String id = (String) transaction.get("id");
+			if (transactionId.equals(id)) return transaction;
+		}
+		return null;
+
+	}
+
+	/**
+	 * Gets the db.
+	 *
+	 * @return the db
+	 */
+	public MongoDatabase getDb() {
+		return db;
+	}
+
+	/**
+	 * Sets the db.
+	 *
+	 * @param db
+	 *            the db to set
+	 */
+	@Required
+	public void setDb(final MongoDatabase db) {
+		this.db = db;
+	}
+
+	/**
+	 * {@inheritDoc}
+	 *
+	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#readMdStore(java.lang.String)
+	 */
+	@Override
+	public String readMdStore(final String mdStoreId) throws MDStoreServiceException {
+		verifyConsistency();
+		final DBObject mdstoreInfo = getMDStoreAsDBObject(mdStoreId);
+		if (mdstoreInfo == null) throw new MDStoreServiceException("Error, unable to find Mdstore with Id " + mdStoreId);
+		final String currentId = (String) mdstoreInfo.get("currentId");
+		final BasicDBList values = (BasicDBList) mdstoreInfo.get("expiring");
+		updateMdstoreUsed(values, currentId);
+		this.getManagerTable().findOneAndReplace(new BasicDBObject("_id", mdstoreInfo.get("_id")), mdstoreInfo);
+		return currentId;
+
+	}
+
+
+	/**
+	 * Update mdstore used.
+	 *
+	 * @param values
+	 *            the values
+	 * @param mdId
+	 *            the md id
+	 */
+	private void updateMdstoreUsed(final BasicDBList values, final String mdId) {
+		if (values.size() > 0) {
+			for (Object value : values) {
+				final DBObject obj = (DBObject) value;
+				final String id = (String) obj.get("id");
+				if (mdId.equals(id)) {
+					obj.put("lastRead", new Date());
+					return;
+				}
+			}
+		}
+		final BasicDBObject readStore = new BasicDBObject();
+		readStore.put("id", mdId);
+		readStore.put("lastRead", new Date());
+		values.add(readStore);
+	}
+
+	/**
+	 * Gets the manager table.
+	 *
+	 * @return the managerTable
+	 */
+	public MongoCollection<DBObject> getManagerTable() {
+		return managerTable;
+	}
+
+	/**
+	 * Sets the manager table.
+	 *
+	 * @param managerTable
+	 *            the managerTable to set
+	 */
+	public void setManagerTable(final MongoCollection<DBObject> managerTable) {
+		this.managerTable = managerTable;
+	}
+
+	/*
+	 * (non-Javadoc)
+	 *
+	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#getInfoForCurrentMdStore(java.lang.String)
+	 */
+	@Override
+	public MDStoreManagerInfo getInfoForCurrentMdStore(final String mdStoreId) throws MDStoreServiceException {
+		verifyConsistency();
+		final DBObject mdstoreInfo = getMDStoreAsDBObject(mdStoreId);
+		if (mdstoreInfo == null) throw new MDStoreServiceException("Error, unable to find Mdstore with Id " + mdStoreId);
+		final MDStoreManagerInfo result = new MDStoreManagerInfo();
+		result.setCurrentId((String) mdstoreInfo.get("currentId"));
+		result.setMdId((String) mdstoreInfo.get("mdId"));
+		final BasicDBList values = (BasicDBList) mdstoreInfo.get("expiring");
+		for (Object v : values) {
+			final MDStoreExpiredInfo stillused = new MDStoreExpiredInfo();
+			final DBObject value = (DBObject) v;
+			stillused.setId((String) value.get("id"));
+			stillused.setLastRead((Date) value.get("lastRead"));
+			result.addExpiredItem(stillused);
+		}
+		final BasicDBList transactions = (BasicDBList) mdstoreInfo.get("transactions");
+		if (transactions != null) {
+			for (Object tx : transactions) {
+				final MDStoreTransactionInfo transaction = new MDStoreTransactionInfo();
+				final DBObject value = (DBObject) tx;
+				final String transactionId = (String) value.get("id");
+				transaction.setId(transactionId);
+				transaction.setDate((Date) value.get("date"));
+				transaction.setRefresh((Boolean) value.get("refresh"));
+				transaction.setSize(db.getCollection(transactionId).count());
+				result.addTransactionInfo(transaction);
+			}
+		}
+		return result;
+	}
+
+	/*
+	 * (non-Javadoc)
+	 *
+	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#dropUsed(java.lang.String, java.lang.String)
+	 */
+	@Override
+	public Boolean dropUsed(final String mdId, final String idToDrop) throws MDStoreServiceException {
+		verifyConsistency();
+		final DBObject mdstoreInfo = getMDStoreAsDBObject(mdId);
+		if (mdstoreInfo == null) throw new MDStoreServiceException("Error, unable to find Mdstore with Id " + mdId);
+		return dropStore(mdstoreInfo, idToDrop, "expiring");
+	}
+
+	private boolean dropStore(DBObject mdstoreInfo, final String idToDrop, String transactionListName) throws MDStoreServiceException {
+		final BasicDBList transactionList = (BasicDBList) mdstoreInfo.get(transactionListName);
+		for (int i = 0; i < transactionList.size(); i++) {
+			final DBObject value = (DBObject) transactionList.get(i);
+			final String currentUsedId = (String) value.get("id");
+			if (currentUsedId.equals(idToDrop)) {
+				db.getCollection(idToDrop).drop();
+				db.getCollection("discarded-" + idToDrop).drop();
+				transactionList.remove(value);
+				mdstoreInfo.put(transactionListName, transactionList);
+				this.getManagerTable().findOneAndReplace(new BasicDBObject("_id", mdstoreInfo.get("_id")), mdstoreInfo);
+				return true;
+			}
+		}
+		throw new MDStoreServiceException("Error, unable to drop collection " + idToDrop);
+	}
+
+	/*
+	 * (non-Javadoc)
+	 *
+	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#garbage()
+	 */
+	@Override
+	public void garbage() throws MDStoreServiceException {
+		verifyConsistency();
+		log.info("Start garbage collection of MdStore");
+		final FindIterable<DBObject> it = this.managerTable.find();
+		int totalDeleted = 0;
+		for (DBObject currentObject :  it){
+			if (log.isDebugEnabled()) {
+				log.debug("start to check id: " + currentObject.get("currentId"));
+			}
+			garbageExpiring(currentObject, (String) currentObject.get("currentId"));
+			garbageTransactions(currentObject, (String) currentObject.get("currentId"));
+			this.getManagerTable().findOneAndReplace(new BasicDBObject("_id", currentObject.get("_id")), currentObject);
+		}
+
+		// DELETING Collection that are not in the metadataManager table
+		MongoIterable<String> collections = this.db.listCollectionNames();
+		for (String collection : collections) {
+			if ((collection.length() > 30) && (!collection.contains("discarded-"))) {
+				final DBObject item = getMetadataObjectForCollections(collection);
+
+				if (shouldDelete(collection, item)) {
+					if (log.isDebugEnabled()) {
+						log.debug("delete collection: " + collection + " from mongo");
+					}
+					db.getCollection(collection).drop();
+					db.getCollection("discarded-" + collection).drop();
+					if (log.isDebugEnabled()) {
+						log.debug("delete collection: discarded-" + collection + " from mongo");
+					}
+				}
+			}
+		}
+
+		log.info("Complete garbage collection of MdStore, total store deleted: " + totalDeleted);
+	}
+
+	private DBObject getMetadataObjectForCollections(final String collectionName) {
+		if (collectionName == null) return null;
+		final String postfix = "_TURTdG9yZURTUmVzb3VyY2VzL01EU3RvcmVEU1Jlc291cmNlVHlwZQ==";
+		final String tmp = collectionName.contains("discarded-") ? StringUtils.substringAfter(collectionName, "discarded-") : collectionName;
+		final String collectionNameCleaned = StringUtils.substringBefore(tmp, "::") + postfix;
+
+		//DBObject query = QueryBuilder.start("mdId").is(collectionNameCleaned).get();
+		Bson query = Filters.eq("mdId", collectionNameCleaned);
+		return this.managerTable.find(query).first();
+
+	}
+
+	private boolean shouldDelete(final String collectionName, final DBObject metadataManagerInstance) {
+		log.debug("should delete instance "+metadataManagerInstance);
+		if((metadataManagerInstance== null) || (metadataManagerInstance.get("currentId")== null)){
+			log.debug("the instance has not currentID");
+			return true;
+		}
+		String currentId = (String) metadataManagerInstance.get("currentId");
+		if (collectionName.equals(currentId)) return false;
+		BasicDBList expiringList = (BasicDBList) metadataManagerInstance.get("expiring");
+		if (findInList(expiringList, collectionName, "id")) return false;
+		BasicDBList transactionsList = (BasicDBList) metadataManagerInstance.get("transactions");
+		return !findInList(transactionsList, collectionName, "id");
+	}
+
+	private boolean findInList(final BasicDBList list, final String object, final String tagname) {
+		if (list == null) return false;
+		for (Object o : list) {
+			DBObject currentObject = (DBObject) o;
+			final String id = (String) currentObject.get(tagname);
+			if (id.equals(object)) return true;
+		}
+		return false;
+	}
+
+	/**
+	 * Delete.
+	 *
+	 * @param list
+	 *            the list
+	 * @param toRemove
+	 *            the to remove
+	 */
+	private void delete(final BasicDBList list, final List<DBObject> toRemove) {
+
+		for (final DBObject obj : toRemove) {
+			if (log.isDebugEnabled()) {
+				log.debug("deleting " + obj);
+			}
+			list.remove(obj);
+		}
+	}
+
+	/**
+	 * Garbage transactions.
+	 *
+	 * @param currentObject
+	 *            the current object
+	 * @param currentId
+	 *            the current id
+	 */
+	private void garbageTransactions(final DBObject currentObject, final String currentId) {
+		if (log.isDebugEnabled()) {
+			log.debug("Start garbage transactions ");
+		}
+
+		final BasicDBList expiring = (BasicDBList) currentObject.get("transactions");
+		if ((expiring == null) || (expiring.size() <= getMaxTransactions())) return;
+
+		List<DBObject> expiringList = Lists.newArrayList();
+
+		for (Object o : expiring) {
+			final DBObject cobj = (DBObject) o;
+			if (cobj != null) {
+				expiringList.add((DBObject) o);
+			}
+
+		}
+
+		expiringList.sort(MDStoreUtils.getComparatorOnDate());
+
+		List<DBObject> toRemove = Lists.newArrayList();
+		int i = 0;
+
+		// We should remove the k item less recent
+		// where k = numberOftotalTransaction - maxNumberOfTransaction
+		// k = numberOfItemToRemove
+
+		while (((expiringList.size() - toRemove.size()) > getMaxTransactions()) || (i < expiringList.size())) {
+			DBObject currentObj = expiringList.get(i++);
+			String objectId = (String) currentObj.get("id");
+			if (!objectId.equals(currentId)) {
+				if (log.isDebugEnabled()) {
+					log.debug("delete collection: " + objectId + " from mongo");
+				}
+				db.getCollection(objectId).drop();
+				db.getCollection("discarded-" + objectId).drop();
+				if (log.isDebugEnabled()) {
+					log.debug("delete collection: discarded-" + objectId + " from mongo");
+				}
+				toRemove.add(currentObj);
+			} else {
+				if (log.isDebugEnabled()) {
+					log.debug("Cannot remove transaction " + objectId + " because is the currentId: " + currentId);
+				}
+			}
+		}
+
+		delete(expiring, toRemove);
+		log.info("Deleted " + toRemove.size() + " transactions, mdStore Id:" + currentObject.get("mdId"));
+	}
+
+	/**
+	 * Garbage expiring.
+	 *
+	 * @param currentObject
+	 *            the current object
+	 * @param currentId
+	 *            the current id
+	 */
+	private void garbageExpiring(final DBObject currentObject, final String currentId) {
+		if (log.isDebugEnabled()) {
+			log.debug("Start to search expiring mdstores for id: " + currentObject.get("mdId"));
+		}
+		final BasicDBList expiring = (BasicDBList) currentObject.get("expiring");
+		final List<DBObject> toRemove = Lists.newArrayList();
+		if (log.isDebugEnabled()) {
+			if (expiring == null) {
+				log.debug("expiring list is null");
+			} else {
+				log.debug("expiring list size is :" + expiring.size());
+			}
+		}
+		if ((expiring == null) || (expiring.size() == 0)) {
+			log.debug("Deleted  0  expired  collections, mdStore Id:" + currentObject.get("mdId"));
+			return;
+		}
+		for (Object anExpiring : expiring) {
+			final DBObject currentExpiredStore = (DBObject) anExpiring;
+			final String currentUsedId = (String) currentExpiredStore.get("id");
+			final Days d = getExpiringDays(currentExpiredStore, "lastRead");
+			if (log.isDebugEnabled()) {
+				log.debug("the store :" + currentId + " expired since " + d.getDays() + "days ");
+			}
+			// DELETE the collection where the last time they was read
+			// is more than 3 days ago
+			if (d.getDays() > getExpiredDays()) {
+				if (!currentUsedId.equals(currentId)) {
+					db.getCollection(currentUsedId).drop();
+					db.getCollection("discarded-" + currentUsedId).drop();
+					log.debug("deleted collection " + currentUsedId);
+				}
+				toRemove.add(currentExpiredStore);
+			}
+		}
+		delete(expiring, toRemove);
+		log.debug("Deleted expired " + toRemove.size() + "collections, mdStore Id:" + currentObject.get("mdId"));
+	}
+
+	/**
+	 * Gets the expiring days.
+	 *
+	 * @param value
+	 *            the value
+	 * @param paramName
+	 *            the param name
+	 * @return the expiring days
+	 */
+	private Days getExpiringDays(final DBObject value, final String paramName) {
+		final Date lastRead = (Date) value.get(paramName);
+		final DateTime last = new DateTime(lastRead);
+		final DateTime today = new DateTime();
+		final Days d = Days.daysBetween(last, today);
+		return d;
+	}
+
+	/**
+	 * Gets the expired days.
+	 *
+	 * @return the expiredDays
+	 */
+	public int getExpiredDays() {
+		if (this.expiredDays == 0) return 3;
+		return expiredDays;
+	}
+
+	/**
+	 * Sets the expired days.
+	 *
+	 * @param expiredDays
+	 *            the expiredDays to set
+	 */
+	public void setExpiredDays(final int expiredDays) {
+		this.expiredDays = expiredDays;
+	}
+
+	/**
+	 * Update incremental.
+	 *
+	 * @param transactionId
+	 *            the transaction id
+	 * @param currentId
+	 *            the current id
+	 */
+	private void updateIncremental(final String transactionId, final String currentId) {
+		final MongoCollection<DBObject> transaction = db.getCollection(transactionId, DBObject.class);
+		final MongoCollection<DBObject> mdstore = db.getCollection(currentId, DBObject.class);
+		final FindIterable<DBObject> it = transaction.find().noCursorTimeout(true);
+		for (DBObject currentObj : it) {
+			final String id = (String) currentObj.get("id");
+			BasicDBObject newObj = (BasicDBObject)((BasicDBObject) currentObj).copy();
+			//Must not have _id in the new object
+			newObj.remove("_id");
+			//setting to journaled write concern to be sure that when the write returns everything has been flushed to disk (https://docs.mongodb.org/manual/faq/developers/#when-does-mongodb-write-updates-to-disk)
+			//the explicit fsync command can't be run anymore: 'Command failed with error 13: 'fsync may only be run against the admin database.'
+			final MongoCollection<DBObject> mdstoreWrite = mdstore.withWriteConcern(WriteConcern.JOURNALED);
+			mdstoreWrite.replaceOne(new BasicDBObject("id", id), newObj, new UpdateOptions().upsert(true));
+		}
+	}
+
+	/*
+	 * (non-Javadoc)
+	 *
+	 * @see eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager#dropTransaction(java.lang.String, java.lang.String)
+	 */
+	@Override
+	public Boolean dropTransaction(final String mdId, final String idToDrop) throws MDStoreServiceException {
+		verifyConsistency();
+		final DBObject mdstoreInfo = getMDStoreAsDBObject(mdId);
+		if (mdstoreInfo == null) throw new MDStoreServiceException("Error, unable to find Mdstore with Id " + mdId);
+		return dropStore(mdstoreInfo, idToDrop, "transactions");
+	}
+
+	public void garbageTransactionsOnStart() throws MDStoreServiceException {
+		verifyConsistency();
+		FindIterable<DBObject> it = this.managerTable.find();
+
+		final List<String> currentMdStoreId = Lists.newArrayList();
+		for (DBObject currentObject : it){
+			currentMdStoreId.add((String) currentObject.get("currentId"));
+			final BasicDBList transactions = (BasicDBList) currentObject.get("transactions");
+			if ((transactions != null) && (transactions.size() > 0)) {
+				for (Object tx : transactions) {
+					final DBObject currentTransactions = (DBObject) tx;
+					final String id = (String) currentTransactions.get("id");
+					db.getCollection(id).drop();
+					db.getCollection("discarded-" + id).drop();
+					log.debug("deleted collection " + id);
+				}
+				currentObject.put("transactions", new BasicDBList());
+				this.getManagerTable().findOneAndReplace(new BasicDBObject("_id", currentObject.get("_id")), currentObject);
+			}
+		}
+
+		//DELETING ALL THE DISCARDED COLLECTION THAT DISCARDED COLLECTION OF THE CURRENT MDSTORE
+		final ArrayList<String> collectionsNames = Lists.newArrayList(db.listCollectionNames());
+
+		for (String item : collectionsNames) {
+			if (item.startsWith("discarded-")) {
+				final String currentCollection = StringUtils.substringAfter(item, "discarded-");
+				if (!currentMdStoreId.contains(currentCollection)) {
+					log.info("Deleting discarded collection :" + item);
+					this.db.getCollection(item).drop();
+				}
+			}
+		}
+	}
+
+	/**
+	 * Gets the max transactions.
+	 *
+	 * @return the maxTransactions
+	 */
+	public int getMaxTransactions() {
+		return maxTransactions;
+	}
+
+	/**
+	 * Sets the max transactions.
+	 *
+	 * @param maxTransactions
+	 *            the maxTransactions to set
+	 */
+	public void setMaxTransactions(final int maxTransactions) {
+		this.maxTransactions = maxTransactions;
+	}
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/mongodb/MongoBulkWritesManager.java

@@ -0,0 +1,147 @@
+package eu.dnetlib.data.mdstore.modular.mongodb;
+
+import java.util.List;
+import java.util.Map;
+import javax.annotation.Nullable;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.mongodb.BasicDBObject;
+import com.mongodb.DBObject;
+import com.mongodb.WriteConcern;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.model.*;
+import eu.dnetlib.data.mdstore.MDStoreServiceException;
+import eu.dnetlib.data.mdstore.modular.MDFormatDescription;
+import eu.dnetlib.data.mdstore.modular.RecordParser;
+import eu.dnetlib.data.mdstore.modular.mongodb.utils.IndexFieldRecordParser;
+import eu.dnetlib.data.mdstore.modular.mongodb.utils.IndexFieldRecordParserException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import static eu.dnetlib.data.mdstore.modular.MDStoreConstants.*;
+
+public class MongoBulkWritesManager {
+
+	private static final Log log = LogFactory.getLog(MongoBulkWritesManager.class);
+	private final boolean discardRecords;
+	private final boolean indexRecords;
+	private final IndexFieldRecordParser indexFieldRecordParser = new IndexFieldRecordParser();
+	private final List<MDFormatDescription> mdref;
+	private RecordParser recordParser;
+	private MongoCollection<DBObject> validCollection;
+	private List<WriteModel<DBObject>> validBulkOperationList;
+	private BulkWriteOptions writeOptions;
+	private MongoCollection<DBObject> discardedCollection;
+	private List<WriteModel<DBObject>> discardedBulkOperationList;
+	private int bulkSize;
+
+	public MongoBulkWritesManager(final MongoCollection<DBObject> collection,
+			final MongoCollection<DBObject> discardedCollection,
+			final List<MDFormatDescription> mdref,
+			final int bulkSize,
+			final RecordParser parser,
+			final boolean discardRecords) {
+		this.validCollection = collection.withWriteConcern(WriteConcern.ACKNOWLEDGED);
+		this.validBulkOperationList = Lists.newArrayList();
+
+		this.discardedCollection = discardedCollection.withWriteConcern(WriteConcern.ACKNOWLEDGED);
+		this.discardedBulkOperationList = Lists.newArrayList();
+
+		this.bulkSize = bulkSize;
+		this.recordParser = parser;
+		this.discardRecords = discardRecords;
+		this.mdref = mdref;
+
+		this.indexRecords = (this.mdref != null &&  !this.mdref.isEmpty());
+		this.writeOptions = new BulkWriteOptions().ordered(false);
+	}
+
+	public void insert(final String record) throws MDStoreServiceException {
+		Map<String, String> recordProperties = null;
+		try {
+			recordProperties = recordParser.parseRecord(record);
+
+		} catch (Throwable e) {
+			if (discardRecords) {
+				log.debug("unhandled exception: " + e.getMessage());
+				discardRecord(record);
+			}
+		}
+		Map<String, List<String>> indexRecordField = null;
+		try {
+			if (indexRecords) {
+				indexRecordField = indexFieldRecordParser.parseRecord(record, mdref);
+			}
+		} catch (IndexFieldRecordParserException e) {
+			// could not index record fields
+			throw new MDStoreServiceException("Are you using the correct type of store / index definition for the records in " + validCollection.getNamespace() + " ?", e);
+		}
+
+		log.debug("found props: " + recordProperties);
+		if (recordProperties.containsKey(ID)) {
+			final DBObject obj = buildDBObject(record, recordProperties, indexRecordField);
+			if (log.isDebugEnabled()) {
+				log.debug("Saving object" + obj);
+			}
+			validBulkOperationList.add(new ReplaceOneModel(new BasicDBObject(ID, obj.get(ID)), obj, new UpdateOptions().upsert(true)));
+			if (((validBulkOperationList.size() % bulkSize) == 0) && !validBulkOperationList.isEmpty()) {
+				validCollection.bulkWrite(validBulkOperationList, writeOptions);
+				validBulkOperationList.clear();
+			}
+		} else {
+			if (discardRecords) {
+				log.debug("parsed record seems invalid");
+				discardRecord(record);
+			}
+		}
+	}
+
+	private void discardRecord(final String record) {
+		discardedBulkOperationList.add(new InsertOneModel(new BasicDBObject(BODY, record)));
+
+		if (((discardedBulkOperationList.size() % bulkSize) == 0) && !discardedBulkOperationList.isEmpty()) {
+			discardedCollection.bulkWrite(discardedBulkOperationList, writeOptions);
+			discardedBulkOperationList.clear();
+		}
+	}
+
+	public void flushBulks() {
+		//setting to journaled write concern to be sure that when the write returns everything has been flushed to disk (https://docs.mongodb.org/manual/faq/developers/#when-does-mongodb-write-updates-to-disk)
+		//the explicit fsync command can't be run anymore: 'Command failed with error 13: 'fsync may only be run against the admin database.'
+		if (!validBulkOperationList.isEmpty()) {
+			validCollection = getCollectionWithWriteConcern(validCollection, WriteConcern.JOURNALED);
+			validCollection.bulkWrite(validBulkOperationList, writeOptions);
+		}
+		if (!discardedBulkOperationList.isEmpty()) {
+			discardedCollection = getCollectionWithWriteConcern(discardedCollection, WriteConcern.JOURNALED);
+			discardedCollection.bulkWrite(discardedBulkOperationList, writeOptions);
+		}
+		//setting write concern back to ACKNOWLEDGE to avoid the execution of future writes all in Journaled mode
+		validCollection = getCollectionWithWriteConcern(validCollection, WriteConcern.ACKNOWLEDGED);
+		discardedCollection = getCollectionWithWriteConcern(discardedCollection, WriteConcern.ACKNOWLEDGED);
+	}
+
+	protected DBObject buildDBObject(final String record, final Map<String, String> recordProperties, final Map<String, List<String>> indexFieldProperties) {
+		final DBObject obj = new BasicDBObject();
+		obj.put(ID, recordProperties.get(ID));
+		obj.put(ORIGINALID, recordProperties.get(ORIGINALID));
+		obj.put(BODY, record);
+		obj.put(TIMESTAMP, Long.valueOf(recordProperties.get(TIMESTAMP)));
+		if (indexFieldProperties != null)
+			obj.putAll(Maps.filterKeys(indexFieldProperties, new Predicate<String>() {
+				//ensure we do not override the mandatory fields above with some unexpected value
+				@Override
+				public boolean apply(@Nullable final String s) {
+					return !s.equalsIgnoreCase(ID) && !s.equalsIgnoreCase(ORIGINALID) && !s.equalsIgnoreCase(BODY) && !s.equalsIgnoreCase(TIMESTAMP);
+				}
+			}));
+		return obj;
+	}
+
+	private MongoCollection<DBObject> getCollectionWithWriteConcern(MongoCollection<DBObject> collection, WriteConcern writeConcern) {
+		return collection.withWriteConcern(writeConcern);
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/mongodb/MongoMDStore.java

@@ -0,0 +1,314 @@
+package eu.dnetlib.data.mdstore.modular.mongodb;
+
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.regex.Pattern;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.mongodb.BasicDBObject;
+import com.mongodb.DBObject;
+import com.mongodb.QueryBuilder;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.ListIndexesIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.model.IndexOptions;
+import eu.dnetlib.data.mdstore.DocumentNotFoundException;
+import eu.dnetlib.data.mdstore.MDStoreServiceException;
+import eu.dnetlib.data.mdstore.modular.MDFormatDescription;
+import eu.dnetlib.data.mdstore.modular.RecordParser;
+import eu.dnetlib.data.mdstore.modular.connector.MDStore;
+import eu.dnetlib.enabling.resultset.ResultSetListener;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.bson.conversions.Bson;
+import org.springframework.beans.factory.annotation.Required;
+
+public class MongoMDStore implements MDStore {
+
+	private static final int BULK_SIZE = 500;
+	private static final Log log = LogFactory.getLog(MongoMDStore.class);
+
+	private final boolean discardRecords;
+	private String id;
+	private MongoDatabase mongoDatabase;
+	private MongoCollection<DBObject> collection;
+	private MongoCollection<DBObject> discardedCollection;
+	private List<MDFormatDescription> mdformats;
+
+	private RecordParser recordParser;
+
+	private static List<String> indices = Lists.newArrayList("id", "timestamp", "originalId");
+	private final IndexOptions options = new IndexOptions().background(true);
+
+	public MongoMDStore(final String id,
+			final MongoCollection<DBObject> collection,
+			final RecordParser recordParser,
+			final boolean discardRecords,
+			final MongoDatabase mongoDatabase) {
+		this.id = id;
+		this.mongoDatabase = mongoDatabase;
+		this.collection = collection;
+		this.discardedCollection = this.mongoDatabase.getCollection("discarded-" + StringUtils.substringBefore(id, "_"), DBObject.class);
+		this.recordParser = recordParser;
+		this.discardRecords = discardRecords;
+	}
+
+	@Override
+	public int feed(final Iterable<String> records, final boolean incremental, final List<MDFormatDescription> mdformats) {
+		this.mdformats = mdformats;
+		return feed(records, incremental);
+	}
+
+	@Override
+	public int feed(final Iterable<String> records, final boolean incremental) {
+		// TODO: remove incremental from MDStore API. It is used in MDStoreModular. Useless here.
+
+		ensureIndices();
+
+		final BlockingQueue<Object> queue = new ArrayBlockingQueue<>(100);
+		final Object sentinel = new Object();
+		int countStored = 0;
+		final Callable<Integer> writer = () -> {
+			final MongoBulkWritesManager bulkWritesManager =
+					new MongoBulkWritesManager(collection, discardedCollection, mdformats, BULK_SIZE, recordParser, discardRecords);
+			int count = 0;
+			while (true) {
+				try {
+					final Object record = queue.take();
+					if (record == sentinel) {
+						bulkWritesManager.flushBulks();
+						break;
+					}
+					count++;
+					bulkWritesManager.insert((String) record);
+				} catch (final InterruptedException e) {
+					log.fatal("got exception in background thread", e);
+					throw new IllegalStateException(e);
+				}
+			}
+			log.debug(String.format("extracted %s records from feeder queue", count));
+			return count;
+		};
+		final ExecutorService executorService = Executors.newSingleThreadExecutor();
+		Future<Integer> storedCountInt = executorService.submit(writer);
+
+		try {
+			log.info("feeding mdstore " + id);
+			if (records != null) {
+				for (final String record : records) {
+					queue.put(record);
+				}
+			}
+			queue.put(sentinel);
+			countStored = storedCountInt.get().intValue();
+		} catch (final InterruptedException e) {
+			log.error("Error on feeding mdstore with id:" + id, e);
+			throw new IllegalStateException(e);
+		} catch (ExecutionException e) {
+			log.error("Error on feeding mdstore with id:" + id, e);
+			throw new IllegalStateException(e);
+		}
+		log.info("finished feeding mdstore " + id);
+		return countStored;
+	}
+
+	public void ensureIndices() {
+		for (final String key : indices) {
+			collection.createIndex(new BasicDBObject(key, 1), options);
+		}
+		if (mdformats != null) {
+			for (final MDFormatDescription description : mdformats) {
+				collection.createIndex(new BasicDBObject(description.getName(), 1), options);
+			}
+		}
+	}
+
+	public boolean isIndexed() {
+		final ListIndexesIterable<DBObject> found = collection.listIndexes(DBObject.class);
+		return Sets.newHashSet(Iterables.transform(found, dbo -> {
+			final Set<String> keyset = ((DBObject) dbo.get("key")).toMap().keySet();
+			return Iterables.getFirst(keyset, "");
+		})).containsAll(indices);
+	}
+
+	/**
+	 * Method searches for the given string grep into this collection and replaces it with the given replacement.
+	 *
+	 * @param grep
+	 *            the string to search
+	 * @param replace
+	 *            the replacement
+	 */
+	public void replace(final String grep, final String replace) {
+		final Pattern regex = Pattern.compile(grep, Pattern.MULTILINE);
+		BasicDBObject query = (BasicDBObject) QueryBuilder.start("body").regex(regex).get();
+		final FindIterable<DBObject> matches = collection.find(query, DBObject.class);
+		//final DBCursor matches = collection.find(QueryBuilder.start("body").regex(regex).get());
+		if (log.isDebugEnabled())
+			log.debug("FOUND: " + Lists.newArrayList(matches).size());
+
+		for (final DBObject match : matches) {
+			final DBObject o = new BasicDBObject(match.toMap());
+			o.put("body", regex.matcher((String) match.get("body")).replaceAll(replace));
+			collection.findOneAndReplace(new BasicDBObject("_id", o.get("_id")), o);
+		}
+	}
+
+	@Override
+	public ResultSetListener deliver(final String from, final String until, final String recordFilter) throws MDStoreServiceException {
+		return deliver(from, until, recordFilter, new SerializeMongoRecord());
+	}
+
+	@Override
+	public ResultSetListener deliverIds(final String from, final String until, final String recordFilter) throws MDStoreServiceException {
+		return deliver(from, until, recordFilter, new SerializeMongoRecordId());
+	}
+
+	public ResultSetListener deliver(final String from, final String until, final String recordFilter, final Function<DBObject, String> serializer)
+			throws MDStoreServiceException {
+		final Pattern filter = (recordFilter != null) && (recordFilter.length() > 0) ? Pattern.compile(recordFilter, Pattern.MULTILINE) : null;
+
+		return new MongoResultSetListener(collection, parseLong(from), parseLong(until), filter, serializer);	}
+
+	private Long parseLong(final String s) throws MDStoreServiceException {
+		if (StringUtils.isBlank(s)) {
+			return null;
+		}
+		try {
+			return Long.valueOf(s);
+		} catch (NumberFormatException e) {
+			throw new MDStoreServiceException("Invalid date, expected java.lang.Long, or null", e);
+		}
+	}
+
+	@Override
+	public Iterable<String> iterate() {
+		return () -> Iterators.transform(collection.find().iterator(), arg -> (String) arg.get("body"));
+	}
+
+	@Override
+	public void deleteRecord(final String recordId) {
+		collection.deleteOne(new BasicDBObject("id", recordId));
+	}
+
+	@Override
+	public String getRecord(final String recordId) throws DocumentNotFoundException {
+		final DBObject obj = collection.find(new BasicDBObject("id", recordId)).first();
+		if (obj == null || !obj.containsField("body")) throw new DocumentNotFoundException(String.format(
+				"The document with id '%s' does not exist in mdstore: '%s'", recordId, id));
+		final String body = (String) obj.get("body");
+		if (body.trim().length() == 0) throw new DocumentNotFoundException(String.format("The document with id '%s' does not exist in mdstore: '%s'",
+				recordId, id));
+		return new SerializeMongoRecord().apply(obj);
+	}
+
+	@Override
+	public List<String> deliver(final String mdId, final int pageSize, final int offset, final Map<String, String> queryParam) {
+		final QueryBuilder query = QueryBuilder.start();
+
+		for (String key : queryParam.keySet()) {
+			query.and(key).regex(Pattern.compile(queryParam.get(key), Pattern.LITERAL));
+		}
+
+		FindIterable<DBObject> dbObjects = offset > 0
+				? collection.find((Bson) query.get()).limit(pageSize).skip(offset)
+				: collection.find((Bson) query.get()).limit(pageSize);
+
+
+		queryParam.put("count", "" + collection.count((Bson) query.get()));
+
+		final List<String> result = new ArrayList<>();
+		for (final DBObject item : dbObjects) {
+			result.add(item.get("body").toString());
+		}
+
+		return result;
+	}
+
+	@Override
+	public void truncate() {
+		collection.drop();
+		discardedCollection.drop();
+	}
+
+	public DBObject getMDStoreMetadata() {
+		return mongoDatabase.getCollection("metadata", DBObject.class).find(new BasicDBObject("mdId", getId())).first();
+	}
+
+	@Override
+	public String getFormat() {
+		return (String) getMDStoreMetadata().get("format");
+	}
+
+	@Override
+	public String getInterpretation() {
+		return (String) getMDStoreMetadata().get("interpretation");
+	}
+
+	@Override
+	public String getLayout() {
+		return (String) getMDStoreMetadata().get("layout");
+	}
+
+	@Override
+	public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public MongoCollection<DBObject> getCollection() {
+		return collection;
+	}
+
+	public void setCollection(final MongoCollection<DBObject> collection) {
+		this.collection = collection;
+	}
+
+	public RecordParser getRecordParser() {
+		return recordParser;
+	}
+
+	@Required
+	public void setRecordParser(final RecordParser recordParser) {
+		this.recordParser = recordParser;
+	}
+
+	@Override
+	public int getSize() {
+		return (int) collection.count();
+	}
+
+	public MongoCollection<DBObject> getDiscardedCollection() {
+		return discardedCollection;
+	}
+
+	public void setDiscardedCollection(final MongoCollection<DBObject> discardedCollection) {
+		this.discardedCollection = discardedCollection;
+	}
+
+	private class SerializeMongoRecord implements Function<DBObject, String> {
+
+		@Override
+		public String apply(final DBObject arg) {
+			return (String) arg.get("body");
+		}
+	}
+
+	private class SerializeMongoRecordId implements Function<DBObject, String> {
+
+		@Override
+		public String apply(final DBObject arg) {
+			return (String) arg.get("id");
+		}
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/mongodb/MongoResultSetListener.java

@@ -0,0 +1,126 @@
+package eu.dnetlib.data.mdstore.modular.mongodb;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.mongodb.BasicDBObject;
+import com.mongodb.DBObject;
+import com.mongodb.MongoClient;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.model.Sorts;
+import eu.dnetlib.enabling.resultset.ResultSet;
+import eu.dnetlib.enabling.resultset.ResultSetAware;
+import eu.dnetlib.enabling.resultset.ResultSetListener;
+import eu.dnetlib.miscutils.maps.ConcurrentSizedMap;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.bson.BsonDocument;
+import org.bson.conversions.Bson;
+
+import static com.mongodb.client.model.Filters.*;
+
+public class MongoResultSetListener implements ResultSetListener, ResultSetAware {
+
+	private static final Log log = LogFactory.getLog(MongoResultSetListener.class);
+
+	private ConcurrentSizedMap<Integer, String> lastKeys = new ConcurrentSizedMap<>();
+	private Bson sortByIdAsc = Sorts.orderBy(Sorts.ascending("id"));
+
+	private Function<DBObject, String> serializer;
+	private MongoCollection<DBObject> collection;
+	private Bson query;
+
+	public MongoResultSetListener(final MongoCollection<DBObject> collection, final Long from, final Long until, final Pattern filter, final Function<DBObject, String> serializer) {
+		this.collection = collection;
+		this.serializer = serializer;
+		this.query = query(from, until, filter);
+		log.debug("Query on mongo: "+this.query.toBsonDocument(BsonDocument.class, MongoClient.getDefaultCodecRegistry()));
+	}
+
+	@Override
+	public List<String> getResult(final int fromPosition, final int toPosition) {
+
+		ArrayList<DBObject> page = null;
+
+		String lastKey = lastKeys.get(fromPosition);
+		if (lastKey != null) {
+			page = continueFrom(lastKey, (toPosition - fromPosition) + 1);
+		} else {
+			page = fetchNew(fromPosition - 1, (toPosition - fromPosition) + 1);
+		}
+
+		if (!page.isEmpty()) {
+			DBObject last = page.get(page.size() - 1);
+			lastKeys.put(toPosition + 1, (String) last.get("id"));
+		}
+
+		if (log.isDebugEnabled()) {
+			log.info(String.format("got %s records from %s to %s", page.size(), fromPosition, toPosition));
+		}
+
+		return Lists.newArrayList(Iterables.transform(page, serializer));
+	}
+
+	private ArrayList<DBObject> fetchNew(final int from, final int size) {
+		final FindIterable<DBObject> it = collection.find(query).batchSize(size);
+		return Lists.newArrayList(it.sort(sortByIdAsc).skip(from).limit(size));
+	}
+
+	private ArrayList<DBObject> continueFrom(final String lastKey, final int size) {
+		if (log.isDebugEnabled()) {
+			log.debug("trying to continue from previous key: " + lastKey);
+		}
+		final Bson q = and(query, gt("id", lastKey));
+		final FindIterable<DBObject> it = collection.find(q).batchSize(size).sort(sortByIdAsc).limit(size);
+		return Lists.newArrayList(it);
+	}
+
+	private Bson query(final Long from, final Long until, final Pattern pattern) {
+		final Bson dateFilter = dateQuery(from, until);
+		final Bson regexFilter = regexQuery(pattern);
+		if (dateFilter != null & regexFilter != null) {
+			return and(dateFilter, regexFilter);
+		} else if (dateFilter != null) {
+			return dateFilter;
+		} else if (regexFilter != null) {
+			return regexFilter;
+		}
+		return new BasicDBObject();
+	}
+
+	private Bson dateQuery(final Long from, final Long until) {
+		if (from != null & until != null) {
+			return and(gt("timestamp", from), lt("timestamp", until));
+		}
+		if (from != null) {
+			return gt("timestamp", from);
+		}
+		if (until != null) {
+			return lt("timestamp", until);
+		}
+		return null;
+	}
+
+	private Bson regexQuery(final Pattern pattern) {
+		if (pattern != null) {
+			return regex("body", pattern);
+		}
+		return null;
+	}
+
+	@Override
+	public int getSize() {
+		return (int) collection.count(query);
+	}
+
+	@Override
+	public void setResultSet(final ResultSet resultSet) {
+		resultSet.close();
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/mongodb/utils/EnsureIndexJob.java

@@ -0,0 +1,48 @@
+package eu.dnetlib.data.mdstore.modular.mongodb.utils;
+
+import eu.dnetlib.data.mdstore.modular.MDStoreDescription;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Required;
+
+import eu.dnetlib.data.mdstore.MDStoreServiceException;
+import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;
+import eu.dnetlib.data.mdstore.modular.mongodb.MongoMDStore;
+import eu.dnetlib.enabling.tools.AbstractSchedulable;
+
+public class EnsureIndexJob extends AbstractSchedulable {
+
+	private static final Log log = LogFactory.getLog(EnsureIndexJob.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	private MDStoreDao dao;
+
+	@Override
+	protected void doExecute() {
+		log.info("performing mdstore index check");
+
+		try {
+			for (MDStoreDescription mdstore : getDao().listMDStores()) {
+				try {
+					log.info("ensureindex for mdStoreId:" + mdstore.getId());
+					((MongoMDStore) getDao().getMDStore(mdstore.getId())).ensureIndices();
+				} catch (Throwable e) {
+					log.warn("unable to reindex mdstore: " + mdstore.getId(), e);
+				}
+			}
+		} catch (MDStoreServiceException e) {
+			log.warn("unable to reindex mdstore ", e);
+		}
+
+		log.info("mdstore index check completed");
+	}
+
+	@Required
+	public void setDao(final MDStoreDao dao) {
+		this.dao = dao;
+	}
+
+	public MDStoreDao getDao() {
+		return dao;
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/mongodb/utils/IndexFieldRecordParser.java

@@ -0,0 +1,55 @@
+package eu.dnetlib.data.mdstore.modular.mongodb.utils;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.ximpleware.AutoPilot;
+import com.ximpleware.VTDGen;
+import com.ximpleware.VTDNav;
+import eu.dnetlib.data.mdstore.modular.MDFormatDescription;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Created by sandro on 11/29/16.
+ */
+public class IndexFieldRecordParser {
+
+    private static final Log log = LogFactory.getLog(IndexFieldRecordParser.class);
+
+    private static List<String> getTextValue(final AutoPilot ap, final VTDNav vn, final String xpath) throws Exception {
+        List<String> results = new ArrayList<>();
+        ap.selectXPath(xpath);
+        while (ap.evalXPath() != -1) {
+            int t = vn.getText();
+            if (t > -1) results.add(vn.toNormalizedString(t));
+        }
+        return results;
+    }
+
+    public Map<String, List<String>> parseRecord(final String record, final List<MDFormatDescription> mdformats) throws IndexFieldRecordParserException {
+        if (mdformats == null || mdformats.size() == 0)
+            return null;
+        final Map<String, List<String>> result = new HashMap<>();
+
+        try {
+            final VTDGen vg = new VTDGen();
+            vg.setDoc(record.getBytes());
+            vg.parse(true);
+            final VTDNav vn = vg.getNav();
+            final AutoPilot ap = new AutoPilot(vn);
+
+            for (MDFormatDescription description : mdformats) {
+                List<String> xpathResult = getTextValue(ap, vn, description.getXpath());
+                result.put(description.getName(), xpathResult);
+            }
+            return result;
+        } catch (Throwable e) {
+            throw new IndexFieldRecordParserException("Cannot index record", e);
+        }
+    }
+
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/mongodb/utils/IndexFieldRecordParserException.java

@@ -0,0 +1,25 @@
+package eu.dnetlib.data.mdstore.modular.mongodb.utils;
+
+/**
+ * Created by Alessia Bardi on 14/06/2017.
+ *
+ * @author Alessia Bardi
+ */
+public class IndexFieldRecordParserException extends Exception {
+
+	public IndexFieldRecordParserException() {
+		super();
+	}
+
+	public IndexFieldRecordParserException(final String message) {
+		super(message);
+	}
+
+	public IndexFieldRecordParserException(final String message, final Throwable cause) {
+		super(message, cause);
+	}
+
+	public IndexFieldRecordParserException(final Throwable cause) {
+		super(cause);
+	}
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/mongodb/utils/MDStoreUtils.java

@@ -0,0 +1,33 @@
+package eu.dnetlib.data.mdstore.modular.mongodb.utils;
+
+import java.util.Comparator;
+import java.util.Date;
+
+import com.google.common.base.Function;
+import com.mongodb.DBObject;
+
+import eu.dnetlib.miscutils.functional.UnaryFunction;
+
+public class MDStoreUtils {
+
+	public static UnaryFunction<String, DBObject> mdId() {
+		return arg -> (String) arg.get("mdId");
+	}
+
+	public static UnaryFunction<Boolean, DBObject> dboFilter(final String format, final String layout, final String interpretation) {
+		return dbo -> dbo.get("format").toString().equals(format) && dbo.get("layout").toString().equals(layout)
+				&& dbo.get("interpretation").toString().equals(interpretation);
+	}
+
+	public static Function<DBObject, String> body() {
+		return dbo -> (String) dbo.get("body");
+	}
+
+	public static Comparator<DBObject> getComparatorOnDate() {
+		return (o1, o2) -> {
+			Date d1 = (Date) o1.get("date");
+			Date d2 = (Date) o2.get("date");
+			return d1.compareTo(d2);
+		};
+	}
+}

dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/modular/mongodb/utils/MetadataCheckJob.java

@@ -0,0 +1,159 @@
+package eu.dnetlib.data.mdstore.modular.mongodb.utils;
+
+import java.io.StringReader;
+import java.util.List;
+import javax.xml.ws.Endpoint;
+
+import com.mongodb.BasicDBObject;
+import com.mongodb.DBObject;
+import com.mongodb.client.MongoCollection;
+import eu.dnetlib.data.mdstore.MDStoreServiceException;
+import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;
+import eu.dnetlib.data.mdstore.modular.mongodb.MDStoreDaoImpl;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.soap.EndpointReferenceBuilder;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.Document;
+import org.dom4j.DocumentException;
+import org.dom4j.io.SAXReader;
+import org.springframework.beans.factory.annotation.Required;
+
+public class MetadataCheckJob {
+
+	private static final Log log = LogFactory.getLog(MetadataCheckJob.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	/**
+	 * service locator.
+	 */
+	private UniqueServiceLocator serviceLocator;
+
+	/**
+	 * {@link Endpoint}
+	 */
+	private Endpoint endpoint;
+
+	/**
+	 * {@link EndpointReferenceBuilder}
+	 */
+	private EndpointReferenceBuilder<Endpoint> eprBuilder;
+
+	/**
+	 * MDStore dao.
+	 */
+	private MDStoreDao dao;
+
+	private boolean runOnStart;
+
+	public void runOnStart() throws MDStoreServiceException {
+		if (isRunOnStart()) {
+			log.info("running mdStore metadata check on service start");
+			repairMetadata();
+		}
+	}
+
+	/**
+	 * Job execution method.
+	 *
+	 * @throws MDStoreServiceException in case of ISLookUpException or DocumentException
+	 */
+	public void repairMetadata() throws MDStoreServiceException {
+		MongoCollection<DBObject> metadata = ((MDStoreDaoImpl) getDao()).getDb().getCollection("metadata", DBObject.class);
+		if (metadata.count() != 0) {
+			log.debug("mdStore metadata doesn't need to be repaired");
+			return;
+		}
+		try {
+			List<String> mdStores =
+					serviceLocator.getService(ISLookUpService.class).quickSearchProfile(
+							"//RESOURCE_PROFILE[" +
+									".//RESOURCE_TYPE/@value='MDStoreDSResourceType' and " +
+									".//RESOURCE_URI/@value='" + getServiceAddress() + "']");
+
+			log.debug("repairing mdstore metadata");
+
+			if (mdStores != null) {
+				for (String MDStoreProfile : mdStores) {
+					final DBObject mdInfo = getMdInfo(MDStoreProfile);
+					metadata.findOneAndReplace(new BasicDBObject("mdId", mdInfo.get("mdId")), mdInfo);
+				}
+			}
+			log.debug("FINISHED repairing mdstore metadata");
+
+		} catch (ISLookUpException | DocumentException e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+	/**
+	 * Helper method, gets an MDStore profile and returns a DBObject.
+	 *
+	 * @param MDStoreProfile as obtain from the IS
+	 * @return a DBObject representing the metadata informations
+	 * @throws DocumentException when parsing invalid xml
+	 */
+	private DBObject getMdInfo(final String MDStoreProfile) throws DocumentException {
+		Document doc = new SAXReader().read(new StringReader(MDStoreProfile));
+
+		DBObject dbo = new BasicDBObject();
+		dbo.put("mdId", doc.valueOf("//RESOURCE_IDENTIFIER/@value"));
+		dbo.put("format", doc.valueOf("//METADATA_FORMAT/text()"));
+		dbo.put("layout", doc.valueOf("//METADATA_FORMAT_LAYOUT/text()"));
+		dbo.put("interpretation", doc.valueOf("//METADATA_FORMAT_INTERPRETATION/text()"));
+		dbo.put("size", doc.valueOf("//NUMBER_OF_RECORDS/text()"));
+
+		return dbo;
+	}
+
+	private String getServiceAddress() {
+		return getEprBuilder().getAddress(getEndpoint()) + "?wsdl";
+	}
+
+	public EndpointReferenceBuilder<Endpoint> getEprBuilder() {
+		return eprBuilder;
+	}
+
+	@Required
+	public void setEprBuilder(final EndpointReferenceBuilder<Endpoint> eprBuilder) {
+		this.eprBuilder = eprBuilder;
+	}
+
+	public Endpoint getEndpoint() {
+		return endpoint;
+	}
+
+	@Required
+	public void setEndpoint(final Endpoint endpoint) {
+		this.endpoint = endpoint;
+	}
+
+	public MDStoreDao getDao() {
+		return dao;
+	}
+
+	@Required
+	public void setDao(final MDStoreDao dao) {
+		this.dao = dao;
+	}
+
+	public boolean isRunOnStart() {
+		return runOnStart;
+	}
+
+	@Required
+	public void setRunOnStart(final boolean runOnStart) {
+		this.runOnStart = runOnStart;
+	}
+
+	public UniqueServiceLocator getServiceLocator() {
+		return serviceLocator;
+	}
+	
+	@Required
+	public void setServiceLocator(UniqueServiceLocator serviceLocator) {
+		this.serviceLocator = serviceLocator;
+	}
+
+}

dnet-data-services/src/main/resources/eu/dnetlib/data/mdstore/modular/applicationContext-modular-mdstore.properties

@@ -0,0 +1,10 @@
+services.mdstore.rsfactory=resultSetFactory
+services.mdstore.dao=mongodbMDStoreDao
+services.mdstore.syncFeed=true
+services.mdstore.recordParser=eu.dnetlib.data.mdstore.modular.StreamingRecordParser
+services.mdstore.rsfactory.pagesize=20
+services.mdstore.rsfactory.connectTimeout=60000
+services.mdstore.rsfactory.timeout=1200000
+services.mdstore.discardrecords=true
+services.mdstore.transaction=mongoMdStoreTransaction
+services.mdstore.readlock.expire.days=3

dnet-data-services/src/main/resources/eu/dnetlib/data/mdstore/modular/applicationContext-modular-mdstore.xml

@@ -0,0 +1,102 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns:jaxws="http://cxf.apache.org/jaxws" xmlns:p="http://www.springframework.org/schema/p"
+       xmlns:http="http://cxf.apache.org/transports/http/configuration"
+       xmlns:t="http://dnetlib.eu/springbeans/t"
+       xmlns:template="http://dnetlib.eu/springbeans/template" xmlns="http://www.springframework.org/schema/beans"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+
+
+                                    http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd
+                            http://cxf.apache.org/jaxws http://cxf.apache.org/schemas/jaxws.xsd
+                            http://dnetlib.eu/springbeans/template http://dnetlib.eu/springbeans/template.xsd">
+
+	<!-- beans -->
+	<bean id="mdStoreService" class="eu.dnetlib.data.mdstore.modular.ModularMDStoreService"
+		init-method="start" destroy-method="stop" p:notificationHandler-ref="mdstoreNotificationHandler"
+		p:iterableResultSetFactory-ref="iterableResultSetFactory"
+		p:feeder-ref="mdstoreFeeder" p:retriever-ref="mdstoreRetriever" />
+
+	<bean id="mdstoreNotificationHandler"
+		class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerExecutorNotificationHandler"
+		p:blackboardExecutor-ref="mdstoreBlackboardExecutor" />
+
+	<bean id="mdstoreBlackboardExecutor"
+		class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerActionExecutor"
+		p:blackboardHandler-ref="blackboardHandler"
+		p:actionType="eu.dnetlib.data.mdstore.modular.action.MDStoreActions"
+		p:incomplete="false">
+		<property name="actionMap">
+			<map>
+				<entry key="CREATE">
+					<bean class="eu.dnetlib.data.mdstore.modular.action.CreateAction"
+						p:profileCreator-ref="mdstoreProfileCreator"
+						p:dao-ref="${services.mdstore.dao}" />
+				</entry>
+				<entry key="DELETE">
+					<bean class="eu.dnetlib.data.mdstore.modular.action.DeleteAction"
+						p:serviceLocator-ref="uniqueServiceLocator" p:dao-ref="${services.mdstore.dao}" />
+				</entry>
+				<entry key="FEED">
+					<bean class="eu.dnetlib.data.mdstore.modular.action.FeedAction"
+						p:dao-ref="${services.mdstore.dao}" 
+						p:feeder-ref="mdstoreFeeder"/>
+				</entry>
+				<entry key="RUN_PLUGIN">
+					<bean class="eu.dnetlib.data.mdstore.modular.action.PluginAction"
+						p:dao-ref="${services.mdstore.dao}" />
+				</entry>				
+			</map>
+		</property>
+	</bean>
+
+	<bean id="mdstoreProfileCreator" class="eu.dnetlib.data.mdstore.modular.MDStoreProfileCreator"
+		p:serviceLocator-ref="uniqueServiceLocator" p:mdstoreDsTemplate-ref="mdstoreDsTemplate"
+		p:endpoint-ref="mdStoreServiceEndpoint" p:eprBuilder-ref="jaxwsEndpointReferenceBuilder" />
+
+	<bean id="mdstoreDsTemplate"
+		class="eu.dnetlib.springutils.stringtemplate.StringTemplateFactory"
+		p:template="classpath:/eu/dnetlib/data/mdstore/modular/mdstoreds-template.xml"
+		scope="prototype" />
+
+	<bean id="mdstoreFeeder" class="eu.dnetlib.data.mdstore.modular.MDStoreFeeder"
+		p:dao-ref="${services.mdstore.dao}" p:resultSetClientFactory-ref="mdstoreResultSetClientFactory"
+		p:syncFeed="${services.mdstore.syncFeed}" p:serviceLocator-ref="uniqueServiceLocator" />
+
+	<bean id="mdstoreResultSetClientFactory" parent="resultSetClientFactory"
+		p:pageSize="${services.mdstore.rsfactory.pagesize}"
+		p:timeout="${services.mdstore.rsfactory.timeout}"
+	    p:connectTimeout="${services.mdstore.rsfactory.connectTimeout}"/>
+
+
+	<bean id="mdstoreRetriever" class="eu.dnetlib.data.mdstore.modular.MDStoreRetriever"
+		p:dao-ref="${services.mdstore.dao}" p:resultSetFactory-ref="${services.mdstore.rsfactory}" />
+
+ 	<bean id="dummyMDstorePlugin" class="eu.dnetlib.data.mdstore.modular.action.DummyPlugin" />
+	
+	<bean id="mdStorePluginEnumerator" class="eu.dnetlib.data.mdstore.modular.action.MDStorePluginEnumerator" />
+
+
+    <bean id="mdstoreUtils" class="eu.dnetlib.data.mdstore.modular.MDStoreUtils"/>
+	
+
+	<!-- <bean id="mdstoreRecordParser" class="eu.dnetlib.data.mdstore.modular.SimpleRecordParser" 
+		/> -->
+<!-- 	<bean id="mdstoreRecordParser"  -->
+<!-- 		factory-bean="recordParserFactory" factory-method="newInstance"/> -->
+	
+	<bean id="recordParserFactory" class="eu.dnetlib.data.mdstore.modular.RecordParserFactory"
+		p:parserType="${services.mdstore.recordParser}" />
+		
+	<bean id="bulkRecordMapperFactory" class="eu.dnetlib.data.mdstore.modular.BulkRecordMapperFactory" />
+
+	<!-- endpoints -->
+	<jaxws:endpoint id="mdStoreServiceEndpoint" implementor="#mdStoreService"
+		implementorClass="eu.dnetlib.data.mdstore.MDStoreService" address="/mdStore" />
+
+	<template:instance name="serviceRegistrationManager"
+		t:serviceRegistrationManagerClass="eu.dnetlib.enabling.tools.registration.ValidatingServiceRegistrationManagerImpl"
+		t:name="mdStoreServiceRegistrationManager" t:service="mdStoreService"
+		t:endpoint="mdStoreServiceEndpoint" t:jobScheduler="jobScheduler"
+		t:serviceRegistrator="blackboardServiceRegistrator" />
+</beans>

dnet-data-services/src/main/resources/eu/dnetlib/data/mdstore/modular/mdstoreServiceStatusTemplate.xml.st

@@ -0,0 +1,5 @@
+<STATUS>
+	<HANDLED_DATASTRUCTURE>$status.handledDatastructures$</HANDLED_DATASTRUCTURE>
+	<USED_DISKSPACE>$status.usedDiskSpace$</USED_DISKSPACE>
+	<LAST_UPDATE value="$status.date$"/>
+</STATUS>

dnet-data-services/src/main/resources/eu/dnetlib/data/mdstore/modular/mdstoreds-template.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0"?>
+<RESOURCE_PROFILE>
+	<HEADER>
+		<RESOURCE_IDENTIFIER value="" />
+		<RESOURCE_TYPE value="MDStoreDSResourceType" />
+		<RESOURCE_KIND value="MDStoreDSResources" />
+		<RESOURCE_URI value="$serviceUri$?wsdl" />
+		<DATE_OF_CREATION value="" />
+	</HEADER>
+	<BODY>
+		<CONFIGURATION>
+			<METADATA_FORMAT>$format$</METADATA_FORMAT>
+			<METADATA_FORMAT_INTERPRETATION>$interpretation$</METADATA_FORMAT_INTERPRETATION>
+			<METADATA_FORMAT_LAYOUT>$layout$</METADATA_FORMAT_LAYOUT>
+		</CONFIGURATION>
+		<STATUS>
+			<PENULTIMATE_STORAGE_DATE></PENULTIMATE_STORAGE_DATE>
+			<LAST_STORAGE_DATE></LAST_STORAGE_DATE>
+			<NUMBER_OF_RECORDS>0</NUMBER_OF_RECORDS>
+			<FETCHING_FREQUENCY />
+			<STATISTICS_FIELDS />
+		</STATUS>
+		<SECURITY_PARAMETERS />
+	</BODY>
+</RESOURCE_PROFILE>

dnet-data-services/src/test/java/eu/dnetlib/data/mdstore/modular/mongodb/ConfigurationTestConfig.java

@@ -0,0 +1,45 @@
+package eu.dnetlib.data.mdstore.modular.mongodb;
+
+import java.net.UnknownHostException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.MongoDatabase;
+import eu.dnetlib.data.mdstore.modular.RecordParserFactory;
+import eu.dnetlib.data.mdstore.modular.StreamingRecordParser;
+import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;
+import eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class ConfigurationTestConfig {
+
+	@Bean
+	public MongoDatabase db() throws UnknownHostException {
+		final MongoClient mongo = new MongoClient("localhost", 27017);
+		return mongo.getDatabase("mdstore_test");
+	}
+
+	@Bean
+	public MDStoreTransactionManager manager() throws UnknownHostException {
+		final MDStoreTransactionManagerImpl manager = new MDStoreTransactionManagerImpl();
+		manager.setDb(db());
+		return manager;
+	}
+
+	@Bean
+	public RecordParserFactory recordParserFactory() {
+		final RecordParserFactory rpfactory = new RecordParserFactory();
+		rpfactory.setParserType(StreamingRecordParser.class);
+		return rpfactory;
+	}
+
+	@Bean
+	public MDStoreDao mdstoreDao() throws UnknownHostException {
+		final MDStoreDaoImpl dao = new MDStoreDaoImpl();
+		dao.setDb(db());
+		dao.setRecordParserFactory(recordParserFactory());
+		return dao;
+	}
+
+}

dnet-data-services/src/test/java/eu/dnetlib/data/mdstore/modular/mongodb/FeedSpeedTest.java

@@ -0,0 +1,159 @@
+package eu.dnetlib.data.mdstore.modular.mongodb;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.UUID;
+
+import com.mongodb.DBObject;
+import com.mongodb.client.MongoDatabase;
+import eu.dnetlib.data.mdstore.MDStoreServiceException;
+import eu.dnetlib.data.mdstore.modular.RecordParserFactory;
+import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;
+import org.apache.commons.io.IOUtils;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+
+@Ignore
+@RunWith(SpringJUnit4ClassRunner.class)
+@ContextConfiguration(classes = ConfigurationTestConfig.class)
+public class FeedSpeedTest {
+
+	private static final int N_RECORDS = 68271;
+
+	@Autowired
+	private MongoDatabase db;
+
+	@Autowired
+	private MDStoreDao dao;
+
+	@Autowired
+	private RecordParserFactory recordParserfactory;
+
+	@Before
+	public void setup() throws MDStoreServiceException {
+		dao.createMDStore("speed_test", "testFormat", "testInterpretation", "testLayout");
+	}
+
+
+	@Test
+	public void testSpeedFromFolder() throws IOException {
+		Iterable<String> iterable = new Iterable<String>() {
+
+			private int counter = 0;
+			private double last = System.currentTimeMillis();
+
+			@Override
+			public Iterator<String> iterator() {
+				return new Iterator<String>() {
+
+					@Override
+					public boolean hasNext() {
+						return counter < N_RECORDS;
+					}
+
+					@Override
+					public String next() {
+						if (counter % 10000 == 0) {
+							System.out.println("10K records processed in " + (System.currentTimeMillis() - last) / 1000 + " seconds");
+							last = System.currentTimeMillis();
+						}
+
+						File f = new File(String.format("/var/lib/eagle/content/EDH/HD%06d.xml", counter++));
+						if (f.exists()) {
+							try {
+								FileInputStream fileInputStream = new FileInputStream(f);
+								String s = IOUtils.toString(fileInputStream);
+								fileInputStream.close();
+								return s;
+							} catch (Exception e) {
+								return null;
+							}
+						} else {
+							try {
+								FileInputStream fileInputStream = new FileInputStream(new File("/var/lib/eagle/content/EDH/HD000001.xml"));
+								String s = IOUtils.toString(fileInputStream);
+								fileInputStream.close();
+								return s;
+							} catch (Exception e) {
+								return null;
+							}
+						}
+					}
+
+					@Override
+					public void remove() {}
+				};
+			}
+		};
+
+		MongoMDStore mdStore =
+				new MongoMDStore(UUID.randomUUID().toString(), db.getCollection("speed_test", DBObject.class), recordParserfactory.newInstance(), true, db);
+		mdStore.feed(iterable, false);
+	}
+
+	//@Ignore
+	@Test
+	public void testFeedSpeedFromTemplate() throws MDStoreServiceException, IOException {
+		MongoMDStore mdStore =
+				new MongoMDStore(UUID.randomUUID().toString(), db.getCollection("speed_test", DBObject.class), recordParserfactory.newInstance(), false, db);
+		mdStore.feed(new Iterable<String>() {
+
+			private int counter = 0;
+			private double last = System.currentTimeMillis();
+			private String templateRecord = IOUtils.toString(new ClassPathResource("/eu/dnetlib/data/mdstore/modular/mongodb/templateRecord.xml")
+					.getInputStream());
+
+			@Override
+			public Iterator<String> iterator() {
+				return new Iterator<String>() {
+
+					@Override
+					public boolean hasNext() {
+						return counter < N_RECORDS;
+					}
+
+					@Override
+					public String next() {
+						if (counter % 10000 == 0) {
+							System.out.println("10K records processed in " + (System.currentTimeMillis() - last) / 1000 + " seconds");
+							last = System.currentTimeMillis();
+						}
+
+						File f = new File(String.format("/var/lib/eagle/content/EDH/HD%06d.xml", counter++));
+						if (f.exists()) {
+							try {
+								FileInputStream fileInputStream = new FileInputStream(f);
+								String s = IOUtils.toString(fileInputStream);
+								fileInputStream.close();
+								return s;
+							} catch (Exception e) {
+								return null;
+							}
+						} else {
+							counter++;
+							try {
+								FileInputStream fileInputStream = new FileInputStream(new File("/var/lib/eagle/content/EDH/HD000009.xml"));
+								String s = IOUtils.toString(fileInputStream);
+								fileInputStream.close();
+								return s;
+							} catch (Exception e) {
+								return null;
+							}
+						}
+					}
+
+					@Override
+					public void remove() {}
+				};
+			}
+		}, false);
+	}
+}

dnet-data-services/src/test/java/eu/dnetlib/data/mdstore/modular/mongodb/IndexFieldParserTest.java

@@ -0,0 +1,51 @@
+package eu.dnetlib.data.mdstore.modular.mongodb;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import eu.dnetlib.data.mdstore.MDStoreServiceException;
+import eu.dnetlib.data.mdstore.modular.MDFormatDescription;
+import eu.dnetlib.data.mdstore.modular.mongodb.utils.IndexFieldRecordParser;
+import eu.dnetlib.data.mdstore.modular.mongodb.utils.IndexFieldRecordParserException;
+import org.apache.commons.io.IOUtils;
+import org.junit.Test;
+
+/**
+ * Created by sandro on 11/29/16.
+ */
+public class IndexFieldParserTest {
+
+
+    @Test
+    public void parserTest() throws IOException, MDStoreServiceException, IndexFieldRecordParserException {
+        InputStream inputStream = this.getClass().getResourceAsStream("/eu/dnetlib/data/mdstore/modular/mongodb/inputRecord.xml");
+
+        final String inputRecord = IOUtils.toString(inputStream);
+
+        final IndexFieldRecordParser parser = new IndexFieldRecordParser();
+
+        final List<MDFormatDescription> mdref = new ArrayList<>();
+
+        mdref.add(new MDFormatDescription("id", "//*[local-name()='objIdentifier']"));
+        mdref.add(new MDFormatDescription("title", "//*[local-name()='title']"));
+        mdref.add(new MDFormatDescription("creator", "//*[local-name()='creator']"));
+
+        Map<String, List<String>> stringListMap = parser.parseRecord(inputRecord, mdref);
+
+
+        for (String key : stringListMap.keySet()) {
+            System.out.println("key = " + key);
+            for (String value : stringListMap.get(key)) {
+                System.out.println("\t" + value);
+            }
+
+        }
+
+
+    }
+
+
+}

dnet-data-services/src/test/java/eu/dnetlib/data/mdstore/modular/mongodb/MDStoreDaoImplTest.java

@@ -0,0 +1,78 @@
+package eu.dnetlib.data.mdstore.modular.mongodb;
+
+import com.mongodb.BasicDBObject;
+import com.mongodb.DBObject;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import eu.dnetlib.data.mdstore.MDStoreServiceException;
+import eu.dnetlib.data.mdstore.modular.connector.MDStoreDBStatus;
+import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;
+import eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+
+import static org.junit.Assert.assertEquals;
+
+@Ignore
+@RunWith(SpringJUnit4ClassRunner.class)
+@ContextConfiguration(classes = ConfigurationTestConfig.class)
+public class MDStoreDaoImplTest {
+
+	@Autowired
+	private MDStoreDao dao;
+	@Autowired
+	private MongoDatabase db;
+	@Autowired
+	private MDStoreTransactionManager manager;
+
+	@After
+	public void tearDown() throws MDStoreServiceException {
+		dao.deleteMDStore("1");
+		dao.deleteMDStore("2");
+		dao.deleteMDStore("3");
+		dao.deleteMDStore("4");
+		dao.deleteMDStore("5");
+		dao.deleteMDStore("6");
+	}
+
+	@Before
+	public void setUp() throws Exception {
+		dao.createMDStore("1", "F", "I", "L");
+		dao.createMDStore("2", "F", "I", "L");
+		dao.createMDStore("3", "F", "I", "L");
+		dao.createMDStore("4", "F", "I", "L");
+		dao.createMDStore("5", "F1", "I", "L");
+		dao.createMDStore("6", "F1", "I", "L");
+
+		final MongoCollection<DBObject> metadata = db.getCollection(MDStoreDaoImpl.METADATA_NAME, DBObject.class);
+
+		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "1"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));
+		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "2"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));
+		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "3"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));
+		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "4"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));
+		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "5"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));
+		metadata.findOneAndUpdate(new BasicDBObject(MDStoreDaoImpl.MD_ID, "6"), new BasicDBObject("$set", new BasicDBObject(MDStoreDaoImpl.SIZE, 10)));
+
+	}
+
+	@Test
+	public void test() throws MDStoreServiceException {
+		assertEquals(40, dao.getSumOfSizes("F", "L", "I"));
+		assertEquals(20, dao.getSumOfSizes("F1", "L", "I"));
+		assertEquals(0, dao.getSumOfSizes("F_0", "L", "I"));
+	}
+
+	@Test
+	public void getDBStatusTest() {
+		final MDStoreDBStatus dbStatus = dao.getDBStatus();
+		System.out.println(dbStatus);
+
+	}
+
+}

dnet-data-services/src/test/java/eu/dnetlib/data/mdstore/modular/mongodb/MDStoreTransactionManagerTest.java

@@ -0,0 +1,149 @@
+package eu.dnetlib.data.mdstore.modular.mongodb;
+
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+
+import com.mongodb.DB;
+import com.mongodb.DBCollection;
+import com.mongodb.DBCursor;
+import com.mongodb.Mongo;
+import com.mongodb.client.MongoDatabase;
+import eu.dnetlib.data.mdstore.MDStoreServiceException;
+import eu.dnetlib.data.mdstore.modular.connector.MDStore;
+import eu.dnetlib.data.mdstore.modular.connector.MDStoreDao;
+import eu.dnetlib.data.mdstore.modular.connector.MDStoreTransactionManager;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+
+// TODO: reimplement tests
+
+@Ignore
+@RunWith(SpringJUnit4ClassRunner.class)
+@ContextConfiguration(classes = ConfigurationTestConfig.class)
+public class MDStoreTransactionManagerTest {
+
+	@Autowired
+	private MongoDatabase db;
+
+	@Autowired
+	private MDStoreTransactionManager manager;
+
+	@Autowired
+	private MDStoreDao dao;
+
+	@Test
+	public void testCreateandRetrieve() throws MDStoreServiceException {
+		UUID idCreation = UUID.randomUUID();
+		db.getCollection("metadataManager").drop();
+		((MDStoreTransactionManagerImpl) manager).setManagerTable(null);
+
+		String mdId = idCreation.toString() + "_TURTdG9yZURTUmVzb3VyY2VzL01EU3RvcmVEU1Jlc291cmNlVHlwZQ==";
+		manager.createMDStore(mdId);
+
+		Assert.assertNotNull(manager.startTransaction(mdId, true));
+		Assert.assertNotNull(manager.startTransaction(mdId, true));
+
+		String s = manager.getMDStoreCollection(mdId);
+		Assert.assertNotNull(s);
+		this.manager.dropMDStore(mdId);
+		s = manager.getMDStoreCollection(mdId);
+		Assert.assertNull(s);
+		db.getCollection("metadataManager").drop();
+		((MDStoreTransactionManagerImpl) manager).setManagerTable(null);
+	}
+
+	@Test
+	public void testReadMdStore() throws MDStoreServiceException {
+		UUID idCreation = UUID.randomUUID();
+		String mdId = idCreation.toString() + "_TURTdG9yZURTUmVzb3VyY2VzL01EU3RvcmVEU1Jlc291cmNlVHlwZQ==";
+		manager.createMDStore(mdId);
+		Assert.assertNotNull(manager.readMdStore(mdId));
+		Assert.assertNotNull(manager.startTransaction(mdId, true));
+		Assert.assertNotNull(manager.readMdStore(mdId));
+		db.getCollection("metadataManager").drop();
+		((MDStoreTransactionManagerImpl) manager).setManagerTable(null);
+		db.getCollection("metadataManager").drop();
+		((MDStoreTransactionManagerImpl) manager).setManagerTable(null);
+
+	}
+
+	@Test
+	public void testCommit() throws MDStoreServiceException {
+		UUID idCreation = UUID.randomUUID();
+		String mdId = idCreation.toString() + "_TURTdG9yZURTUmVzb3VyY2VzL01EU3RvcmVEU1Jlc291cmNlVHlwZQ==";
+		manager.createMDStore(mdId);
+		String idCurrent = manager.readMdStore(mdId);
+		String transaction = manager.startTransaction(mdId, true);
+		// Assert.assertTrue(manager.commit(transaction, mdId));
+		Assert.assertNotSame(idCurrent, manager.readMdStore(mdId));
+
+	}
+
+	@Ignore
+	@Test
+	public void testDateTime() throws MDStoreServiceException, UnknownHostException {
+		Mongo mongo = new Mongo("localhost", 27017);
+		DB dbinput = mongo.getDB("mdstore");
+		DBCollection inputCollection = dbinput.getCollection("70e07e9f-b3bf-4423-8777-b159819e0c6a");
+
+		Assert.assertNotNull(inputCollection.findOne().get("body"));
+		UUID idCreation = UUID.randomUUID();
+
+		String mdId = idCreation.toString() + "_TURTdG9yZURTUmVzb3VyY2VzL01EU3RvcmVEU1Jlc291cmNlVHlwZQ==";
+
+		manager.createMDStore(mdId);
+		dao.createMDStore(mdId, "a", "a", "a");
+
+		String transId = manager.startTransaction(mdId, true);
+
+		ArrayList<String> data = new ArrayList<String>();
+		DBCursor cursor = inputCollection.find();
+
+		for (int i = 0; i < 1000; i++) {
+			data.add((String) cursor.next().get("body"));
+		}
+		dao.getMDStore(transId).feed(data, true);
+		// manager.commit(transId, mdId);
+
+		cursor = inputCollection.find();
+		transId = manager.startTransaction(mdId, false);
+
+		data.clear();
+		for (int i = 0; i < 10; i++) {
+			data.add(cursor.next().get("body").toString().replace("oai:pumaoai.isti.cnr.it:", "SUUUCAAA"));
+		}
+
+		String currentId = manager.readMdStore(mdId);
+
+		final MDStore newMdstore = dao.getMDStore(currentId);
+
+		new Thread(() -> {
+			List<String> dataInput = null;
+			try {
+				dataInput = newMdstore.deliver("", "", null).getResult(0, 10);
+				for (int i = 0; i < 10; i++) {
+					try {
+						Thread.sleep(1000);
+					} catch (InterruptedException e) {
+						// TODO Auto-generated catch block
+						e.printStackTrace();
+					}
+					System.out.println(dataInput.get(i));
+				}
+			} catch (MDStoreServiceException e) {
+				e.printStackTrace();
+			}
+		}).start();
+
+		dao.getMDStore(transId).feed(data, true);
+		// manager.commit(transId, mdId);
+
+	}
+}

dnet-data-services/src/test/java/eu/dnetlib/data/mdstore/modular/mongodb/MongoBulkWritesManagerTest.java

@@ -0,0 +1,49 @@
+package eu.dnetlib.data.mdstore.modular.mongodb;
+
+import java.util.Map;
+
+import com.google.common.collect.Maps;
+import com.mongodb.DBObject;
+import com.mongodb.WriteConcern;
+import com.mongodb.client.MongoCollection;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.mockito.Mockito.when;
+
+/**
+ * Created by Alessia Bardi on 2019-04-12.
+ *
+ * @author Alessia Bardi
+ */
+@RunWith(MockitoJUnitRunner.class)
+public class MongoBulkWritesManagerTest {
+
+	private MongoBulkWritesManager mng;
+
+	@Mock
+	MongoCollection<DBObject> coll;
+
+	@Before
+	public void setUp(){
+		MockitoAnnotations.initMocks(this);
+		when(coll.withWriteConcern(WriteConcern.ACKNOWLEDGED)).thenReturn(coll);
+		mng = new MongoBulkWritesManager(coll, coll, null, 10, null, false);
+	}
+
+	@Test
+	public void buildDBObjectTest(){
+		Map<String, String> props = Maps.newHashMap();
+		props.put("timestamp", "1555078665140");
+		props.put("id", "od______4301::5af4702a60ddf0615fd1dfd6ded104df");
+		props.put("originalId", "x");
+		props.put("body","<body/>");
+
+		DBObject obj = mng.buildDBObject("<x/>", props, null);
+		System.out.println(obj);
+	}
+}

dnet-data-services/src/test/java/eu/dnetlib/enabling/tools/blackboard/BlackboardServerActionExecutorTest.java

@@ -0,0 +1,58 @@
+package eu.dnetlib.enabling.tools.blackboard;
+
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import javax.annotation.Resource;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+
+import eu.dnetlib.data.mdstore.modular.action.MDStoreActions;
+
+@RunWith(SpringJUnit4ClassRunner.class)
+@ContextConfiguration
+public class BlackboardServerActionExecutorTest {
+
+	@Resource
+	public transient BlackboardServerHandler blackboardHandler;
+
+	@Resource
+	public transient BlackboardServerActionExecutor<MDStoreActions> executor;
+
+	@Before
+	public void setUp() throws Exception {
+
+	}
+
+	@Test
+	public void testExecutor() {
+		assertNotNull(executor);
+
+		BlackboardJob job = mock(BlackboardJob.class);
+		when(job.getAction()).thenReturn("CREATE");
+
+		executor.execute(job);
+
+		verify(blackboardHandler).done(eq(job));
+	}
+
+	@Test
+	public void testExecutorUnimplemented() {
+		assertNotNull(executor);
+
+		BlackboardJob job = mock(BlackboardJob.class);
+		when(job.getAction()).thenReturn("DELETE");
+
+		executor.execute(job);
+
+		verify(blackboardHandler).failed(eq(job), (Throwable) anyObject());
+	}
+}

dnet-data-services/src/test/java/eu/dnetlib/enabling/tools/blackboard/SampleCreateAction.java

@@ -0,0 +1,12 @@
+package eu.dnetlib.enabling.tools.blackboard;
+
+import eu.dnetlib.data.mdstore.modular.action.MDStoreActions;
+
+public class SampleCreateAction implements BlackboardServerAction<MDStoreActions> {
+
+	@Override
+	public void execute(BlackboardServerHandler handler, BlackboardJob job) {
+		handler.done(job);
+	}
+
+}

dnet-data-services/src/test/java/eu/dnetlib/test/utils/MockBeanFactory.java

@@ -0,0 +1,55 @@
+package eu.dnetlib.test.utils;
+
+
+import org.springframework.beans.factory.FactoryBean;
+
+import static org.mockito.Mockito.mock;
+
+/**
+ * Return a mockito mock for a given class.
+ * This class should be updated according to new Spring4 factory Bean
+ *
+ * @author marko
+ *
+ */
+@Deprecated 
+public class MockBeanFactory implements FactoryBean {
+
+	/**
+	 * class to mock.
+	 */
+	private Class<?> clazz;
+
+	/**
+	 * {@inheritDoc}
+	 * @see FactoryBean#getObject()
+	 */
+	public Object getObject() throws Exception {
+		return mock(clazz);
+	}
+
+	/**
+	 * {@inheritDoc}
+	 * @see FactoryBean#getObjectType()
+	 */
+	public Class<?> getObjectType() {
+		return clazz;
+	}
+
+	/**
+	 * {@inheritDoc}
+	 * @see FactoryBean#isSingleton()
+	 */
+	public boolean isSingleton() {
+		return true;
+	}
+
+	public Class<?> getClazz() {
+		return clazz;
+	}
+
+	public void setClazz(final Class<?> clazz) {
+		this.clazz = clazz;
+	}
+
+}

dnet-data-services/src/test/resources/eu/dnetlib/data/mdstore/modular/mongodb/inputRecord.xml

@@ -0,0 +1,92 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<record xmlns:dc="http://purl.org/dc/elements/1.1/"
+        xmlns:dr="http://www.driver-repository.eu/namespace/dr"
+        xmlns:dri="http://www.driver-repository.eu/namespace/dri"
+        xmlns:oaf="http://namespace.openaire.eu/oaf"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+    <header xmlns="http://namespace.openaire.eu/">
+        <dri:objIdentifier>od______2367::0001a50c6388e9bfcb791a924ec4b837</dri:objIdentifier>
+        <dri:recordIdentifier>oai:pumaoai.isti.cnr.it:cnr.imati/cnr.ian.pv/1999-PP-018</dri:recordIdentifier>
+        <dri:dateOfCollection/>
+        <dri:mdFormat/>
+        <dri:mdFormatInterpretation/>
+        <dri:repositoryId>
+            10d18b66-1d2a-4579-9adc-aa57b0821c7f_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=
+        </dri:repositoryId>
+        <dr:objectIdentifier/>
+        <dr:dateOfCollection>2016-11-22T09:15:30.817Z</dr:dateOfCollection>
+        <dr:dateOfTransformation>2016-11-22T09:18:15.904Z</dr:dateOfTransformation>
+        <oaf:datasourceprefix>od______2367</oaf:datasourceprefix>
+    </header>
+    <metadata xmlns="http://namespace.openaire.eu/">
+        <dc:title>Anisotropic mechanisms for multiphasic unipolar electrograms. Simulation studies and experimental
+            recordings
+        </dc:title>
+        <dc:creator>Colli Franzone, Piero,</dc:creator>
+        <dc:creator>Guerri, Luciano,</dc:creator>
+        <dc:creator>Pennacchio, Micol,</dc:creator>
+        <dc:creator>Taccardi, Bruno</dc:creator>
+        <dc:date>1999-09-30</dc:date>
+        <dc:description>The origin of the multiple, complex morphologies observed in
+            unipolar epicardial electrograms, and their relationships with myocardial
+            architecture, have not been fully elucidated. To clarify this problem we
+            simulated electrograms (EGs) with a model representing the heart as an
+            anisotropic bidomain with unequal anisotropy ratio, ellipsoidal ventricular
+            geometry, transmural fiber rotation, epi-endocardial obliqueness of fiber
+            direction and a simplified conduction system. The electrograms were compared
+            with those directly recorded from the surface of isolated dog hearts immersed
+            in a conducting medium. The model accurately reproduced the recorded EG
+            morphologies for excitation wave fronts that reach the recording sites by
+            spreading either along or across fibers.The
+            origin of the multiple waves that constitute the QRS complex could be better
+            understood after splitting the current sources, the potential distributions and
+            the EGs into a field component (further subdivided into an axial and a conormal component) and a "reference"
+            component. The split model provides an explanation of the interaction between the three-dimensional geometry
+            and direction of
+            propagation of a spreading wave front, the architecture of the fibers through
+            which excitation is spreading, the potential distributions and the QRS wave
+            forms. Because epicardial potentials, electrograms and isochrone contours can be
+            computed noninvasively from body surface measurements, interpreting epicardial
+            EGs in terms of intramural events may have clinical relevance.
+        </dc:description>
+        <dc:identifier>http://puma.isti.cnr.it/dfdownloadnew.php?ident=cnr.imati/cnr.ian.pv/1999-PP-018</dc:identifier>
+        <dc:identifier>
+            http://puma.isti.cnr.it/rmydownload.php?filename=cnr.imati/cnr.ian.pv/1999-PP-018/1999-PP-018_0.ps
+        </dc:identifier>
+        <dc:language>eng</dc:language>
+        <dc:source>Preprint ercim.cnr.ian//1999-1151, 1999.</dc:source>
+        <dc:subject>Electrograms, bidomain model, reference potential, cardiac potential maps, anisotropic propagation,
+            source splitting
+        </dc:subject>
+        <dc:subject>info:eu-repo/classification/msc/78A70,65N30</dc:subject>
+        <dc:rights>info:eu-repo/semantics/openAccess</dc:rights>
+        <dc:type>info:eu-repo/semantics/preprint</dc:type>
+        <dr:CobjCategory>0016</dr:CobjCategory>
+        <dr:CobjIdentifier/>
+        <oaf:dateAccepted>1999-09-30</oaf:dateAccepted>
+        <oaf:collectedDatasourceid>opendoar____::2367</oaf:collectedDatasourceid>
+        <oaf:accessrights>OPEN</oaf:accessrights>
+        <oaf:hostedBy id="opendoar____::2367" name="PUblication MAnagement"/>
+        <oaf:collectedFrom id="opendoar____::2367" name="PUblication MAnagement"/>
+    </metadata>
+    <about xmlns:oai="http://www.openarchives.org/OAI/2.0/" xmlns="http://namespace.openaire.eu/">
+        <provenance xmlns="http://www.openarchives.org/OAI/2.0/provenance"
+                    xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/provenance http://www.openarchives.org/OAI/2.0/provenance.xsd">
+            <originDescription altered="true" harvestDate="2016-11-22T09:15:30.817Z">
+                <baseURL>http://pumaoai.isti.cnr.it/openoai2.php</baseURL>
+                <identifier>oai:pumaoai.isti.cnr.it:cnr.imati/cnr.ian.pv/1999-PP-018</identifier>
+                <datestamp>1999-10-08</datestamp>
+                <metadataNamespace>http://www.openarchives.org/OAI/2.0/oai_dc/</metadataNamespace>
+            </originDescription>
+        </provenance>
+        <oaf:datainfo>
+            <oaf:inferred>false</oaf:inferred>
+            <oaf:deletedbyinference>false</oaf:deletedbyinference>
+            <oaf:trust>0.9</oaf:trust>
+            <oaf:inferenceprovenance/>
+            <oaf:provenanceaction classid="sysimport:crosswalk:repository"
+                                  classname="sysimport:crosswalk:repository"
+                                  schemeid="dnet:provenanceActions" schemename="dnet:provenanceActions"/>
+        </oaf:datainfo>
+    </about>
+</record>

dnet-data-services/src/test/resources/eu/dnetlib/data/mdstore/modular/mongodb/templateRecord.xml

@@ -0,0 +1,42 @@
+<oai:record xmlns="http://namespace.openaire.eu/"
+  xmlns:dc="http://purl.org/dc/elements/1.1/"
+  xmlns:dr="http://www.driver-repository.eu/namespace/dr"
+  xmlns:dri="http://www.driver-repository.eu/namespace/dri"
+  xmlns:oai="http://www.openarchives.org/OAI/2.0/"
+  xmlns:prov="http://www.openarchives.org/OAI/2.0/provenance" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <oai:header>
+    <dri:objIdentifier>{id}</dri:objIdentifier>
+    <dri:recordIdentifier>entityId:2533216</dri:recordIdentifier>
+    <dri:dateOfCollection>2015-01-09T12:15:02.177+01:00</dri:dateOfCollection>
+    <dri:repositoryId>8e17a4fe-b77d-406b-8c75-4cfa233092d2_UmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZXMvUmVwb3NpdG9yeVNlcnZpY2VSZXNvdXJjZVR5cGU=</dri:repositoryId>
+    <dri:datasourceprefix>DAI</dri:datasourceprefix>
+  </oai:header>
+  <oai:metadata>
+    <eagle:eagleObject xmlns:eagle="http://www.eagle-network.eu/eagle">
+      <eagle:recordSourceInfo
+        landingPage="http://arachne.uni-koeln.de/entity/2533216" providerName="Arachne">entityId:2533216</eagle:recordSourceInfo>
+      <eagle:editingInfo>
+        <eagle:dateEdited>0000-00-00</eagle:dateEdited>
+        <eagle:metadataEditor>unknown</eagle:metadataEditor>
+      </eagle:editingInfo>
+      <eagle:metadataIpr uri="http://creativecommons.org/licenses/by/3.0/">This file is licensed under the Creative Commons Attribution 3.0 Unported license.</eagle:metadataIpr>
+      <eagle:title lang="de">CMS-XII-d009c-3_168111,02.jpg</eagle:title>
+      <!-- objekt http://arachne.uni-koeln.de/entity/1160926 -->
+      <eagle:description lang="de">Foto von: Siegel CMS XII D009c</eagle:description>
+      <eagle:entityType>visual</eagle:entityType>
+      <eagle:visualRepresentation>
+        <eagle:representationType>image</eagle:representationType>
+        <eagle:url>http://arachne.uni-koeln.de/entity/2533216</eagle:url>
+        <eagle:thumbnail>http://arachne.dainst.org/data/image/thumbnail/2533216</eagle:thumbnail>
+        <eagle:visualRepresentationIpr uri="http://creativecommons.org/licenses/by/3.0/">This file is licensed under the Creative Commons Attribution 3.0 Unported license.</eagle:visualRepresentationIpr>
+        <eagle:format>jpg</eagle:format>
+        <!-- objekt http://arachne.uni-koeln.de/entity/1160926 -->
+        <eagle:hasArtifact>
+          <eagle:artifactTitle>Siegel CMS XII D009c</eagle:artifactTitle>
+          <eagle:objectType>Dreiseitiges Prisma Mallia Steatitgruppe</eagle:objectType>
+          <eagle:material>weicher Stein</eagle:material>
+        </eagle:hasArtifact>
+      </eagle:visualRepresentation>
+    </eagle:eagleObject>
+  </oai:metadata>
+</oai:record>

dnet-data-services/src/test/resources/eu/dnetlib/enabling/tools/blackboard/BlackboardServerActionExecutorTest-context.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns="http://www.springframework.org/schema/beans"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:p="http://www.springframework.org/schema/p"
+	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd">
+
+	<bean id="blackboardHandler" class="eu.dnetlib.test.utils.MockBeanFactory"
+		p:clazz="eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler" />
+
+	<bean id="executor"
+		class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerActionExecutor"
+		p:blackboardHandler-ref="blackboardHandler" p:actionType="eu.dnetlib.data.mdstore.modular.action.MDStoreActions"
+		p:incomplete="true">
+		<property name="actionMap">
+			<map>
+				<entry key="CREATE">
+					<bean class="eu.dnetlib.enabling.tools.blackboard.SampleCreateAction" />
+				</entry>
+			</map>
+		</property>
+
+	</bean>
+</beans>

dnet-information-service/pom.xml

@@ -98,7 +98,10 @@
             <groupId>org.apache.cxf</groupId>
             <artifactId>cxf-rt-frontend-jaxws</artifactId>
         </dependency>
-
+        <dependency>
+            <groupId>javax.servlet</groupId>
+            <artifactId>javax.servlet-api</artifactId>
+        </dependency>
 
         <!-- Test deps -->
         <dependency>

dnet-information-service/src/main/java/eu/dnetlib/enabling/inspector/BackupController.java

@@ -0,0 +1,259 @@
+package eu.dnetlib.enabling.inspector;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.StringWriter;
+import java.util.Arrays;
+import java.util.Comparator;
+
+import javax.annotation.Resource;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.cxf.helpers.IOUtils;
+import org.springframework.stereotype.Controller;
+import org.springframework.ui.Model;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+
+import eu.dnetlib.enabling.is.sn.ISSNServiceCore;
+import eu.dnetlib.xml.database.XMLDatabase;
+
+/**
+ * This controller offers a simple way to backup the xmldb.
+ *
+ * @author michele
+ *
+ */
+@Controller
+public class BackupController extends AbstractInspectorController {
+
+	/**
+	 * BUFFER size for file copy.
+	 */
+	private static final int BUFFER_SIZE = 4096;
+
+	/**
+	 * logger.
+	 */
+	private static final Log log = LogFactory.getLog(BackupController.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	/**
+	 * xmldb.
+	 */
+	@Resource(name = "existDatabase")
+	private transient XMLDatabase xmlDatabase;
+	
+	/**
+	 * is sn subscription registries.
+	 */
+	@Resource(name = "isSNServiceCore")
+	private transient ISSNServiceCore issnServiceCore;
+	
+	private enum BackupType {
+		profile, subscription
+	}
+
+	@RequestMapping(value = "/inspector/backups.do")
+	public void backup(final Model model) {
+		// only view
+	}
+	
+	@RequestMapping(value = "/inspector/backupProfiles.do")
+	public void backupProfiles(
+			final Model model,
+			@RequestParam(value = "exec", required = false) final String exec,
+			@RequestParam(value = "delete", required = false) final String delete) {
+
+		if (exec != null && exec.equals("1")) {
+			execProfileBackup(model);
+		}
+		if (delete != null) {
+			deleteBackup(model, xmlDatabase.getBackupDir(), delete);
+		}
+
+		listBackups(model, xmlDatabase.getBackupDir());
+	}
+	
+	@RequestMapping(value = "/inspector/backupSubscriptions.do")
+	public void backupSubscriptions(
+			final Model model,
+			@RequestParam(value = "exec", required = false) final String exec,
+			@RequestParam(value = "delete", required = false) final String delete) {
+
+		if (exec != null && exec.equals("1")) {
+			execSubscriptionsBackup(model);
+		}
+		if (delete != null) {
+			deleteBackup(model, issnServiceCore.getBackupDir(), delete);
+		}
+
+		listBackups(model, issnServiceCore.getBackupDir());
+	}
+
+	/**
+	 * Executes a backup.
+	 *
+	 * @param model
+	 *            mvc model
+	 */
+	private void execProfileBackup(final Model model) {
+		try {
+			xmlDatabase.backup();
+			log.info("Backup done");
+		} catch (final Exception e) {
+			model.addAttribute("message", "Backup failed: please retry later");
+			log.fatal("Backup failed", e);
+		}
+	}
+
+	/**
+	 * Executes a backup.
+	 *
+	 * @param model
+	 *            mvc model
+	 */
+	private void execSubscriptionsBackup(final Model model) {
+		try {
+			issnServiceCore.backup();
+			log.info("Backup done");
+		} catch (final Exception e) {
+			model.addAttribute("message", "Backup failed: please retry later");
+			log.fatal("Backup failed", e);
+		}
+	}	
+	
+	/**
+	 * Deletes a backup.
+	 *
+	 * @param model
+	 *            mvc model
+	 * @param backup
+	 *            backup file name
+	 */
+	private void deleteBackup(final Model model, final String backupDir, final String backup) {
+		try {
+			final String logFile = backup.replaceFirst("data-", "report-").replace(".zip", ".log");
+			(new File(backupDir + "/" + backup)).delete();
+			(new File(backupDir + "/" + logFile)).delete();
+			log.info("Backup deleted");
+		} catch (final Exception e) {
+			model.addAttribute("message", "failed: " + e.getMessage());
+			log.info("Backup deletion failed: " + e.getMessage());
+		}
+	}
+
+	/**
+	 * List backups.
+	 *
+	 * @param model
+	 *            mvc mode
+	 */
+	private void listBackups(final Model model, final String backupDir) {
+		model.addAttribute("size", 0);
+		
+		final File dir = new File(backupDir);
+		if (dir.exists() && dir.isDirectory()) {
+			final FilenameFilter filter = new FilenameFilter() { // NOPMD
+				@Override
+				public boolean accept(final File dir, final String name) {
+					return (name.startsWith("data-") && name.endsWith(".zip"));
+				}
+			};
+			final File[] list = dir.listFiles(filter);
+			if (list != null) {
+				Arrays.sort(list, new Comparator<File>(){
+				    @Override
+					public int compare(final File f1, final File f2) {
+				        return Long.valueOf(f1.lastModified()).compareTo(f2.lastModified());
+				    } });				
+				
+				model.addAttribute("size", list.length);
+				model.addAttribute("backups", list);
+			}
+		}
+	}
+
+	/**
+	 * get a backup Log.
+	 *
+	 * @throws IOException
+	 *             could happen
+	 * @param model
+	 *            mvc model
+	 * @param backup
+	 *            the name of backup
+	 * @throws
+	 */
+	@RequestMapping(value = "/inspector/backupLog.do")
+	void backupLog(final Model model, 
+			@RequestParam(value = "backup") final String backup,
+			@RequestParam(value = "type") final String type) throws IOException {
+		
+		final String logFile = backup.replaceFirst("data-", "report-").replace(".zip", ".log");
+		
+		final FileReader freader = new FileReader(getBackupDir(type) + "/" + logFile);
+		final StringWriter blog = new StringWriter();
+
+		try {
+			IOUtils.copy(freader, blog, BUFFER_SIZE);
+		} finally {
+			freader.close();
+		}
+
+		model.addAttribute("log", blog);
+		model.addAttribute("backup", backup);
+	}
+
+	/**
+	 * Download a backup.
+	 *
+	 * @param response
+	 *            response
+	 * @param backup
+	 *            the name of backup
+	 * @param out
+	 *            response stream
+	 * @throws IOException
+	 *             could happen
+	 */
+	@RequestMapping(value = "/inspector/backupDownload.do")
+	void downloadBackup(
+			final HttpServletResponse response,
+			@RequestParam(value = "backup", required = true) final String backup,
+			@RequestParam(value = "type", required = true) final String type,
+			final OutputStream out) throws IOException {
+		response.setContentType("application/zip");
+
+		final String endUserFilename = backup.replaceFirst("data", "dlib-backup-" + type);
+		response.addHeader("Content-disposition", "attachment; filename=" + endUserFilename);
+
+		final InputStream backupStream = new FileInputStream(getBackupDir(type) + "/" + backup);
+
+		try {
+			IOUtils.copy(backupStream, out);
+		} finally {
+			backupStream.close();
+		}
+
+		out.flush();
+		out.close();
+	}
+	
+	private String getBackupDir(final String type) {
+		switch(BackupType.valueOf(type)) {
+			case profile:
+				return xmlDatabase.getBackupDir();
+			case subscription:
+				return issnServiceCore.getBackupDir();
+			default:
+				throw new IllegalArgumentException("wrong backup type parameter: " + type);
+		}
+	}	
+}

dnet-information-service/src/main/java/eu/dnetlib/enabling/inspector/GenericInfoController.java

@@ -0,0 +1,60 @@
+package eu.dnetlib.enabling.inspector;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.RuntimeMXBean;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import javax.annotation.Resource;
+
+import org.springframework.stereotype.Controller;
+import org.springframework.ui.Model;
+import org.springframework.web.bind.annotation.RequestMapping;
+
+import eu.dnetlib.miscutils.datetime.DateUtils;
+import eu.dnetlib.miscutils.datetime.HumanTime;
+
+/**
+ * This controller offers generic info about current node (ip, uptime, ...) 
+ * 
+ * @author michele
+ *
+ */
+@Controller
+public class GenericInfoController extends AbstractInspectorController {
+
+	@Resource(name="containerInfo")
+	Map<String,String> containerInfo;
+	
+	@RequestMapping(value = "/inspector/info.do")
+	void query(final Model model) {
+		RuntimeMXBean mxbean = ManagementFactory.getRuntimeMXBean();
+		
+		Map<String,String> sysinfo = new LinkedHashMap<String, String>();
+		sysinfo.put("Uptime", HumanTime.exactly(mxbean.getUptime()));
+		sysinfo.put("Start Time", DateUtils.calculate_ISO8601(mxbean.getStartTime()));
+
+		sysinfo.put("JVM Name", mxbean.getVmName());
+		sysinfo.put("JVM Vendor", mxbean.getVmVendor());
+		sysinfo.put("JVM Version", mxbean.getVmVersion());
+		sysinfo.put("JVM Spec Name", mxbean.getSpecName());
+		sysinfo.put("JVM Spec Vendor", mxbean.getSpecVendor());
+		sysinfo.put("JVM Spec Version", mxbean.getSpecVersion());
+
+		sysinfo.put("Running JVM Name", mxbean.getName());
+		sysinfo.put("Management Spec Version", mxbean.getManagementSpecVersion());
+		
+		sysinfo.put("Classpath", mxbean.getClassPath().replaceAll(":", " : "));
+		sysinfo.put("Boot ClassPath", mxbean.getBootClassPath().replaceAll(":", " : "));
+		sysinfo.put("Input arguments", mxbean.getInputArguments().toString());
+		sysinfo.put("Library Path", mxbean.getLibraryPath().replaceAll(":", " : "));
+
+		sysinfo.put("SystemProperties", mxbean.getSystemProperties().toString());
+		
+		
+		model.addAttribute("containerInfo", containerInfo);
+		model.addAttribute("sysInfo", sysinfo);
+	}
+	
+}
+

dnet-information-service/src/main/java/eu/dnetlib/enabling/inspector/QueryController.java

@@ -0,0 +1,79 @@
+package eu.dnetlib.enabling.inspector;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+import javax.annotation.Resource;
+
+import eu.dnetlib.xml.database.XMLDatabase;
+import org.apache.commons.lang3.StringEscapeUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.stereotype.Controller;
+import org.springframework.ui.Model;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.xmldb.api.base.XMLDBException;
+
+/**
+ * This controller offers a simple way to run arbitrary queries on the xmldb.
+ *
+ * @author marko
+ *
+ */
+@Controller
+public class QueryController extends AbstractInspectorController {
+
+	/**
+	 * logger.
+	 */
+	private static final Log log = LogFactory.getLog(QueryController.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	/**
+	 * xmldb.
+	 */
+	@Resource(name = "existDatabase")
+	private transient XMLDatabase xmlDatabase;
+	
+	/**
+	 * utility to parse resource ids and allows to navigate them.
+	 */
+	@Resource(name = "resourcelinkTool")
+	private ResourceLinkTool linkTool;	
+
+	/**
+	 * run a query.
+	 *
+	 * @param model
+	 *            mvc model
+	 * @param query
+	 *            query (optional)
+	 * @throws
+	 */
+	@RequestMapping(value = "/inspector/query.do")
+	void query(final Model model, @RequestParam(value = "query", required = false) final String query) {
+		if (query != null) {
+			log.info("running query: " + query);
+
+			try {
+				final Iterator<String> it = xmlDatabase.xquery(query);
+				final List<String> res = StreamSupport.stream(Spliterators.spliteratorUnknownSize(it, Spliterator.IMMUTABLE), false)
+						.map(StringEscapeUtils::escapeHtml4)
+						.map(linkTool::linkfyToHtml)
+						.collect(Collectors.toList());
+
+				model.addAttribute("size", res.size());
+				model.addAttribute("results", res);
+			} catch (XMLDBException e) {
+				model.addAttribute("message", "failed: " + e.getMessage());
+			}
+		}
+
+		model.addAttribute("query", query);
+
+	}
+
+}

dnet-information-service/src/main/java/eu/dnetlib/enabling/inspector/RegistryController.java

@@ -0,0 +1,70 @@
+package eu.dnetlib.enabling.inspector;
+
+import java.io.IOException;
+
+import javax.annotation.Resource;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.xpath.XPathExpressionException;
+
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.xml.sax.SAXException;
+
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.enabling.tools.CompatResourceIdentifierResolverImpl;
+import eu.dnetlib.enabling.tools.OpaqueResource;
+import eu.dnetlib.enabling.tools.ResourceIdentifierResolver;
+import eu.dnetlib.enabling.tools.StringOpaqueResource;
+
+/**
+ * This inspector allows the admin to quickly register a profile, like a repository profile.
+ * 
+ * @author marko
+ * 
+ */
+@Controller
+public class RegistryController extends AbstractInspectorController {
+
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	/**
+	 * manages resource identifier mappings with the abstract xmldb namespace (files/collections).
+	 */
+	@Resource
+	private ResourceIdentifierResolver resIdResolver = new CompatResourceIdentifierResolverImpl();
+
+	@RequestMapping(value = "/inspector/registerProfile.do", method = RequestMethod.GET)
+	public void registerProfile() {}
+
+	@RequestMapping(value = "/inspector/registerProfile2.do", method = RequestMethod.POST)
+	public String doRegisterProfile(
+			@RequestParam(value = "source") final String source,
+			@RequestParam(value = "valid", required = false) final String valid,
+			@RequestParam(value = "pending", required = false) final String pending) throws ISRegistryException, XPathExpressionException, SAXException,
+			IOException, ParserConfigurationException {
+
+		String id = null;
+
+		if (valid != null && !"".equals(valid)) {
+			id = serviceLocator.getService(ISRegistryService.class, true).registerProfile(source);
+		} else {
+			id = serviceLocator.getService(ISRegistryService.class, true).insertProfileForValidation(resourceTypeFor(source), source);
+		}
+
+		String collection = resIdResolver.getCollectionName(id);
+		String file = resIdResolver.getFileName(id);
+
+		return "redirect:index.do/db/DRIVER/" + collection + "/" + file + "/show";
+	}
+
+	private String resourceTypeFor(final String source) throws XPathExpressionException, SAXException, IOException, ParserConfigurationException {
+		OpaqueResource resource = new StringOpaqueResource(source);
+		return resource.getResourceType();
+	}
+
+}

dnet-information-service/src/main/java/eu/dnetlib/enabling/inspector/ResourceLinkTool.java

@@ -0,0 +1,100 @@
+package eu.dnetlib.enabling.inspector;
+
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.springframework.beans.factory.annotation.Required;
+
+import com.google.common.collect.Lists;
+
+import eu.dnetlib.enabling.tools.ResourceIdentifierResolver;
+
+/**
+ * This class allows to create links between profiles.
+ *  
+ * @author claudio
+ *
+ */
+public class ResourceLinkTool {
+
+	/**
+	 * Regular expression used to match resource ids.
+	 */
+	private static String REGEX = "([a-zA-Z0-9]+\\-)+[a-zA-Z0-9]+_[a-zA-Z0-9]+";
+	
+	/**
+	 * Base url.
+	 */
+	private String serviceBaseUrl;
+	
+	/**
+	 * ResourceIdentifierResolver.
+	 */
+	private ResourceIdentifierResolver resolver;
+	
+	/**
+	 * Method parses a profile and transforms all the ids into an html link
+	 * 
+	 * @param profile
+	 * 			the given profile.
+	 * @return
+	 * 			the linkfied profile
+	 */
+	public String linkfyToHtml(final String profile) {
+		String tmp = new String(profile);		
+		for (String id : enumerateIds(profile))
+			tmp = tmp.replaceAll(id, toLink(id));
+
+		return tmp;
+	}
+	
+	/**
+	 * Performs the actual transformation.
+	 * 
+	 * @param id
+	 * @return
+	 */
+	protected String toLink(String id) {
+		return "<a href=\"" + serviceBaseUrl + 
+					"/inspector/index.do/db/DRIVER/" + 
+					getResolver().getCollectionName(id) + "/" + 
+					getResolver().getFileName(id) + "/show\">" + id + "</a>";
+	}
+
+	/**
+	 * Lists all the ids in the given profile.
+	 * 
+	 * @param profile
+	 * @return
+	 */
+	private List<String> enumerateIds(final String profile) {
+		List<String> ids = Lists.newArrayList(); 
+		String tmp = new String(profile);
+		
+		Pattern p = Pattern.compile(REGEX);
+		Matcher m = p.matcher(tmp);
+		
+		while(m.find())
+			ids.add(m.group());
+		return ids;
+	}
+
+	@Required
+	public void setResolver(ResourceIdentifierResolver resolver) {
+		this.resolver = resolver;
+	}
+
+	public ResourceIdentifierResolver getResolver() {
+		return resolver;
+	}
+
+	@Required
+	public void setServiceBaseUrl(String serviceBaseUrl) {
+		this.serviceBaseUrl = serviceBaseUrl;
+	}
+
+	public String getServiceBaseUrl() {
+		return serviceBaseUrl;
+	}
+}

dnet-information-service/src/main/java/eu/dnetlib/enabling/inspector/ResourceTreeController.java

@@ -0,0 +1,479 @@
+package eu.dnetlib.enabling.inspector;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Resource;
+import javax.servlet.ServletOutputStream;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringEscapeUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.DocumentException;
+import org.springframework.stereotype.Controller;
+import org.springframework.ui.Model;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.xmldb.api.base.XMLDBException;
+
+import com.google.common.collect.Lists;
+
+import eu.dnetlib.miscutils.collections.MappedCollection;
+import eu.dnetlib.miscutils.functional.UnaryFunction;
+import eu.dnetlib.xml.database.XMLDatabase;
+
+/**
+ * test controller.
+ *
+ * @author marko
+ */
+@Controller
+public class ResourceTreeController extends AbstractInspectorController { // NOPMD
+
+	/**
+	 * The list view uses this model to represent a collection
+	 *
+	 * @author marko
+	 */
+	public class CollectionModel {
+
+		/**
+		 * absolute path used to query the xmldb.
+		 */
+		private String path;
+		/**
+		 * relative path, used to construct the uri used by the view.
+		 */
+		private String rel;
+
+		/**
+		 * the collection name.
+		 */
+		private String name;
+
+		public CollectionModel(final String path, final String rel, final String name) {
+			super();
+			this.path = path;
+			this.rel = rel;
+			this.name = name;
+		}
+
+		/**
+		 * We want to be able to skip useless collections which have only one child collection etc etc.
+		 * <p>
+		 * <p>
+		 * This method returns us the deepest path containing only one collection at each level
+		 * </p>
+		 *
+		 * @return list of collection names to be displayed in one "row"
+		 */
+		public Collection<CollectionModel> getCollectionPath() {
+			final ArrayList<CollectionModel> res = Lists.newArrayList(this);
+
+			try {
+				List<String> children = xmlDatabase.listChildCollections(path + '/' + name);
+				if (children.size() == 1) {
+					res.addAll(new CollectionModel(path + '/' + name, rel + '/' + name, children.get(0)).getCollectionPath());
+				}
+
+				return res;
+			} catch (XMLDBException e) {
+				return res;
+			}
+		}
+
+		/**
+		 * Uri is computed from relative base path.
+		 *
+		 * @return
+		 */
+		public String getUrl() {
+			return (rel + '/' + getName());
+		}
+
+		public String getPath() {
+			return path;
+		}
+
+		public void setPath(final String path) {
+			this.path = path;
+		}
+
+		public String getName() {
+			return name;
+		}
+
+		public void setName(final String name) {
+			this.name = name;
+		}
+
+	}
+
+	/**
+	 * base index.do path.
+	 */
+	private static final String INDEX_DO = "/inspector/index.do";
+
+	/**
+	 * logger.
+	 */
+	private static final Log log = LogFactory.getLog(ResourceTreeController.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	/**
+	 * xml database.
+	 */
+	@Resource(name = "existDatabase")
+	private transient XMLDatabase xmlDatabase;
+
+	/**
+	 * utility to parse resource ids and allows to navigate them.
+	 */
+	@Resource(name = "resourcelinkTool")
+	private eu.dnetlib.enabling.inspector.ResourceLinkTool linkTool;
+
+	/**
+	 * debug.
+	 */
+	public ResourceTreeController() {
+		super();
+		log.info("ResourceTreeController created");
+	}
+
+	/**
+	 * handles relative paths.
+	 *
+	 * @return redirect
+	 */
+	// @RequestMapping("/inspector/")
+	// String indexSlash() {
+	// return "redirect:index.do/db/list";
+	// }
+
+	/**
+	 * handles relative paths.
+	 *
+	 * @return redirect
+	 */
+	@RequestMapping("/inspector/index.do")
+	String indexDo() {
+		return "redirect:index.do/db/list";
+	}
+
+	/**
+	 * index.
+	 *
+	 * @param model   model
+	 * @param request http request
+	 * @throws XMLDBException happens
+	 */
+	@RequestMapping("/inspector/index.do/**/list")
+	String list(final Model model, final HttpServletRequest request) throws XMLDBException {
+		final String path = request.getPathInfo().replace(INDEX_DO, "").replace("/list", "");
+
+		log.debug("xml db: " + xmlDatabase);
+
+		final Collection<String> children = xmlDatabase.listChildCollections(path);
+		final Collection<String> files = xmlDatabase.list(path);
+
+		Collections.sort((List<String>) children);
+		Collections.sort((List<String>) files);
+
+		UnaryFunction<CollectionModel, String> mapper = new UnaryFunction<CollectionModel, String>() {
+
+			@Override
+			public CollectionModel evaluate(final String name) {
+				return new CollectionModel(path, ".", name);
+			}
+		};
+		final Collection<CollectionModel> richChildren = Lists.newArrayList(new MappedCollection<CollectionModel, String>(children, mapper));
+
+		model.addAttribute("path", path);
+		model.addAttribute("pathComponents", extractPathComponents(path, ""));
+		model.addAttribute("collections", richChildren);
+		model.addAttribute("files", files);
+		model.addAttribute("title", "Title");
+
+		return "inspector/index";
+	}
+
+	/**
+	 * return a list of pairs (name, relative url bases) for each path component.
+	 *
+	 * @param path slash separated path
+	 * @param base prepend this to all paths
+	 * @return list of path components
+	 */
+	private List<Map<String, String>> extractPathComponents(final String path, final String base) {
+		final String[] rawPathComponents = path.split("/");
+		final List<Map<String, String>> pathComponents = new ArrayList<Map<String, String>>();
+		for (String rawPathComponent : rawPathComponents) {
+			final Map<String, String> pathElement = new HashMap<String, String>(); // NOPMD
+			pathElement.put("name", rawPathComponent);
+
+			pathComponents.add(pathElement);
+		}
+		Collections.reverse(pathComponents);
+		final StringBuffer current = new StringBuffer(base); // NOPMD
+		for (Map<String, String> pathComponent : pathComponents) {
+			pathComponent.put("url", current.toString());
+			current.append("../");
+		}
+		Collections.reverse(pathComponents);
+		return pathComponents;
+	}
+
+	/**
+	 * show a file.
+	 *
+	 * @param model   model
+	 * @param request request
+	 * @return view name
+	 * @throws XMLDBException happens
+	 */
+	@RequestMapping("/inspector/index.do/**/show")
+	String show(final Model model, final HttpServletRequest request) throws XMLDBException {
+
+		final String path = request.getPathInfo().replace(INDEX_DO, "").replace("/show", "");
+		log.info("index: " + path);
+
+		final File fileHelper = new File(path);
+		final String collection = fileHelper.getParent();
+		final String fileName = fileHelper.getName();
+		final TouchUtils touch = new TouchUtils();
+
+		String file = xmlDatabase.read(fileName, collection);
+		if (file == null) {
+			file = "no such file, click on edit to create";
+		}
+
+		file = touch.spanize(file);
+		file = StringEscapeUtils.escapeHtml4(file);
+		file = touch.escape(file);
+		// log.info("XML ESCAPED:" + file);
+
+		model.addAttribute("file", linkTool.linkfyToHtml(file));
+		model.addAttribute("pathComponents", extractPathComponents(collection, "../"));
+
+		return "inspector/show";
+	}
+
+	@RequestMapping("/inspector/index.do/**/touch")
+	public void updateDate(@RequestParam(value = "xpath", required = true) final String xpath,
+			final HttpServletRequest request,
+			final HttpServletResponse response) throws XMLDBException {
+
+		final String path = request.getPathInfo().replace(INDEX_DO, "").replace("/touch", "");
+		final File fileHelper = new File(path);
+		final String collection = fileHelper.getParent();
+		final String fileName = fileHelper.getName();
+		final TouchUtils touch = new TouchUtils();
+
+		String file = xmlDatabase.read(fileName, collection);
+		if (file != null) {
+			String updatedProfile;
+			try {
+				updatedProfile = touch.updateProfile(file, xpath);
+				xmlDatabase.update(fileName, collection, updatedProfile);
+			} catch (DocumentException e) {
+				log.warn(e);
+			}
+
+		}
+	}
+
+	/**
+	 * Show raw profile.
+	 *
+	 * @param request  servlet request
+	 * @param response servlet response
+	 * @throws XMLDBException could happen
+	 * @throws IOException
+	 */
+	@RequestMapping("/inspector/index.do/**/raw")
+	public void raw(final HttpServletRequest request, final HttpServletResponse response) throws XMLDBException, IOException {
+
+		final String path = request.getPathInfo().replace(INDEX_DO, "").replace("/raw", "");
+		// log.info("index: " + path);
+
+		final File fileHelper = new File(path);
+		final String collection = fileHelper.getParent();
+		final String fileName = fileHelper.getName();
+
+		String file = xmlDatabase.read(fileName, collection);
+		if (file == null) {
+			file = "no such file to show";
+		}
+
+		response.setContentType("text/xml");
+		ServletOutputStream out = response.getOutputStream();
+		IOUtils.copy(new StringReader(file), out);
+		out.flush();
+		out.close();
+	}
+
+	/**
+	 * show a file editor.
+	 *
+	 * @param model   model
+	 * @param request request
+	 * @return view name
+	 * @throws XMLDBException happens
+	 */
+	@RequestMapping("/inspector/index.do/**/edit")
+	public String edit(final Model model, final HttpServletRequest request) throws XMLDBException {
+
+		final String path = request.getPathInfo().replace(INDEX_DO, "").replace("/edit", "");
+
+		final File fileHelper = new File(path);
+		final String collection = fileHelper.getParent();
+		final String fileName = fileHelper.getName();
+
+		final String file = xmlDatabase.read(fileName, collection);
+		if (file == null) {
+			model.addAttribute("creating", "true");
+		}
+
+		model.addAttribute("file", StringEscapeUtils.escapeHtml4(file));
+		model.addAttribute("pathComponents", extractPathComponents(collection, "../"));
+
+		return "inspector/edit";
+	}
+
+	/**
+	 * update or create a file.
+	 *
+	 * @param model   model
+	 * @param request request
+	 * @return view name
+	 * @throws XMLDBException happens
+	 */
+	@RequestMapping("/inspector/index.do/**/save")
+	public String save(final Model model, final HttpServletRequest request) throws XMLDBException {
+
+		final String path = request.getPathInfo().replace(INDEX_DO, "").replace("/save", "");
+
+		final File fileHelper = new File(path);
+		final String collection = fileHelper.getParent();
+		final String fileName = fileHelper.getName();
+
+		log.info("saving: " + path);
+		final String source = request.getParameter("source");
+
+		if ("true".equals(request.getParameter("creating"))) {
+			xmlDatabase.create(fileName, collection, source);
+		} else {
+			xmlDatabase.update(fileName, collection, source);
+		}
+
+		return "redirect:show";
+	}
+
+	/**
+	 * delete a file.
+	 *
+	 * @param model   model
+	 * @param request request
+	 * @return view name
+	 * @throws XMLDBException happens
+	 */
+	@RequestMapping("/inspector/index.do/**/delete")
+	public String delete(final Model model, final HttpServletRequest request) throws XMLDBException {
+
+		final String path = request.getPathInfo().replace(INDEX_DO, "").replace("/delete", "");
+
+		final File fileHelper = new File(path);
+		final String collection = fileHelper.getParent();
+		final String fileName = fileHelper.getName();
+
+		log.info("deleting: " + path);
+		xmlDatabase.remove(fileName, collection);
+
+		return "redirect:../list";
+	}
+
+	/**
+	 * delete a collection.
+	 *
+	 * @param model   model
+	 * @param request request
+	 * @return view name
+	 * @throws XMLDBException happens
+	 */
+	@RequestMapping("/inspector/index.do/**/deleteCollection")
+	public String deleteCollection(final Model model, final HttpServletRequest request) throws XMLDBException {
+
+		final String path = request.getPathInfo().replace(INDEX_DO, "").replace("/deleteCollection", "");
+
+		xmlDatabase.removeCollection(path);
+
+		return "redirect:../list";
+	}
+
+	/**
+	 * present a create form for new subcollection
+	 *
+	 * @param model   model
+	 * @param request request
+	 * @return view name
+	 * @throws XMLDBException happens
+	 */
+	@RequestMapping("/inspector/index.do/**/createsubcoll")
+	public String createSubCollection(final Model model, final HttpServletRequest request) throws XMLDBException {
+		return "inspector/createsubcoll";
+	}
+
+	/**
+	 * @param request request
+	 * @return view name
+	 * @throws XMLDBException happens
+	 */
+	@RequestMapping("/inspector/index.do/**/savesubcoll")
+	public String saveSubCollection(final HttpServletRequest request, @RequestParam("collectionPath") final String collectionPath)
+			throws XMLDBException {
+		final String path = request.getPathInfo().replace(INDEX_DO, "").replace("/savesubcoll", "");
+		String fullPath = path + "/" + collectionPath;
+		log.info("Creating subcollection: " + fullPath);
+		if (!xmlDatabase.collectionExists(fullPath)) {
+			xmlDatabase.createCollection(fullPath);
+		} else {
+			log.info("Subcollection: " + fullPath + " already exists");
+		}
+		return "redirect:../list";
+	}
+
+	/**
+	 * present a create form which will redirect to the edit form.
+	 *
+	 * @param model   model
+	 * @param request request
+	 * @return view name
+	 * @throws XMLDBException happens
+	 */
+	@RequestMapping("/inspector/index.do/**/create")
+	public String create(final Model model, final HttpServletRequest request) throws XMLDBException {
+		return "inspector/create";
+	}
+
+	/**
+	 * sample controller.
+	 *
+	 * @param model model
+	 */
+	@RequestMapping("/inspector/gadget.do")
+	public void gadget(final Model model) {
+		log.info("GADGED CALLED");
+
+		model.addAttribute("items", new String[] { "one", "two", "three" });
+	}
+
+}

dnet-information-service/src/main/java/eu/dnetlib/enabling/inspector/SubscriptionController.java

@@ -0,0 +1,200 @@
+package eu.dnetlib.enabling.inspector;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.annotation.Resource;
+import javax.xml.transform.dom.DOMResult;
+import javax.xml.xpath.XPathExpressionException;
+import javax.xml.xpath.XPathFactory;
+
+import eu.dnetlib.enabling.is.sn.SubscriptionRegistry;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.stereotype.Controller;
+import org.springframework.ui.Model;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+
+import eu.dnetlib.enabling.is.sn.NotificationInvocationLogger;
+import eu.dnetlib.enabling.is.sn.resourcestate.ResourceStateSubscription;
+import eu.dnetlib.enabling.is.sn.resourcestate.ResourceStateSubscriptionRegistry;
+import eu.dnetlib.miscutils.coupling.StaticCondition;
+
+/**
+ * Low-level basic interface for managing SN subscriptions.
+ *
+ * @author marko
+ *
+ */
+@Controller
+public class SubscriptionController extends AbstractInspectorController {
+	/**
+	 * logger.
+	 */
+	private static final Log log = LogFactory.getLog(SubscriptionController.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	/**
+	 * is sn subscription registries.
+	 */
+	@Resource(name = "issResourceStateNotificationRegistry")
+	private transient SubscriptionRegistry registry;
+
+	/**
+	 * allows to control the global enable of issn.
+	 */
+	@Resource(name = "issnInhibitionCondition")
+	private transient StaticCondition inhibitionCondition;
+
+	/**
+	 * issn invocation logger.
+	 */
+	@Resource
+	private transient NotificationInvocationLogger invocationLogger;
+
+	/**
+	 * show a list of subscriptions.
+	 *
+	 * @param model
+	 *            view parameters
+	 * @return view
+	 * @throws XPathExpressionException
+	 *             could happen
+	 */
+	@RequestMapping("/inspector/sn.do")
+	String listSubscriptions(final Model model) throws XPathExpressionException {
+
+		log.debug("registries: " + registry);
+
+		final List<Map<String, String>> subscriptions = new ArrayList<Map<String, String>>();
+
+		for (ResourceStateSubscription sub : registry.listSubscriptions()) {
+			final Map<String, String> attrs = new HashMap<String, String>(); // NOPMD
+			attrs.put("prefix", sub.getPrefix());
+			attrs.put("type", sub.getType());
+			attrs.put("resourceId", sub.getResourceId());
+			attrs.put("xpath", sub.getXpath());
+			attrs.put("id", sub.getSubscriptionId());
+
+			final DOMResult result = new DOMResult(); // NOPMD
+			sub.getSubscriberAsEpr().writeTo(result);
+			attrs.put("address", XPathFactory.newInstance().newXPath().evaluate("//*[local-name() = 'Address']", result.getNode()));
+			subscriptions.add(attrs);
+		}
+
+		model.addAttribute("subscriptions", subscriptions);
+		model.addAttribute("enabled", !inhibitionCondition.isTrue());
+		return "inspector/subscriptions";
+	}
+	
+	/**
+	 * show a list of subscriptions.
+	 *
+	 * @param model
+	 *            view parameters
+	  @param address
+	 *            the address prefix used as filter
+	 * @return view
+	 * @throws XPathExpressionException
+	 *             could happen
+	 */
+	@RequestMapping("/inspector/snByAddress.do")
+	String listSubscriptionsByAddress(final Model model, @RequestParam(value="address", required=false) final String address) throws XPathExpressionException {
+
+		log.debug("registries: " + registry);
+
+		final List<Map<String, String>> subscriptions = new ArrayList<Map<String, String>>();
+		log.debug("Address is "+address);
+		
+			model.addAttribute("address", address);
+			
+			for (ResourceStateSubscription sub : registry.listSubscriptions()) {
+				final DOMResult result = new DOMResult(); // NOPMD
+				sub.getSubscriberAsEpr().writeTo(result);
+				final String addr = XPathFactory.newInstance().newXPath().evaluate("//*[local-name() = 'Address']", result.getNode());
+
+				if ( address==null || (addr != null && addr.startsWith(address)) ) {
+					final Map<String, String> attrs = new HashMap<String, String>(); // NOPMD
+					attrs.put("prefix", sub.getPrefix());
+					attrs.put("type", sub.getType());
+					attrs.put("resourceId", sub.getResourceId());
+					attrs.put("xpath", sub.getXpath());
+					attrs.put("id", sub.getSubscriptionId());
+                    attrs.put("address", addr);
+					subscriptions.add(attrs);
+				}
+			}
+
+		model.addAttribute("subscriptions", subscriptions);
+		model.addAttribute("enabled", !inhibitionCondition.isTrue());
+		return "inspector/subscriptionsByAddress";
+	}
+	/**
+	 * toggle global notification inhibit flag.
+	 *
+	 * @return view name (redirect)
+	 */
+	@RequestMapping("/inspector/toggleNotifications.do")
+	String deleteSubscription() {
+		inhibitionCondition.setCondition(! inhibitionCondition.isTrue());
+		return "redirect:sn.do";
+	}
+
+	/**
+	 * delete a issn subscription.
+	 *
+	 * @param subscriptionId
+	 *            subscription identifier web parameter
+	 * @return view name (redirect)
+	 */
+	@RequestMapping("/inspector/deleteSubscription.do")
+	String deleteSubscription(@RequestParam("id") final String subscriptionId) {
+		registry.removeSubscription(subscriptionId);
+		return "redirect:sn.do";
+	}
+	
+	/**
+	 * delete issn subscriptions by address prefix.
+	 * @param model
+	 *            view parameters
+	 * @param address
+	 *            address prefix 
+	 * @return view name (redirect)
+	 * @throws XPathExpressionException 
+	 */
+	@RequestMapping("/inspector/deleteSubscriptionsByAddress.do")
+	String deleteSubscriptionsByAddress(final Model model, @RequestParam("address") final String address) throws XPathExpressionException {
+		
+		final List<String> deleted = new ArrayList<String>();
+		
+		if (address != null && address.length() > "http://".length()) {
+
+			for (ResourceStateSubscription sub : registry.listSubscriptions()) {
+				final DOMResult result = new DOMResult(); // NOPMD
+				sub.getSubscriberAsEpr().writeTo(result);
+				final String addr = XPathFactory.newInstance().newXPath().evaluate("//*[local-name() = 'Address']", result.getNode());
+
+				if (addr != null && addr.startsWith(address)) {
+					String id = sub.getSubscriptionId();
+					registry.removeSubscription(id);
+					deleted.add(id + " ("+ addr +")");
+				}
+			}
+		}
+		
+		model.addAttribute("deleted", deleted);
+		
+		return "inspector/deleteSubscriptionsByAddress";
+	}
+	/**
+	 * Show log of notifications.
+	 *
+	 * @param model mvc model
+	 */
+	@RequestMapping("/inspector/notificationLog.do")
+	void notificationLog(final Model model) {
+		model.addAttribute("log", invocationLogger.getEntries());
+	}
+}

dnet-information-service/src/main/java/eu/dnetlib/enabling/inspector/TouchUtils.java

@@ -0,0 +1,131 @@
+package eu.dnetlib.enabling.inspector;
+
+import java.io.StringReader;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.dom4j.Attribute;
+import org.dom4j.Document;
+import org.dom4j.DocumentException;
+import org.dom4j.Element;
+import org.dom4j.Node;
+import org.dom4j.io.SAXReader;
+
+import com.google.common.collect.Maps;
+
+import eu.dnetlib.miscutils.datetime.DateUtils;
+import eu.dnetlib.miscutils.dom4j.XPathHelper;
+
+/**
+ * Utility class that provides the "touch" functionality
+ * 
+ * @author claudio
+ *
+ */
+public class TouchUtils {
+
+	/**
+	 * logger
+	 */
+	private static final Log log = LogFactory.getLog(TouchUtils.class);
+	
+	/**
+	 * Updates the date value of the given xpath
+	 * 
+	 * @param file
+	 * @param xpath
+	 * @return
+	 * @throws DocumentException
+	 */
+	public String updateProfile(String file, String xpath) throws DocumentException {
+		SAXReader reader = new SAXReader();
+
+		log.debug("updating XPath: " + xpath);
+		Document document = reader.read(new StringReader(file));
+
+		Node node = document.selectSingleNode(xpath);
+
+		if (node == null || (node.getNodeType() != Node.ELEMENT_NODE && node.getNodeType() != Node.ATTRIBUTE_NODE))
+			throw new DocumentException("Invalid xpath: " + xpath + ": " + node);
+
+		if (verifyIsDate(node.getText()))
+			node.setText(DateUtils.now_ISO8601());
+
+		return document.asXML();
+	}
+
+	/**
+	 * 
+	 * @param text
+	 *            is the value to check
+	 * @return true if text is a well formatted timestamp
+	 */
+	protected boolean verifyIsDate(String text) {
+		try {
+			new DateUtils().parse(text);
+			return true;
+		} catch (IllegalStateException e) {
+			return false;
+		}
+	}
+
+	/**
+	 * Adds "<span id='xpath'>item</span>" where: - item is an element or an attribute name - xpath is the corresponding
+	 * xpath
+	 * 
+	 * @param file
+	 * @return the "spanized" xml file
+	 */
+	protected String spanize(String file) {
+		SAXReader reader = new SAXReader();
+		try {
+			Document document = reader.read(new StringReader(file));
+			traverse(document.getRootElement(), "/*");
+			
+			//TODO find a solution to this crime (damn dom4j!)
+			return document.asXML().replace("&apos;", "'");
+		} catch (DocumentException e) {
+			log.warn(e);
+			return e.getMessage();
+		}
+	}
+
+	private void traverse(Element element, String xpathFather) {
+		int i = 1;
+
+		for (Element node : XPathHelper.selectElements(element, "./*")) {
+			String xpath = xpathFather + "/*[" + (i++) + "]";
+
+			Map<Attribute, String> attributeXPaths = Maps.newHashMap();
+
+			@SuppressWarnings("unchecked")
+			List<Attribute> attributes = (List<Attribute>) node.attributes();
+			for (Attribute attribute : attributes)
+				attributeXPaths.put(attribute, xpath + "/@" + attribute.getName());
+
+			node.attributes().clear();
+			for (Entry<Attribute, String> entry : attributeXPaths.entrySet()) {
+				Attribute attribute = entry.getKey();
+				String modifiedName = "_|span id='" + entry.getValue() + "'|_" + attribute.getName() + "_|/span|_";
+
+				node.addAttribute(modifiedName, attribute.getValue());
+			}
+			if (node.isTextOnly())
+				node.setName("_|span id='" + xpath + "'|_" + node.getName() + "_|/span|_");
+			traverse(node, xpath);
+		}
+	}
+
+	/**
+	 * 
+	 * @param file
+	 * @return
+	 */
+	public String escape(String file) {
+		return file.replace("_|", "<").replace("|_", ">");
+	}
+	
+}

dnet-information-service/src/main/java/eu/dnetlib/enabling/inspector/ValidationController.java

@@ -0,0 +1,159 @@
+package eu.dnetlib.enabling.inspector;
+
+import java.io.File;
+import java.io.IOException;
+
+import javax.annotation.Resource;
+import javax.servlet.http.HttpServletRequest;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.xpath.XPathExpressionException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.stereotype.Controller;
+import org.springframework.ui.Model;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.xml.sax.SAXException;
+import org.xmldb.api.base.XMLDBException;
+
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.enabling.tools.OpaqueResource;
+import eu.dnetlib.enabling.tools.ResourceIdentifierResolver;
+import eu.dnetlib.enabling.tools.StringOpaqueResource;
+import eu.dnetlib.xml.database.XMLDatabase;
+
+/**
+ * Implement mid-level functionality for validation and invalidation of profiles browsed with the ResourceTreeController.
+ * 
+ * @see ResourceTreeController
+ * @author marko
+ * 
+ */
+@Controller
+public class ValidationController extends AbstractInspectorController {
+
+	/**
+	 * logger.
+	 */
+	private static final Log log = LogFactory.getLog(ValidationController.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	/**
+	 * base index.do path.
+	 */
+	private static final String INDEX_DO = "/inspector/index.do";
+
+	/**
+	 * xml database.
+	 */
+	@Resource(name = "existDatabase")
+	private transient XMLDatabase xmlDatabase;
+
+	/**
+	 * service locator.
+	 */
+	@Resource
+	private UniqueServiceLocator serviceLocator;
+
+	/**
+	 * resolve xmldb ids to resource ids.
+	 */
+	@Resource
+	private transient ResourceIdentifierResolver resIdResolver;
+
+	/**
+	 * high level: profile validation via registry.
+	 * 
+	 * @param model
+	 *            model
+	 * @param request
+	 *            request
+	 * @return view name
+	 * @throws XMLDBException
+	 *             happens
+	 * @throws ParserConfigurationException
+	 *             shouldn't happen
+	 * @throws IOException
+	 *             shouldn't happen
+	 * @throws SAXException
+	 *             shouldn't happen
+	 * @throws XPathExpressionException
+	 *             shouldn't happen
+	 * @throws ISRegistryException
+	 *             happens
+	 * @throws ISLookUpException
+	 *             happens
+	 */
+	@RequestMapping("/inspector/index.do/**/validate")
+	String validate(final Model model, final HttpServletRequest request) throws XMLDBException, XPathExpressionException, SAXException, IOException,
+			ParserConfigurationException, ISRegistryException, ISLookUpException {
+
+		final String path = request.getPathInfo().replace(INDEX_DO, "").replace("/validate", "");
+
+		final File fileHelper = new File(path);
+		final String collection = fileHelper.getParent();
+		final String fileName = fileHelper.getName();
+
+		log.info("validating: " + path);
+		final OpaqueResource resource = new StringOpaqueResource(xmlDatabase.read(fileName, collection));
+		log.info("validating a " + resource.getResourceType());
+
+		final String newId = serviceLocator.getService(ISRegistryService.class, true).validateProfile(resource.getResourceId());
+		final OpaqueResource valid = new StringOpaqueResource(serviceLocator.getService(ISLookUpService.class).getResourceProfile(newId));
+
+		final String vFileName = resIdResolver.getFileName(valid.getResourceId());
+		final String vCollectionName = resIdResolver.getCollectionName(valid.getResourceId());
+
+		return "redirect:../../../" + vCollectionName + "/" + vFileName + "/show";
+	}
+
+	/**
+	 * high level: profile invalidation via registry.
+	 * 
+	 * @param model
+	 *            model
+	 * @param request
+	 *            request
+	 * @return view name
+	 * @throws XMLDBException
+	 *             happens
+	 * @throws ParserConfigurationException
+	 *             shouldn't happen
+	 * @throws IOException
+	 *             shouldn't happen
+	 * @throws SAXException
+	 *             shouldn't happen
+	 * @throws XPathExpressionException
+	 *             shouldn't happen
+	 * @throws ISRegistryException
+	 *             happens
+	 * @throws ISLookUpException
+	 *             happens
+	 */
+	@RequestMapping("/inspector/index.do/**/invalidate")
+	String invalidate(final Model model, final HttpServletRequest request) throws XMLDBException, XPathExpressionException, SAXException, IOException,
+			ParserConfigurationException, ISRegistryException, ISLookUpException {
+
+		final String path = request.getPathInfo().replace(INDEX_DO, "").replace("/invalidate", "");
+
+		final File fileHelper = new File(path);
+		final String collection = fileHelper.getParent();
+		final String fileName = fileHelper.getName();
+
+		log.info("invalidating: " + path);
+		final OpaqueResource resource = new StringOpaqueResource(xmlDatabase.read(fileName, collection));
+		log.info("invalidating a " + resource.getResourceType());
+
+		final String newId = serviceLocator.getService(ISRegistryService.class, true).invalidateProfile(resource.getResourceId());
+		final OpaqueResource invalid = new StringOpaqueResource(serviceLocator.getService(ISLookUpService.class).getResourceProfile(newId));
+
+		final String vFileName = resIdResolver.getFileName(invalid.getResourceId());
+		final String vCollectionName = resIdResolver.getCollectionName(invalid.getResourceId());
+
+		return "redirect:../../../" + vCollectionName + "/" + vFileName + "/show";
+	}
+
+}

dnet-information-service/src/main/resources/eu/dnetlib/enabling/inspector/applicationContext-enabling-inspector.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns="http://www.springframework.org/schema/beans"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:context="http://www.springframework.org/schema/context"
+	xmlns:jee="http://www.springframework.org/schema/jee" xmlns:lang="http://www.springframework.org/schema/lang"
+	xmlns:p="http://www.springframework.org/schema/p" xmlns:tx="http://www.springframework.org/schema/tx"
+	xmlns:util="http://www.springframework.org/schema/util"
+	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+		http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
+		http://www.springframework.org/schema/jee http://www.springframework.org/schema/jee/spring-jee.xsd
+		http://www.springframework.org/schema/lang http://www.springframework.org/schema/lang/spring-lang.xsd
+		http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx.xsd
+		http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
+
+	<bean id="enablingInspectorJmxExporter" 
+		class="org.springframework.jmx.export.MBeanExporter">
+		<property name="beans">
+			<map>
+				<entry key="xmlDb:name=settings" value-ref="existDatabase" />
+			</map>
+		</property>
+	</bean>
+	
+	<bean id="resourcelinkTool"
+		class="eu.dnetlib.enabling.inspector.ResourceLinkTool"
+		p:resolver-ref="resourceIdentifierResolver"
+		p:serviceBaseUrl="/${container.context}/mvc"/>
+
+<!-- http://${container.hostname}:${container.port}/${container.context}/mvc -->
+	
+</beans>
+

dnet-information-service/src/main/resources/eu/dnetlib/enabling/inspector/webContext-enabling-inspector.xml

@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns="http://www.springframework.org/schema/beans"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:context="http://www.springframework.org/schema/context"
+	xmlns:p="http://www.springframework.org/schema/p" xmlns:util="http://www.springframework.org/schema/util"
+	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+		http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
+		http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
+
+	<bean id="enablingInspectorGroup"
+		class="eu.dnetlib.enabling.inspector.StaticEntryPointDescriptorGroup"
+		p:name="enabling">
+		<property name="descriptors">
+			<list>
+				<bean class="eu.dnetlib.enabling.inspector.StaticEntryPointDescriptor"
+					p:name="profiles" p:relativeUrl="index.do" />
+				<bean class="eu.dnetlib.enabling.inspector.StaticEntryPointDescriptor"
+					p:name="registry" p:relativeUrl="registerProfile.do" />
+				<bean class="eu.dnetlib.enabling.inspector.StaticEntryPointDescriptor"
+					p:name="xquery" p:relativeUrl="query.do" />
+				<bean class="eu.dnetlib.enabling.inspector.StaticEntryPointDescriptor"
+					p:name="backup" p:relativeUrl="backups.do" />
+				<bean class="eu.dnetlib.enabling.inspector.StaticEntryPointDescriptor"
+					p:name="subscriptions" p:relativeUrl="sn.do" />
+				<bean class="eu.dnetlib.enabling.inspector.StaticEntryPointDescriptor"
+					p:name="info" p:relativeUrl="info.do" p:hiddenAsDefault="true" />
+			</list>
+		</property>
+	</bean>
+	
+	<util:map id="containerInfo" map-class="java.util.LinkedHashMap">
+		<entry key="Hostname" value="${container.hostname}" />
+		<entry key="Port" value="${container.port}" />
+		<entry key="Context" value="${container.context}" />
+	</util:map>
+</beans>

dnet-information-service/src/main/resources/eu/dnetlib/enabling/views/inspector/backupLog.st

@@ -0,0 +1,4 @@
+$inspector/master(it={
+<h2>Log of backup $backup$</h2>
+<textarea rows="50" cols="100" readonly="readonly">$log$</textarea>
+})$

dnet-information-service/src/main/resources/eu/dnetlib/enabling/views/inspector/backupProfiles.st

@@ -0,0 +1,41 @@
+$inspector/master(it={
+
+<h2>Xmldb backups</h2>
+
+<form method="POST" action="backupProfiles.do">
+	<input type="hidden" name="exec" value="1" />
+	<input type="submit" value="Execute a new backup"/>
+</form>
+
+<p>$message$</p>
+
+<p>Backups (size = $size$)</p>
+<table border="1">
+$backups:{
+	<tr>
+		<td>$it$</td>
+		<td>
+			<form method="POST" action="backupDownload.do">
+				<input type="hidden" name="backup" value="$it.name$" />
+				<input type="hidden" name="type" value="profile" />
+				<input type="submit" value="download"/>
+			</form>
+		</td>
+		<td>
+			<form method="POST" action="backupLog.do">
+				<input type="hidden" name="backup" value="$it.name$" />
+				<input type="hidden" name="type" value="profile" />
+				<input type="submit" value="log"/>
+			</form>
+		</td>
+		<td>
+			<form method="POST" action="backupProfiles.do">
+				<input type="hidden" name="delete" value="$it.name$" />
+				<input type="submit" value="delete"/>
+			</form>
+		</td>
+	<tr>
+}$
+<table>
+
+})$

dnet-information-service/src/main/resources/eu/dnetlib/enabling/views/inspector/backupSubscriptions.st

@@ -0,0 +1,41 @@
+$inspector/master(it={
+
+<h2>Subscription backups</h2>
+
+<form method="POST" action="backupSubscriptions.do">
+	<input type="hidden" name="exec" value="1" />
+	<input type="submit" value="Execute a new backup"/>
+</form>
+
+<p>$message$</p>
+
+<p>Backups (size = $size$)</p>
+<table border="1">
+$backups:{
+	<tr>
+		<td>$it$</td>
+		<td>
+			<form method="POST" action="backupDownload.do">
+				<input type="hidden" name="backup" value="$it.name$" />
+				<input type="hidden" name="type" value="subscription" />
+				<input type="submit" value="download"/>
+			</form>
+		</td>
+		<td>
+			<form method="POST" action="backupLog.do">
+				<input type="hidden" name="backup" value="$it.name$" />
+				<input type="hidden" name="type" value="subscription" />
+				<input type="submit" value="log"/>
+			</form>
+		</td>
+		<td>
+			<form method="POST" action="backupSubscriptions.do">
+				<input type="hidden" name="delete" value="$it.name$" />
+				<input type="submit" value="delete"/>
+			</form>
+		</td>
+	<tr>
+}$
+<table>
+
+})$

dnet-information-service/src/main/resources/eu/dnetlib/enabling/views/inspector/backups.st

@@ -0,0 +1,8 @@
+$inspector/master(it={
+
+<h2>Database backups</h2>
+
+<p><a href="backupProfiles.do">backup xmldb</a></p>
+<p><a href="backupSubscriptions.do">backup subscriptions</a></p>
+
+})$

dnet-information-service/src/main/resources/eu/dnetlib/enabling/views/inspector/create.st

@@ -0,0 +1,10 @@
+$inspector/master(it={
+$inspector/path()$
+
+<ul id="subnav">
+<li><a href="javascript:window.location=document.getElementById('filename').value + '/edit'">create</a></li></li>
+</ul>
+
+<input id="filename" type="text" name="name" value=""/>
+
+})$

dnet-information-service/src/main/resources/eu/dnetlib/enabling/views/inspector/createsubcoll.st

@@ -0,0 +1,13 @@
+$inspector/master(it={
+<h2>Database inspector - new subcollection</h2>
+$inspector/path()$
+
+<ul id="subnav">
+<li><a href="javascript:document.forms[0].submit()">save</a></li></li>
+</ul>
+
+<form action="savesubcoll" method="POST" >
+<input id="collectionPath" type="text" name="collectionPath" value="" style="width:100em"/>
+</form>
+
+})$

dnet-information-service/src/main/resources/eu/dnetlib/enabling/views/inspector/deleteSubscriptionsByAddress.st

@@ -0,0 +1,22 @@
+$inspector/master(it={
+<style type="text/css">
+#subscriptions { list-style-type: }
+</style>
+
+<h2>Subscription &amp; Notifications (by address)</h2>
+
+<p>
+<a href="snByAddress.do">Return</a>
+</p>
+
+<p>Deleted notification:</p>
+
+
+<ol>
+$deleted:{
+	<li>$it$</li>
+}$
+</ol>
+
+
+})$

dnet-information-service/src/main/resources/eu/dnetlib/enabling/views/inspector/edit.st

@@ -0,0 +1,22 @@
+$inspector/master(it={
+<h2>Database inspector - editing</h2>
+$inspector/path()$
+
+<ul id="subnav">
+<li><a href="javascript:document.forms[0].submit()">save</a></li></li>
+</ul>
+
+<form action="save" method="POST" >
+$if(creating)$
+ <p>creating new file:</p>
+$endif$
+
+<p>You can edit the profile from within the browser. Upon save it will be checked for schema compliance.</p>
+<textarea name="source" class="profile" id="profile" autofocus>
+$file$
+</textarea>
+
+ <input type="hidden" name="creating" value="$creating$"/>
+</form>
+
+})$

dnet-information-service/src/main/resources/eu/dnetlib/enabling/views/inspector/gadget.st

@@ -0,0 +1,5 @@
+$inspector/master(it={
+gadget: 
+
+$items:{'$it$'}; separator=", "$
+})$

dnet-information-service/src/main/resources/eu/dnetlib/enabling/views/inspector/index.st

@@ -0,0 +1,15 @@
+$inspector/master(it={
+  <h2>Database inspector</h2>
+  $inspector/path()$
+  <ul id="subnav">
+		<li><a href="create">create xml file</a></li>
+		<li><a onclick="return confirm('do you really want to delete the collection and all its children?') && confirm('really really sure?') && (window.location='deleteCollection')" href="">delete this collection</a></li>
+		<li><a href="createsubcoll">create subcollection</a></li>
+	</ul>
+	
+
+  <ul id="filelist">
+    $collections:{<li>$it.collectionPath:{<a href="$it.url$/list">$it.name$</a>};separator=" / "$</li>}$
+    $files:{<li><a href="$it$/show">$it$</a></li>}$
+  </ul>
+})$