imported dnet-objectstore-rmi, dnet-modular-objectstore-service, dnet-fs-objectstore in dnet-data-services

dnet-data-services/pom.xml

@@ -28,6 +28,10 @@
 			<artifactId>commons-beanutils</artifactId>
 		</dependency>
 
+		<dependency>
+			<groupId>javax.servlet</groupId>
+			<artifactId>javax.servlet-api</artifactId>
+		</dependency>
 		<dependency>
 			<groupId>org.codehaus.groovy</groupId>
 			<artifactId>groovy</artifactId>

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/FileSystemObjectStore.java

@@ -0,0 +1,383 @@
+package eu.dnetlib.data.objectstore.filesystem;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.regex.Pattern;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.gson.Gson;
+import com.mongodb.BasicDBObject;
+import com.mongodb.DBObject;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.model.Filters;
+import com.mongodb.client.result.DeleteResult;
+import eu.dnetlib.data.objectstore.modular.ObjectStoreRecord;
+import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;
+import eu.dnetlib.data.objectstore.rmi.MetadataObjectRecord;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreFileNotFoundException;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.enabling.resultset.ResultSetListener;
+import eu.dnetlib.miscutils.collections.Pair;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.bson.conversions.Bson;
+
+/**
+ * The Class FileSystemObjectStore.
+ *
+ * @author sandro
+ */
+public class FileSystemObjectStore implements ObjectStore {
+
+	/**
+	 *
+	 */
+	private static final String URI_FIELD = "uri";
+
+	/**
+	 *
+	 */
+	private static final String FS_PATH_FIELD = "fsPath";
+
+	/** The Constant log. */
+	private static final Log log = LogFactory.getLog(FileSystemObjectStore.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	/** The id. */
+	private final String id;
+
+	/** The interpretation. */
+	private final String interpretation;
+
+	/** The base path. */
+	private final String basePath;
+
+	/** The base uri. */
+	private final String baseURI;
+
+	/** The mongo metadata. */
+	private final MongoCollection<DBObject> mongoMetadata;
+
+	/**
+	 * Instantiates a new file system object store.
+	 *
+	 * @param identifier
+	 *            the identifier
+	 * @param interpretation
+	 *            the interpretation
+	 * @param basePath
+	 *            the base path
+	 * @param mongoMetadata
+	 *            the mongo metadata
+	 * @param baseURI
+	 *            the base uri
+	 */
+	public FileSystemObjectStore(final String identifier, final String interpretation, final String basePath, final MongoCollection<DBObject> mongoMetadata,
+			final String baseURI) {
+		this.id = identifier;
+		this.basePath = basePath;
+		this.interpretation = interpretation;
+		this.mongoMetadata = mongoMetadata;
+		this.baseURI = baseURI;
+	}
+
+	/**
+	 * {@inheritDoc}
+	 *
+	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#getId()
+	 */
+	@Override
+	public String getId() {
+		return this.id;
+	}
+
+	/**
+	 * {@inheritDoc}
+	 *
+	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#getInterpretation()
+	 */
+	@Override
+	public String getInterpretation() {
+		return this.interpretation;
+	}
+
+	/**
+	 * {@inheritDoc}
+	 *
+	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#feed(java.lang.Iterable, boolean)
+	 */
+	@Override
+	public int feed(final Iterable<ObjectStoreRecord> records, final boolean incremental) throws ObjectStoreServiceException {
+		if (records == null)
+			return 0;
+
+		Path baseDirPath = FileSystems.getDefault().getPath(getBasePath()).resolve(getId());
+		if (!Files.exists(baseDirPath))
+			throw new ObjectStoreServiceException("Error can't feed objects because the folder " + baseDirPath + " does not exist");
+
+		int addedCounter = 0;
+		int nulls = 0;
+		for (ObjectStoreRecord record : records) {
+			String url = feedObject(record);
+			if (StringUtils.isNotBlank(url)) {
+				addedCounter++;
+			} else {
+				nulls++;
+			}
+		}
+		if (nulls > 0) {
+			log.warn(String.format("Found %s null records", nulls));
+		}
+		return addedCounter;
+	}
+
+	/**
+	 * {@inheritDoc}
+	 *
+	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#feedMetadataRecord(java.lang.Iterable, boolean)
+	 *
+	 * This method handles the case of web crawl files and other cases when the metadata in mdstores are also the objects to put into the objectstores.
+	 */
+	@Override
+	public int  feedMetadataRecord(final Iterable<MetadataObjectRecord> records, final boolean incremental) throws ObjectStoreServiceException {
+		Iterable<ObjectStoreRecord> it = Iterables.transform(records, mor -> {
+			ObjectStoreRecord r = new ObjectStoreRecord();
+			r.setInputStream(new ByteArrayInputStream(mor.getRecord().getBytes()));
+			ObjectStoreFile fileMetadata = new ObjectStoreFile();
+			fileMetadata.setObjectID(mor.getId());
+			fileMetadata.setMimeType(mor.getMime());
+			r.setFileMetadata(fileMetadata);
+			return r;
+		});
+		return feed(it, incremental);
+	}
+
+	/**
+	 * {@inheritDoc}
+	 *
+	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#feedObjectRecord(eu.dnetlib.data.objectstore.modular.ObjectStoreRecord)
+	 */
+	@Override
+	public String feedObjectRecord(final ObjectStoreRecord record) throws ObjectStoreServiceException {
+		return feedObject(record);
+	}
+
+	private String feedObject(final ObjectStoreRecord record) {
+		if (record != null) {
+			String objectIdentifier = record.getFileMetadata().getObjectID();
+			if (StringUtils.isNotBlank(objectIdentifier)) {
+				final Path objResolvedPath = FileSystemUtility.objectStoreFilePath(basePath, id, objectIdentifier);
+
+				if (Files.notExists(objResolvedPath)) {
+					try {
+						log.debug("Creation of folder " + objResolvedPath.getParent());
+						Files.createDirectories(objResolvedPath.getParent());
+						log.debug("Folder " + objResolvedPath.getParent() + " created");
+						String md5Sum = null;
+						Long size = new Long(0);
+						if (record.getInputStream() != null) {
+							Pair<String, Long> infos = FileSystemUtility.saveAndGenerateMD5(record.getInputStream(), objResolvedPath);
+							md5Sum = infos.getKey();
+							size = infos.getValue();
+						}
+						final String url =
+								ModularObjectStoreRESTService.retrieveURL(getBaseURI(), getBasePath(), getId(), record.getFileMetadata().getObjectID());
+						if (StringUtils.isNotBlank(md5Sum)) {
+							double timestamp = System.currentTimeMillis();
+							BasicDBObject metadata = new BasicDBObject();
+							metadata.put("id", record.getFileMetadata().getObjectID());
+							metadata.put("mime", record.getFileMetadata().getMimeType());
+							metadata.put("originalObject", record.getFileMetadata().toJSON());
+							metadata.put("timestamp", timestamp);
+							metadata.put("md5Sum", md5Sum);
+							metadata.put("size", size);
+							metadata.put(FS_PATH_FIELD, objResolvedPath.toAbsolutePath().toString());
+							metadata.put(URI_FIELD, url);
+							log.debug("saving metadata object to the collection: " + metadata.toString());
+							mongoMetadata.insertOne(metadata);
+						}
+						return url;
+					} catch (Exception e) {
+						log.error("Something bad happen on inserting Record", e);
+						log.error("Record: " + new Gson().toJson(record.getFileMetadata()));
+					} finally {
+						if (record.getInputStream() != null) {
+							try {
+								record.getInputStream().close();
+							} catch (Exception e) {
+								log.error("Error on close inputStream", e);
+							}
+						}
+					}
+				} else {
+					log.debug("The File in the path" + objResolvedPath.getParent() + "exists ");
+				}
+			}
+		}
+		return null;
+	}
+
+	/**
+	 * {@inheritDoc}
+	 *
+	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#deliver(java.lang.Long, java.lang.Long)
+	 */
+	@Override
+	public ResultSetListener deliver(final Long from, final Long until) throws ObjectStoreServiceException {
+		FileSystemObjectStoreResultSetListener resultSet = new FileSystemObjectStoreResultSetListener();
+		resultSet.setBaseURI(getBaseURI());
+		resultSet.setMongoCollection(mongoMetadata);
+		resultSet.setObjectStoreID(getId());
+		resultSet.setFromDate(from);
+		resultSet.setUntilDate(until);
+		resultSet.setBasePath(getBasePath());
+		return resultSet;
+	}
+
+	/**
+	 * {@inheritDoc}
+	 *
+	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#deliverIds(java.lang.Iterable)
+	 */
+	@Override
+	public ResultSetListener deliverIds(final Iterable<String> ids) throws ObjectStoreServiceException {
+		FileSystemObjectStoreResultSetListener resultSet = new FileSystemObjectStoreResultSetListener();
+		resultSet.setBaseURI(getBaseURI());
+		resultSet.setMongoCollection(mongoMetadata);
+		resultSet.setObjectStoreID(getId());
+		resultSet.setRecords(Lists.newArrayList(ids));
+		resultSet.setBasePath(basePath);
+		return resultSet;
+	}
+
+	/**
+	 * {@inheritDoc}
+	 *
+	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#deliverObject(java.lang.String)
+	 */
+	@Override
+	public ObjectStoreFile deliverObject(final String objectId) throws ObjectStoreServiceException {
+		Bson query = Filters.eq("id", objectId);
+		DBObject resultQuery = mongoMetadata.find(query).first();
+		checkAndGetFsPathField(resultQuery, objectId);
+		return ObjectStoreFileUtility.build(resultQuery, getBaseURI(), getId(), basePath);
+	}
+
+	private String checkAndGetFsPathField(final DBObject resultQuery, final String objectId) throws ObjectStoreServiceException {
+		if (resultQuery == null || !resultQuery.containsField(FS_PATH_FIELD))
+			throw new ObjectStoreFileNotFoundException("Object with identifier :" + objectId + " not found or missing " + FS_PATH_FIELD + " field");
+		String pathStr = (String) resultQuery.get(FS_PATH_FIELD);
+		if (StringUtils.isBlank(pathStr))
+			throw new ObjectStoreFileNotFoundException("Object with identifier :" + objectId + " with blank " + FS_PATH_FIELD);
+		return pathStr;
+	}
+
+	/**
+	 * {@inheritDoc}
+	 *
+	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#getSize()
+	 */
+	@Override
+	public int getSize() throws ObjectStoreServiceException {
+		return (int) mongoMetadata.count();
+	}
+
+	/**
+	 * {@inheritDoc}
+	 *
+	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#deleteObject(java.lang.String)
+	 */
+	@Override
+	public void deleteObject(final String objectId) throws ObjectStoreServiceException {
+		Bson query = Filters.eq("id", objectId);
+		DBObject response = mongoMetadata.find(query).first();
+		String pathStr = checkAndGetFsPathField(response, objectId);
+		Path path = FileSystems.getDefault().getPath(pathStr);
+		if (Files.notExists(path))
+			throw new ObjectStoreFileNotFoundException("Object with identifier :" + objectId + " not found in the assigned path " + path);
+		try {
+			Files.delete(path);
+		} catch (IOException e) {
+			throw new ObjectStoreServiceException("An error occurs on delete file ", e);
+		}
+		mongoMetadata.deleteOne(query);
+	}
+
+	/**
+	 * {@inheritDoc}
+	 *
+	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#getObject(java.lang.String)
+	 */
+	@Override
+	public String getObject(final String recordId) throws ObjectStoreServiceException {
+		Bson query = Filters.eq("id", recordId);
+		DBObject response = mongoMetadata.find(query).first();
+		if (response == null || !response.containsField(URI_FIELD))
+			return null;
+		return (String) response.get(URI_FIELD);
+	}
+
+	/**
+	 * {@inheritDoc}
+	 *
+	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#existIDStartsWith(java.lang.String)
+	 */
+	@Override
+	public boolean existIDStartsWith(final String startId) throws ObjectStoreServiceException {
+		Bson query = Filters.regex("id", Pattern.compile(startId));
+		return mongoMetadata.count(query) > 0;
+	}
+
+	@Override
+	public boolean dropContent() throws ObjectStoreServiceException {
+		if (getBasePath() == null) {
+			throw new ObjectStoreServiceException("Error on dropping object store base_path required");
+		}
+		final Path baseDirPath = FileSystems.getDefault().getPath(getBasePath()).resolve(getId());
+		try {
+			FileSystemUtility.deleteFolderRecursive(baseDirPath);
+		} catch (IOException e) {
+			throw new ObjectStoreServiceException("Error on dropping store ", e);
+		}
+		log.info("Deleted folder" + baseDirPath.toString());
+		if (!Files.exists(baseDirPath)) {
+			log.info("Recreating folder " + baseDirPath);
+			try {
+				Files.createDirectory(baseDirPath);
+			} catch (IOException e) {
+				throw new ObjectStoreServiceException("Error on dropping store ", e);
+			}
+		}
+		final DeleteResult deleteResult = this.mongoMetadata.deleteMany(new BasicDBObject());
+		log.info("Dropped content for object store " + id + ". " + deleteResult.getDeletedCount() + " object(s) deleted.");
+		return true;
+	}
+
+	@Override
+	public String toString() {
+		return "FileSystemObjectStore{" +
+				"id='" + getId() + '\'' +
+				", interpretation='" + getInterpretation() + '\'' +
+				", basePath='" + getBasePath() + '\'' +
+				", baseURI='" + getBaseURI() + '\'' +
+				'}';
+	}
+
+	/**
+	 * Gets the base uri.
+	 *
+	 * @return the baseURI
+	 */
+	public String getBaseURI() {
+		return baseURI;
+	}
+
+	public String getBasePath() {
+		return basePath;
+	}
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/FileSystemObjectStoreDao.java

@@ -0,0 +1,212 @@
+package eu.dnetlib.data.objectstore.filesystem;
+
+import java.io.IOException;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import javax.annotation.Resource;
+
+import com.mongodb.BasicDBObject;
+import com.mongodb.DBObject;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.model.Filters;
+import com.mongodb.client.result.UpdateResult;
+import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;
+import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreFileNotFoundException;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.miscutils.collections.MappedCollection;
+import eu.dnetlib.miscutils.functional.UnaryFunction;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.bson.conversions.Bson;
+
+/**
+ * @author sandro
+ *
+ */
+public class FileSystemObjectStoreDao implements ObjectStoreDao {
+
+	public static final String INTERPRETATION_FIELD = "interpretation";
+	public final static String OBJECTSTORE_METADATA_NAME_FIELD = "metadataObjectStore";
+	public final static String OBJECTSTORE_ID_FIELD = "obsId";
+	public final static String BASE_PATH_FIELD = "basePath";
+	private static final Log log = LogFactory.getLog(FileSystemObjectStoreDao.class); // NOPMD by marko on 11/24/08 5:02 PM
+	private static final String OBJECTSTORE_PROFILE_SUFFIX = "_T2JqZWN0U3RvcmVEU1Jlc291cmNlcy9PYmplY3RTdG9yZURTUmVzb3VyY2VUeXBl";
+	@Resource(name="objectstoreMongoDB")
+	private MongoDatabase db;
+
+	private boolean upsert;
+
+	private String objectStoreRESTURI;
+
+	/**
+	 * {@inheritDoc}
+	 * @throws ObjectStoreServiceException
+	 *
+	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao#getObjectStore(java.lang.String)
+	 */
+	@Override
+	public ObjectStore getObjectStore(final String obsId) throws ObjectStoreServiceException {
+		String currentId = obsId.substring(0, 36);
+		String find_id = obsId;
+		if (find_id.length() == 36) {
+			find_id += OBJECTSTORE_PROFILE_SUFFIX;
+		}
+
+		MongoCollection<DBObject> metadataObjectStore = getDb().getCollection(OBJECTSTORE_METADATA_NAME_FIELD, DBObject.class);
+		Bson query = Filters.eq(OBJECTSTORE_ID_FIELD, find_id);
+		log.debug("QUERY :" + query.toString());
+		DBObject resultQuery = metadataObjectStore.find(query).first();
+		log.debug("result " + resultQuery);
+		if ((resultQuery == null)) throw new ObjectStoreFileNotFoundException("the objectStore with identifier: "+obsId+" was not found");
+
+		final String basePath = resultQuery.get(BASE_PATH_FIELD).toString();
+		final String interpretation = resultQuery.get("interpretation").toString();
+
+		if (!resultQuery.containsField(BASE_PATH_FIELD) || StringUtils.isBlank(basePath))
+			throw new ObjectStoreServiceException("Can't Get Objectstore, the metadata doesn't contain info about the basepath");
+
+		final MongoCollection<DBObject> collection = getDb().getCollection(currentId, DBObject.class);
+		return new FileSystemObjectStore(currentId, interpretation, basePath, collection, objectStoreRESTURI);
+	}
+
+	/**
+	 * {@inheritDoc}
+	 *
+	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao#listObjectStores()
+	 */
+	@Override
+	public List<String> listObjectStores() {
+		MongoCollection<DBObject> metadata = getDb().getCollection(OBJECTSTORE_METADATA_NAME_FIELD, DBObject.class);
+		return MappedCollection.listMap(metadata.find(), new UnaryFunction<String, DBObject>() {
+
+			@Override
+			public String evaluate(final DBObject object) {
+				return (String) object.get(OBJECTSTORE_ID_FIELD);
+			}
+		});
+	}
+
+	/**
+	 * {@inheritDoc}
+	 *
+	 * @throws ObjectStoreServiceException
+	 */
+	@Override
+	public boolean createObjectStore(final String obsId, final String interpretation, final String basePath) throws ObjectStoreServiceException {
+
+		log.debug(String.format("Create object Store method\n\t Id:%s  Interpretation:%s BasePath : %s", obsId, interpretation, basePath) );
+
+		if (StringUtils.isBlank(basePath)) throw new ObjectStoreServiceException("Can't create the object store: the base path cannot be blank");
+		Path path = FileSystems.getDefault().getPath(basePath);
+		if (!Files.exists(path) || !Files.isDirectory(path))
+			throw new ObjectStoreServiceException("Can't create the object store: base path: '" + basePath + "' doesn't exist or it is not a folder");
+		try {
+			String currentObsId = obsId.substring(0, 36);
+			log.debug("Cleaned objectStore Id " + currentObsId);
+			if (Files.exists(path.resolve(currentObsId)))
+				throw new ObjectStoreServiceException("Can't create the object store: base path: '" + path.resolve(currentObsId) + "' already exists");
+			Files.createDirectory(path.resolve(currentObsId));
+			MongoCollection<DBObject> coll = getDb().getCollection(OBJECTSTORE_METADATA_NAME_FIELD, DBObject.class);
+			final BasicDBObject obj = new BasicDBObject();
+			obj.put(OBJECTSTORE_ID_FIELD, obsId);
+			obj.put(INTERPRETATION_FIELD, interpretation);
+			obj.put(BASE_PATH_FIELD, basePath);
+			coll.insertOne(obj);
+			MongoCollection<DBObject> objectStore = getDb().getCollection(currentObsId, DBObject.class);
+			objectStore.createIndex(new BasicDBObject("id", 1));
+			objectStore.createIndex(new BasicDBObject("timestamp", 1));
+			return true;
+		} catch (Throwable e) {
+			throw new ObjectStoreServiceException("Can't Create the object Store id: '" + obsId, e);
+		}
+	}
+
+	/**
+	 * {@inheritDoc}
+	 *
+	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao#updateObjectStore(java.lang.String, java.lang.String)
+	 */
+	@Override
+	public boolean updateObjectStore(final String obsId, final String interpretation) {
+		MongoCollection<DBObject> coll = getDb().getCollection(OBJECTSTORE_METADATA_NAME_FIELD, DBObject.class);
+		final BasicDBObject obj = new BasicDBObject();
+		obj.put("$set", new BasicDBObject(INTERPRETATION_FIELD, interpretation));
+
+		final UpdateResult updateResult = coll.updateOne(Filters.eq(OBJECTSTORE_ID_FIELD, obsId), obj);
+		if (updateResult.isModifiedCountAvailable()) {
+			log.debug("Matched / Modified " + updateResult.getMatchedCount() + " / " + updateResult.getModifiedCount());
+		}
+		return true;
+	}
+
+	/**
+	 * {@inheritDoc}
+	 *
+	 * @throws ObjectStoreServiceException
+	 * @see eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao#deleteObjectStore(java.lang.String)
+	 */
+	@Override
+	public boolean deleteObjectStore(final String obsId) throws ObjectStoreServiceException {
+		MongoCollection<DBObject> coll = getDb().getCollection(OBJECTSTORE_METADATA_NAME_FIELD, DBObject.class);
+		Bson query = Filters.eq(OBJECTSTORE_ID_FIELD, obsId);
+		DBObject resultQuery = coll.find(query).first();
+		String basePath = checkAndGetFsPathField(resultQuery, obsId);
+		String currentObsId = obsId.substring(0, 36);
+		Path basePathFS = FileSystems.getDefault().getPath(basePath, currentObsId);
+		if (!Files.exists(basePathFS))
+			throw new ObjectStoreServiceException("Can't Delete ObjectStore " + obsId + ": the base path does not exist :" + basePathFS);
+		try {
+			FileSystemUtility.deleteFolderRecursive(basePathFS);
+		} catch (IOException e) {
+			throw new ObjectStoreServiceException("Can't Delete ObjectStore " + obsId, e);
+		}
+		coll.deleteOne(Filters.eq(OBJECTSTORE_ID_FIELD, obsId));
+		getDb().getCollection(obsId).drop();
+		return true;
+	}
+
+	@Override
+	public boolean dropContent(final String obsId) throws ObjectStoreServiceException {
+		return getObjectStore(obsId).dropContent();
+	}
+
+	private String checkAndGetFsPathField(final DBObject resultQuery, final String objectStoreID) throws ObjectStoreServiceException {
+		if (resultQuery == null || !resultQuery.containsField(BASE_PATH_FIELD))
+			throw new ObjectStoreServiceException("ObjectStore with identifier :" + objectStoreID + " not found or missing " + BASE_PATH_FIELD + " field");
+		String pathStr = (String) resultQuery.get(BASE_PATH_FIELD);
+		if (StringUtils.isBlank(pathStr))
+			throw new ObjectStoreServiceException("ObjectStore with identifier :" + objectStoreID + " with blank " + BASE_PATH_FIELD);
+		return pathStr;
+	}
+
+
+	public boolean isUpsert() {
+		return upsert;
+	}
+
+	public void setUpsert(final boolean upsert) {
+		this.upsert = upsert;
+	}
+
+	public String getObjectStoreRESTURI() {
+		return objectStoreRESTURI;
+	}
+
+	public void setObjectStoreRESTURI(final String objectStoreRESTURI) {
+		this.objectStoreRESTURI = objectStoreRESTURI;
+	}
+
+	public MongoDatabase getDb() {
+		return db;
+	}
+
+	public void setDb(final MongoDatabase db) {
+		this.db = db;
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/FileSystemObjectStoreResultSetListener.java

@@ -0,0 +1,350 @@
+package eu.dnetlib.data.objectstore.filesystem;
+
+import java.util.List;
+
+import com.google.common.collect.Lists;
+import com.mongodb.DBObject;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoCursor;
+import com.mongodb.client.model.Filters;
+import com.mongodb.client.model.Sorts;
+import eu.dnetlib.enabling.resultset.ResultSet;
+import eu.dnetlib.enabling.resultset.ResultSetAware;
+import eu.dnetlib.enabling.resultset.ResultSetListener;
+import eu.dnetlib.miscutils.collections.MappedCollection;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.bson.conversions.Bson;
+
+/**
+ * The listener interface for receiving fileSystemObjectStoreResultSet events.
+ * The class that is interested in processing a fileSystemObjectStoreResultSet
+ * event implements this interface, and the object created
+ * with that class is registered with a component using the
+ * component's <code>addFileSystemObjectStoreResultSetListener<code> method. When
+ * the fileSystemObjectStoreResultSet event occurs, that object's appropriate
+ * method is invoked.
+ *
+ * @author sandro
+ */
+public class FileSystemObjectStoreResultSetListener implements ResultSetListener, ResultSetAware {
+
+
+	/** The Constant log. */
+	private static final Log log = LogFactory.getLog(FileSystemObjectStoreResultSetListener.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+
+	/** The from date. */
+	private Long fromDate;
+
+	/** The until date. */
+	private Long untilDate;
+
+	/** The records. */
+	private List<String> records;
+
+	/** The object store id. */
+	private String objectStoreID;
+
+
+	/** The mongo collection. */
+	private MongoCollection<DBObject> mongoCollection;
+
+	/** The base uri. */
+	private String baseURI;
+
+	/**
+	 * The base path
+	 */
+	private String basePath;
+
+	/** The current size. */
+	private int currentSize = -1;
+
+	/** The current cursor. */
+	private MongoCursor<DBObject> currentCursor;
+
+	/** The cursor position. */
+	private long cursorPosition;
+
+	/**
+	 * {@inheritDoc}
+	 * @see eu.dnetlib.enabling.resultset.TypedResultSetListener#getResult(int, int)
+	 */
+	@Override
+	public List<String> getResult(final int from, final int to) {
+		if (log.isDebugEnabled()) {
+			log.debug(String.format("ObjectStoreId :%s, from: %d, to: %d", objectStoreID, from, to));
+		}
+		if (records != null) {
+			List<String> ids = Lists.newArrayList();
+			for (int i = from-1; i < Math.min(records.size(),to); i++) {
+				ids.add(records.get(i));
+			}
+			Bson q = Filters.in("id", ids);
+			FindIterable<DBObject> res = getMongoCollection().find(q);
+			return MappedCollection.listMap(res, ObjectStoreFileUtility.asJSON(getBaseURI(), getObjectStoreID(), getBasePath()));
+		} else if ((fromDate != null) && (untilDate != null)) {
+			if ((currentCursor == null) || (cursorPosition > from)) {
+				createCurrentCursor();
+			}
+			while (cursorPosition < from) {
+				currentCursor.next();
+				cursorPosition++;
+			}
+			List<DBObject> result = Lists.newArrayList();
+			for (int i = from; i <= to; i++) {
+				if (currentCursor.hasNext()) {
+					result.add(currentCursor.next());
+					cursorPosition++;
+				}
+			}
+			return MappedCollection.listMap(result, ObjectStoreFileUtility.asJSON(getBaseURI(), getObjectStoreID(), getBasePath()));
+		}
+
+		throw new IllegalArgumentException("Missing parameters on Delivery must provide either from, to, or ObjectStoreIDs");
+	}
+
+	/**
+	 * Creates the current cursor.
+	 */
+	private void createCurrentCursor() {
+		Bson timestampQuery = Filters.and(Filters.gt("timestamp", fromDate.doubleValue()), Filters.lt("timestamp", untilDate.doubleValue()));
+		if (currentCursor != null) {
+			currentCursor.close();
+		}
+		currentCursor = getMongoCollection().find(timestampQuery).sort(Sorts.orderBy(Filters.eq("_id", 1))).iterator();
+		cursorPosition = 1;
+
+	}
+
+	/**
+	 * {@inheritDoc}
+	 * @see eu.dnetlib.enabling.resultset.TypedResultSetListener#getSize()
+	 */
+	@Override
+	public int getSize() {
+		if (currentSize == -1) {
+			currentSize = calculateSize();
+		}
+		return Math.max(0, currentSize - 1);
+	}
+
+	/**
+	 * Calculate size.
+	 *
+	 * @return the int
+	 */
+	private int calculateSize() {
+		if (records != null) {
+			Bson query = Filters.in("id", records);
+			return (int) getMongoCollection().count(query);
+		} else if ((fromDate != null) && (untilDate != null)) {
+			Bson timestampQuery = Filters.and(Filters.gt("timestamp", fromDate.doubleValue()), Filters.lt("timestamp", untilDate.doubleValue()));
+			return (int) getMongoCollection().count(timestampQuery);
+		}
+		return 0;
+	}
+
+
+	/**
+	 * {@inheritDoc}
+	 * @see eu.dnetlib.enabling.resultset.ResultSetAware#setResultSet(eu.dnetlib.enabling.resultset.ResultSet)
+	 */
+	@Override
+	public void setResultSet(final ResultSet resultSet) {
+		resultSet.close();
+	}
+
+
+	/**
+	 * Gets the from date.
+	 *
+	 * @return the fromDate
+	 */
+	public Long getFromDate() {
+		return fromDate;
+	}
+
+
+	/**
+	 * Sets the from date.
+	 *
+	 * @param fromDate the fromDate to set
+	 */
+	public FileSystemObjectStoreResultSetListener setFromDate(final Long fromDate) {
+		this.fromDate = fromDate;
+		return this;
+	}
+
+
+	/**
+	 * Gets the until date.
+	 *
+	 * @return the untilDate
+	 */
+	public Long getUntilDate() {
+		return untilDate;
+	}
+
+
+	/**
+	 * Sets the until date.
+	 *
+	 * @param untilDate the untilDate to set
+	 */
+	public FileSystemObjectStoreResultSetListener setUntilDate(final Long untilDate) {
+		this.untilDate = untilDate;
+		return this;
+	}
+
+
+	/**
+	 * Gets the records.
+	 *
+	 * @return the records
+	 */
+	public List<String> getRecords() {
+		return records;
+	}
+
+
+	/**
+	 * Sets the records.
+	 *
+	 * @param records the records to set
+	 */
+	public void setRecords(final List<String> records) {
+		this.records = records;
+	}
+
+
+	/**
+	 * Gets the object store id.
+	 *
+	 * @return the objectStoreID
+	 */
+	public String getObjectStoreID() {
+		return objectStoreID;
+	}
+
+
+	/**
+	 * Sets the object store id.
+	 *
+	 * @param objectStoreID the objectStoreID to set
+	 */
+	public void setObjectStoreID(final String objectStoreID) {
+		this.objectStoreID = objectStoreID;
+	}
+
+
+
+
+
+	/**
+	 * Gets the base uri.
+	 *
+	 * @return the baseURI
+	 */
+	public String getBaseURI() {
+		return baseURI;
+	}
+
+
+	/**
+	 * Sets the base uri.
+	 *
+	 * @param baseURI the baseURI to set
+	 */
+	public void setBaseURI(final String baseURI) {
+		this.baseURI = baseURI;
+	}
+
+
+	/**
+	 * Gets the current size.
+	 *
+	 * @return the currentSize
+	 */
+	public int getCurrentSize() {
+		return currentSize;
+	}
+
+
+	/**
+	 * Sets the current size.
+	 *
+	 * @param currentSize the currentSize to set
+	 */
+	public void setCurrentSize(final int currentSize) {
+		this.currentSize = currentSize;
+	}
+
+
+	/**
+	 * Gets the current cursor.
+	 *
+	 * @return the currentCursor
+	 */
+	public MongoCursor<DBObject> getCurrentCursor() {
+		return currentCursor;
+	}
+
+
+	/**
+	 * Sets the current cursor.
+	 *
+	 * @param currentCursor the currentCursor to set
+	 */
+	public void setCurrentCursor(final MongoCursor<DBObject> currentCursor) {
+		this.currentCursor = currentCursor;
+	}
+
+
+	/**
+	 * Gets the cursor position.
+	 *
+	 * @return the cursorPosition
+	 */
+	public long getCursorPosition() {
+		return cursorPosition;
+	}
+
+
+	/**
+	 * Sets the cursor position.
+	 *
+	 * @param cursorPosition the cursorPosition to set
+	 */
+	public void setCursorPosition(final long cursorPosition) {
+		this.cursorPosition = cursorPosition;
+	}
+
+	/**
+	 * Gets the mongo collection.
+	 *
+	 * @return the mongo collection
+	 */
+	public MongoCollection<DBObject> getMongoCollection() {
+		return mongoCollection;
+	}
+
+	/**
+	 * Sets the mongo collection.
+	 *
+	 * @param mongoCollection the new mongo collection
+	 */
+	public void setMongoCollection(final MongoCollection<DBObject> mongoCollection) {
+		this.mongoCollection = mongoCollection;
+	}
+
+	public String getBasePath() {
+		return basePath;
+	}
+
+	public void setBasePath(final String basePath) {
+		this.basePath = basePath;
+	}
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/FileSystemUtility.java

@@ -0,0 +1,78 @@
+package eu.dnetlib.data.objectstore.filesystem;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.*;
+import java.nio.file.attribute.BasicFileAttributes;
+
+import eu.dnetlib.miscutils.collections.Pair;
+import eu.dnetlib.miscutils.functional.xml.DnetXsltFunctions;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * The Class FileSystemUtility.
+ *
+ * @author sandro
+ */
+public class FileSystemUtility {
+
+	private static final Log log = LogFactory.getLog(FileSystemUtility.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	public static Pair<String, Long> saveAndGenerateMD5(final InputStream inputStream, final Path filePath) {
+		if(inputStream==null) return null;
+
+		String md5 = null;
+		long size = 0;
+		try {
+			Files.copy(inputStream, filePath);
+
+			FileInputStream fis = new FileInputStream(filePath.toFile());
+			md5 = org.apache.commons.codec.digest.DigestUtils.md5Hex(fis);
+			fis.close();
+			size = Files.size(filePath);
+
+		} catch (IOException e1) {
+			log.error(e1);
+		}
+
+		return new Pair<String, Long>(md5, size);
+	}
+
+	/**
+	 * Delete folder recursive.
+	 *
+	 * @param path the path
+	 * @return true, if successful
+	 * @throws IOException Signals that an I/O exception has occurred.
+	 */
+	public static boolean deleteFolderRecursive(final Path path) throws IOException {
+
+		Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
+			@Override
+			public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
+				Files.delete(file);
+				return FileVisitResult.CONTINUE;
+			}
+
+			@Override
+			public FileVisitResult postVisitDirectory(final Path dir, final IOException exc) throws IOException {
+				Files.delete(dir);
+				return FileVisitResult.CONTINUE;
+			}
+		});
+		return true;
+	}
+
+	public static Path objectStoreFilePath(final String basePath, final String objectStoreId, final String objectIdentifier) {
+		final Path baseDirPath = FileSystems.getDefault().getPath(basePath).resolve(objectStoreId);
+		final String md5id = DnetXsltFunctions.md5(objectIdentifier);
+		final String firstLevel = StringUtils.substring(md5id, 0, 2);
+		final String secondLevel = StringUtils.substring(md5id, 2, 4);
+		final String fileName = StringUtils.substring(md5id, 4) + ".obj";
+		return baseDirPath.resolve(firstLevel).resolve(secondLevel).resolve(fileName);
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/IndexIntegrityChecker.java

@@ -0,0 +1,55 @@
+package eu.dnetlib.data.objectstore.filesystem;
+
+import com.mongodb.BasicDBObject;
+import com.mongodb.DBObject;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.model.IndexOptions;
+import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import static eu.dnetlib.data.objectstore.filesystem.FileSystemObjectStoreDao.OBJECTSTORE_ID_FIELD;
+import static eu.dnetlib.data.objectstore.filesystem.FileSystemObjectStoreDao.OBJECTSTORE_METADATA_NAME_FIELD;
+
+/**
+ * Created by claudio on 15/09/16.
+ */
+public class IndexIntegrityChecker {
+
+	private static final Log log = LogFactory.getLog(IndexIntegrityChecker.class);
+
+	@Autowired
+	private ObjectStoreDao dao;
+
+	public void check() {
+		checkIndexes();
+	}
+
+	private void checkIndexes() {
+		log.info("objectStore indexes integrity start");
+
+		final MongoDatabase db = ((FileSystemObjectStoreDao) dao).getDb();
+		final IndexOptions bg = new IndexOptions().background(true);
+
+		for (String objectStoreId : dao.listObjectStores()) {
+			final String id = StringUtils.substringBefore(objectStoreId, "_");
+			final MongoCollection<DBObject> objectStore = db.getCollection(id, DBObject.class);
+			if (log.isDebugEnabled()) {
+				log.debug(String.format("creating index (id, timestamp) on objectStore %s", id));
+			}
+			objectStore.createIndex(new BasicDBObject("id", 1), bg);
+			objectStore.createIndex(new BasicDBObject("timestamp", 1), bg);
+		}
+		if (log.isDebugEnabled()) {
+			log.debug(String.format("creating index (%s) on %s", OBJECTSTORE_ID_FIELD, OBJECTSTORE_METADATA_NAME_FIELD));
+		}
+		final MongoCollection<DBObject> metadata = db.getCollection(OBJECTSTORE_METADATA_NAME_FIELD, DBObject.class);
+		metadata.createIndex(new BasicDBObject(OBJECTSTORE_ID_FIELD, 1), bg);
+
+		log.info("objectStore indexes integrity completed");
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/ModularObjectStoreRESTService.java

@@ -0,0 +1,79 @@
+package eu.dnetlib.data.objectstore.filesystem;
+
+import java.io.*;
+import java.net.URLEncoder;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import javax.servlet.http.HttpServletResponse;
+
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.miscutils.datetime.HumanTime;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+
+/**
+ * The Class ModularObjectStoreRESTService implement the controller REST of the object Store.
+ */
+@Controller
+public class ModularObjectStoreRESTService {
+
+	private static final Log log = LogFactory.getLog(ModularObjectStoreRESTService.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	public static String retrieveURL(final String baseURI, final String basePath, final String objectStoreId, final String objectId)
+			throws UnsupportedEncodingException {
+		final StringBuilder sb = new StringBuilder(baseURI)
+				.append("?objectStore=" + encode(objectStoreId))
+				.append("&objectId=" + encode(objectId))
+				.append("&basePath=" + encode(basePath));
+		return sb.toString();
+	}
+
+	private static String encode(final String s) throws UnsupportedEncodingException {
+		return URLEncoder.encode(s, "UTF-8");
+	}
+
+	/**
+	 *
+	 * @param res
+	 * @param basePath
+	 * @param objectStoreId
+	 * @param objectId
+	 * @throws IOException
+	 * @throws ObjectStoreServiceException
+	 */
+	@RequestMapping(value = "/**/objectStore/retrieve.do")
+	public void retrieve(final HttpServletResponse res,
+			@RequestParam(value = "basePath", required = true) final String basePath,
+			@RequestParam(value = "objectStore", required = true) final String objectStoreId,
+			@RequestParam(value = "objectId", required = true) final String objectId) throws IOException, ObjectStoreServiceException {
+
+		final long start = System.currentTimeMillis();
+		final Path path = FileSystemUtility.objectStoreFilePath(basePath, objectStoreId, objectId);
+
+		if (!Files.exists(path) || !Files.isReadable(path)) {
+			final String msg = String.format("Object with identifier: %s not found the in %s", objectId, path);
+			res.sendError(HttpServletResponse.SC_NOT_FOUND, msg);
+			log.warn(msg);
+		} else {
+			try (final InputStream is = new BufferedInputStream(new FileInputStream(path.toFile()))) {
+
+				final long size = Files.size(path);
+
+				res.setHeader("Content-Length", String.valueOf(size));
+				IOUtils.copyLarge(is, res.getOutputStream());
+
+				if (log.isDebugEnabled()) {
+					log.debug(String.format("retrive.do completed in %s, objId: %s", HumanTime.exactly(System.currentTimeMillis() - start), objectId));
+				}
+			} catch (IOException e) {
+				final String msg = "unable to close input Stream";
+				res.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, msg);
+				log.error(msg, e);
+			}
+		}
+	}
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/MongoFSOptionsFactory.java

@@ -0,0 +1,34 @@
+package eu.dnetlib.data.objectstore.filesystem;
+
+import com.mongodb.MongoClientOptions;
+import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.FactoryBean;
+
+public class MongoFSOptionsFactory implements FactoryBean<MongoClientOptions> {
+
+	private int connectionsPerHost;
+
+	@Override
+	public MongoClientOptions getObject() throws BeansException {
+		return MongoClientOptions.builder().connectionsPerHost(connectionsPerHost).build();
+	}
+
+	@Override
+	public Class<MongoClientOptions> getObjectType() {
+		return MongoClientOptions.class;
+	}
+
+	@Override
+	public boolean isSingleton() {
+		return false;
+	}
+
+	public int getConnectionsPerHost() {
+		return connectionsPerHost;
+	}
+
+	public void setConnectionsPerHost(final int connectionsPerHost) {
+		this.connectionsPerHost = connectionsPerHost;
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/ObjectStoreFileUtility.java

@@ -0,0 +1,57 @@
+package eu.dnetlib.data.objectstore.filesystem;
+
+import java.io.UnsupportedEncodingException;
+
+import com.mongodb.DBObject;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
+import eu.dnetlib.data.objectstore.rmi.Protocols;
+import eu.dnetlib.miscutils.functional.UnaryFunction;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * The Class ObjectStoreFileBuilder generates an objectStoreFile bean
+ *
+ */
+public class ObjectStoreFileUtility {
+
+	private static final int KB_SIZE = 1024;
+
+	/** The Constant log. */
+	private static final Log log = LogFactory.getLog(ObjectStoreFileUtility.class);
+
+	public static ObjectStoreFile build(final DBObject metadata, final String baseURI, final String objectStoreID, final String basePath) {
+
+		String originalFile = (String) metadata.get("originalObject");
+		ObjectStoreFile original = ObjectStoreFile.createObject(originalFile);
+		ObjectStoreFile newFile = new ObjectStoreFile();
+		newFile.setObjectID((String) metadata.get("id"));
+		newFile.setAccessProtocol(Protocols.HTTP);
+		newFile.setMimeType((String) metadata.get("mime"));
+		newFile.setMd5Sum((String) metadata.get("md5Sum"));
+		try {
+			newFile.setFileSizeKB(Long.parseLong(metadata.get("size").toString()) / KB_SIZE);
+		} catch (Throwable e) {
+			log.error("Error on getting file size", e);
+		}
+		if (originalFile != null) {
+			newFile.setMetadataRelatedID(original.getMetadataRelatedID());
+			if (StringUtils.isBlank(original.getDownloadedURL())) {
+				newFile.setDownloadedURL(original.getURI());
+			} else {
+				newFile.setDownloadedURL(original.getDownloadedURL());
+			}
+		}
+		try {
+			newFile.setURI(ModularObjectStoreRESTService.retrieveURL(baseURI, basePath, objectStoreID, newFile.getObjectID()));
+		} catch (UnsupportedEncodingException e) {
+			log.error("Error on Build objectStoreFile ", e);
+		}
+		return newFile;
+	}
+
+	public static UnaryFunction<String, DBObject> asJSON(final String baseURI, final String objectStoreID, final String basePath) {
+		return input -> build(input, baseURI, objectStoreID, basePath).toJSON();
+	}
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/AbstractObjectStoreAction.java

@@ -0,0 +1,77 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+
+import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardServerAction;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * The Class AbstractObjectStoreAction.
+ */
+public abstract class AbstractObjectStoreAction implements BlackboardServerAction<ObjectStoreActions> {
+
+	/**
+	 * Logger
+	 */
+	private static final Log log = LogFactory.getLog(AbstractObjectStoreAction.class);
+	private final Executor executor = Executors.newCachedThreadPool();
+	/** The object store dao. */
+	private ObjectStoreDao dao;
+
+	protected abstract void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) throws ObjectStoreServiceException;
+
+	@Override
+	public void execute(final BlackboardServerHandler handler, final BlackboardJob job) {
+		executor.execute(new Runnable() {
+
+			@Override
+			public void run() {
+				try {
+					handler.ongoing(job);
+					executeAsync(handler, job);
+				} catch (ObjectStoreServiceException e) {
+					handler.failed(job, e);
+				}
+			}
+		});
+	}
+
+	protected void completeWithSuccess(final BlackboardServerHandler handler, final BlackboardJob job) {
+		// Don't change this synchronization rule
+		synchronized (this) {
+			handler.done(job);
+		}
+	}
+
+	protected void completeWithFail(final BlackboardServerHandler handler, final BlackboardJob job, final Throwable e) {
+		// Don't change this synchronization rule
+		synchronized (this) {
+			handler.failed(job, e);
+		}
+	}
+
+	/**
+	 * Gets the dao.
+	 *
+	 * @return the dao
+	 */
+	public ObjectStoreDao getDao() {
+		return dao;
+	}
+
+	/**
+	 * Sets the dao.
+	 *
+	 * @param dao the new dao
+	 */
+	public void setDao(ObjectStoreDao dao) {
+		this.dao = dao;
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/CreateObjectStoreAction.java

@@ -0,0 +1,62 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * The Class CreateObjectStoreAction is responsible to execute the blacboard action of type CREATE.
+ */
+public class CreateObjectStoreAction extends AbstractObjectStoreAction {
+
+	private static final Log log = LogFactory.getLog(CreateObjectStoreAction.class);
+
+	/** The profile creator. */
+	private ObjectStoreProfileCreator profileCreator;
+
+	/**
+	 * Gets the profile creator.
+	 *
+	 * @return the profile creator
+	 */
+	public ObjectStoreProfileCreator getProfileCreator() {
+		return profileCreator;
+	}
+
+	/**
+	 * Sets the profile creator.
+	 *
+	 * @param profileCreator the new profile creator
+	 */
+	public void setProfileCreator(final ObjectStoreProfileCreator profileCreator) {
+		this.profileCreator = profileCreator;
+	}
+
+	@Override
+	protected void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) {
+		try {
+			final String interpretation = job.getParameters().get("interpretation");
+//			final String basePath = job.getParameters().get("basePath");
+//			if (StringUtils.isBlank(basePath)) {
+//				throw new ObjectStoreServiceException("missing basePath param");
+//			}
+
+			final String objID = profileCreator.registerProfile(interpretation);
+			try {
+				getDao().createObjectStore(objID, interpretation, null);
+			} catch (Throwable e) {
+				log.warn("cannot created objectStore, deleting profile");
+				profileCreator.deleteProfile(objID);
+				throw new ObjectStoreServiceException(e);
+			}
+			job.getParameters().put("objectStoreId", objID);
+			completeWithSuccess(handler, job);
+		} catch (Exception e) {
+			completeWithFail(handler, job, e);
+		}
+
+	}
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/DeleteObjectStoreAction.java

@@ -0,0 +1,55 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;
+import org.springframework.beans.factory.annotation.Autowired;
+
+/**
+ * The Class DeleteObjectStoreAction is responsible to execute the blacboard action of type DELETE.
+ */
+public class DeleteObjectStoreAction extends AbstractObjectStoreAction {
+
+	/**
+	 * The profile creator.
+	 */
+
+	private ObjectStoreProfileCreator profileCreator;
+
+	@Autowired
+	private UniqueServiceLocator serviceLocator;
+
+	@Override
+	protected void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) throws ObjectStoreServiceException {
+		try {
+			final String objID = job.getParameters().get("obsID");
+			serviceLocator.getService(ISRegistryService.class).deleteProfile(objID);
+			getDao().deleteObjectStore(objID);
+			completeWithSuccess(handler, job);
+
+		} catch (Exception e) {
+			completeWithFail(handler, job, e);
+		}
+	}
+
+	/**
+	 * Gets the profile creator.
+	 *
+	 * @return the profile creator
+	 */
+	public ObjectStoreProfileCreator getProfileCreator() {
+		return profileCreator;
+	}
+
+	/**
+	 * Sets the profile creator.
+	 *
+	 * @param profileCreator the new profile creator
+	 */
+	public void setProfileCreator(ObjectStoreProfileCreator profileCreator) {
+		this.profileCreator = profileCreator;
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/DropContentObjectStoreAction.java

@@ -0,0 +1,25 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;
+
+/**
+ * Created by sandro on 2/26/16.
+ */
+public class DropContentObjectStoreAction extends AbstractObjectStoreAction {
+
+	@Override
+	protected void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) throws ObjectStoreServiceException {
+		try {
+			final String objID = job.getParameters().get("obsID");
+
+			final boolean status = getDao().dropContent(objID);
+			job.getParameters().put("dropStatus", "" + status);
+			completeWithSuccess(handler, job);
+		} catch (Exception e) {
+			completeWithFail(handler, job, e);
+		}
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/FeedCompleteObjectAction.java

@@ -0,0 +1,50 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;
+import org.springframework.beans.factory.annotation.Required;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class FeedCompleteObjectAction is responsible to execute the blacboard action of type FEED OBJECT which is Metadata created for WOS case of openaire.
+ */
+public class FeedCompleteObjectAction extends AbstractObjectStoreAction {
+
+	/** The store feeder. */
+	private ModularObjectStoreFeeder storeFeeder;
+
+	@Override
+	protected void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) throws ObjectStoreServiceException {
+		try {
+			final String objStoreID = job.getParameters().get("obsID");
+			final String eprRs = job.getParameters().get("epr");
+			final String mime = job.getParameters().get("mime");
+			int count = storeFeeder.feedMetadataObjectRecord(objStoreID, eprRs, mime);
+			job.getParameters().put("total", "" + count);
+			completeWithSuccess(handler, job);
+		} catch (Exception e) {
+			completeWithFail(handler, job, e);
+		}
+	}
+
+	/**
+	 * Gets the store feeder.
+	 *
+	 * @return the store feeder
+	 */
+	public ModularObjectStoreFeeder getStoreFeeder() {
+		return storeFeeder;
+	}
+
+	/**
+	 * Sets the store feeder.
+	 *
+	 * @param storeFeeder the new store feeder
+	 */
+	@Required
+	public void setStoreFeeder(final ModularObjectStoreFeeder storeFeeder) {
+		this.storeFeeder = storeFeeder;
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/FeedObjectStoreAction.java

@@ -0,0 +1,92 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.data.objectstore.rmi.Protocols;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;
+import org.springframework.beans.factory.annotation.Required;
+
+
+/**
+ * The Class FeedObjectStoreAction to execute the blacboard action of type FEED.
+ */
+public class FeedObjectStoreAction extends AbstractObjectStoreAction {
+
+	/** The profile creator. */
+	private ObjectStoreProfileCreator profileCreator;
+
+	/** The store feeder. */
+	private ModularObjectStoreFeeder storeFeeder;
+
+	@Override
+	protected void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) throws ObjectStoreServiceException {
+		try {
+			final String objStoreID = job.getParameters().get("obsID");
+			final String objID = job.getParameters().get("objID");
+			final String URI = job.getParameters().get("URI");
+			final String eprRs = job.getParameters().get("epr");
+			final String protocolString = job.getParameters().get("protocol");
+
+			final Protocols protocol;
+			if (protocolString == null)
+				protocol = Protocols.None;
+			else
+				protocol = Protocols.valueOf(job.getParameters().get("protocol"));
+			final String mime = job.getParameters().get("mime");
+			final String login = job.getParameters().get("login");
+			final String password = job.getParameters().get("password");
+
+			if (URI != null && URI.length() > 0) {
+				storeFeeder.feedObject(objStoreID, objID, URI, protocol, login,
+						password, mime);
+			} else if (eprRs != null && eprRs.length() > 0) {
+				storeFeeder
+						.feed(objStoreID, eprRs, protocol, login, password, mime);
+			}
+			completeWithSuccess(handler, job);
+		} catch (Exception e) {
+			completeWithFail(handler, job, e);
+		}
+	}
+
+
+
+	/**
+	 * Gets the profile creator.
+	 *
+	 * @return the profile creator
+	 */
+	public ObjectStoreProfileCreator getProfileCreator() {
+		return profileCreator;
+	}
+
+	/**
+	 * Sets the profile creator.
+	 *
+	 * @param profileCreator the new profile creator
+	 */
+	@Required
+	public void setProfileCreator(ObjectStoreProfileCreator profileCreator) {
+		this.profileCreator = profileCreator;
+	}
+
+	/**
+	 * Gets the store feeder.
+	 *
+	 * @return the store feeder
+	 */
+	public ModularObjectStoreFeeder getStoreFeeder() {
+		return storeFeeder;
+	}
+
+	/**
+	 * Sets the store feeder.
+	 *
+	 * @param storeFeeder the new store feeder
+	 */
+	@Required
+	public void setStoreFeeder(ModularObjectStoreFeeder storeFeeder) {
+		this.storeFeeder = storeFeeder;
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ModularObjectStoreDeliver.java

@@ -0,0 +1,128 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import javax.annotation.Resource;
+import javax.xml.ws.wsaddressing.W3CEndpointReference;
+
+import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.enabling.resultset.ResultSetFactory;
+import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;
+import org.springframework.beans.factory.annotation.Required;
+
+/**
+ * The Class ModularObjectStoreDeliver is responsible of delivering data from the object Store.
+ */
+public class ModularObjectStoreDeliver {
+
+	/** The dao. */
+	private ObjectStoreDao dao;
+
+	/** The result set factory. */
+	@Resource
+	private ResultSetFactory resultSetFactory;
+
+	/** The result set client factory. */
+	private ResultSetClientFactory resultSetClientFactory;
+
+	/**
+	 * Gets the dao.
+	 *
+	 * @return the dao
+	 */
+	public ObjectStoreDao getDao() {
+		return dao;
+	}
+
+	/**
+	 * Sets the dao.
+	 *
+	 * @param dao
+	 *            the new dao
+	 */
+	@Required
+	public void setDao(final ObjectStoreDao dao) {
+		this.dao = dao;
+	}
+
+	/**
+	 * Deliver.
+	 *
+	 * @param objectStoreID
+	 *            the object store id
+	 * @param from
+	 *            the from
+	 * @param until
+	 *            the until
+	 * @return the w3 c endpoint reference
+	 * @throws ObjectStoreServiceException
+	 */
+	public W3CEndpointReference deliver(final String objectStoreID, final Long from, final Long until) throws ObjectStoreServiceException {
+		return resultSetFactory.createResultSet(dao.getObjectStore(objectStoreID).deliver(from, until));
+	}
+
+	/**
+	 * Deliver ids.
+	 *
+	 * @param objectStoreID
+	 *            the object store id
+	 * @param eprId
+	 *            the epr id
+	 * @return the w3 c endpoint reference
+	 * @throws ObjectStoreServiceException
+	 */
+	public W3CEndpointReference deliverIds(final String objectStoreID, final W3CEndpointReference eprId) throws ObjectStoreServiceException {
+
+		Iterable<String> ids = resultSetClientFactory.getClient(eprId);
+		return resultSetFactory.createResultSet(dao.getObjectStore(objectStoreID).deliverIds(ids));
+	}
+
+	/**
+	 * Exist id starts with.
+	 *
+	 * @param obsId
+	 *            the obs id
+	 * @param startId
+	 *            the start id
+	 * @return true, if successful
+	 * @throws ObjectStoreServiceException
+	 */
+	public boolean existIDStartsWith(final String obsId, final String startId) throws ObjectStoreServiceException {
+		return dao.getObjectStore(obsId).existIDStartsWith(startId);
+	}
+
+	/**
+	 * Deliver object.
+	 *
+	 * @param objectStoreID
+	 *            the object store id
+	 * @param objectId
+	 *            the object id
+	 * @return the object store file
+	 * @throws ObjectStoreServiceException
+	 */
+	public ObjectStoreFile deliverObject(final String objectStoreID, final String objectId) throws ObjectStoreServiceException {
+		return dao.getObjectStore(objectStoreID).deliverObject(objectId);
+	}
+
+	/**
+	 * Gets the result set client factory.
+	 *
+	 * @return the result set client factory
+	 */
+	public ResultSetClientFactory getResultSetClientFactory() {
+		return resultSetClientFactory;
+	}
+
+	/**
+	 * Sets the result set client factory.
+	 *
+	 * @param resultSetClientFactory
+	 *            the new result set client factory
+	 */
+	@Required
+	public void setResultSetClientFactory(final ResultSetClientFactory resultSetClientFactory) {
+		this.resultSetClientFactory = resultSetClientFactory;
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ModularObjectStoreFeeder.java

@@ -0,0 +1,234 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;
+import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;
+import eu.dnetlib.data.objectstore.rmi.MetadataObjectRecord;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.data.objectstore.rmi.Protocols;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;
+import eu.dnetlib.miscutils.datetime.DateUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Required;
+
+/**
+ * The Class ModularObjectStoreFeeder, responsible to feed data into the object Store
+ */
+public class ModularObjectStoreFeeder {
+
+	private static final Log log = LogFactory.getLog(ModularObjectStoreFeeder.class);
+
+	/** The dao of the objectStore. */
+	private ObjectStoreDao dao;
+
+	/** The result set client factory. */
+	private ResultSetClientFactory resultSetClientFactory;
+
+	/** The service locator. */
+	private UniqueServiceLocator serviceLocator;
+
+	/**
+	 * Gets the dao.
+	 *
+	 * @return the dao
+	 */
+	public ObjectStoreDao getDao() {
+		return dao;
+	}
+
+	/**
+	 * Sets the dao.
+	 *
+	 * @param dao
+	 *            the new dao
+	 */
+	@Required
+	public void setDao(final ObjectStoreDao dao) {
+		this.dao = dao;
+	}
+
+	/**
+	 * Gets the result set client factory.
+	 *
+	 * @return the result set client factory
+	 */
+	public ResultSetClientFactory getResultSetClientFactory() {
+		return resultSetClientFactory;
+	}
+
+	/**
+	 * Sets the result set client factory.
+	 *
+	 * @param resultSetClientFactory
+	 *            the new result set client factory
+	 */
+
+	@Required
+	public void setResultSetClientFactory(final ResultSetClientFactory resultSetClientFactory) {
+		this.resultSetClientFactory = resultSetClientFactory;
+	}
+
+	/**
+	 * Feed metadata object record.
+	 *
+	 * @param objectStoreID
+	 *            the object store id
+	 * @param rsEpr
+	 *            the rs epr
+	 * @param mime
+	 *            the mime
+	 * @throws ObjectStoreServiceException
+	 */
+	public int feedMetadataObjectRecord(final String objectStoreID, final String rsEpr, final String mime) throws ObjectStoreServiceException {
+
+		final Iterable<String> records = resultSetClientFactory.getClient(rsEpr);
+		Iterable<MetadataObjectRecord> toIngest = Iterables.transform(records, input -> {
+			MetadataObjectRecord record = MetadataObjectRecord.initFromJson(input);
+			if (record != null) {
+				record.setMime(mime);
+			} else {
+				log.error("An input record is null :" + input);
+			}
+			return record;
+		});
+		ObjectStore store = dao.getObjectStore(objectStoreID);
+		int size = store.feedMetadataRecord(toIngest, true);
+		touch(objectStoreID, size);
+		return size;
+	}
+
+	/**
+	 * Feed object in the object store starting from an EPR of objectMetadata
+	 *
+	 * @param obsId
+	 *            the objectStore id
+	 * @param rsEpr
+	 *            the result set Endpoint-reference
+	 * @param protocol
+	 *            the protocol
+	 * @param login
+	 *            the login
+	 * @param password
+	 *            the password
+	 * @param mime
+	 *            the mime type
+	 * @throws ObjectStoreServiceException
+	 */
+	public void feed(final String obsId, final String rsEpr, final Protocols protocol, final String login, final String password, final String mime)
+			throws ObjectStoreServiceException {
+		final Iterable<String> records = resultSetClientFactory.getClient(rsEpr);
+		ObjectBroker objectBroker = new ObjectBroker(protocol, login, password, mime);
+		Iterable<ObjectStoreRecord> toIngest = Iterables.transform(records, objectBroker);
+		ObjectStore store = dao.getObjectStore(obsId);
+		int size = store.feed(toIngest, true);
+		touch(obsId, size);
+	}
+
+	/**
+	 * Feed a single object in the object Stroe.
+	 *
+	 * @param objectStoreID
+	 *            the object store id
+	 * @param objectID
+	 *            the object id
+	 * @param URIObject
+	 *            the URI of object
+	 * @param protocol
+	 *            the protocol
+	 * @param login
+	 *            the login
+	 * @param password
+	 *            the password
+	 * @param mime
+	 *            the mime type
+	 * @throws ObjectStoreServiceException
+	 */
+	public void feedObject(final String objectStoreID,
+			final String objectID,
+			final String URIObject,
+			final Protocols protocol,
+			final String login,
+			final String password,
+			final String mime) throws ObjectStoreServiceException {
+		ObjectStoreFile inputFile = new ObjectStoreFile();
+		inputFile.setURI(URIObject);
+		inputFile.setObjectID(objectID);
+		ObjectBroker objectBroker = new ObjectBroker(protocol, login, password, mime);
+		Iterable<ObjectStoreRecord> toIngest = Iterables.transform(Lists.newArrayList(inputFile.toJSON()), objectBroker);
+		ObjectStore store = dao.getObjectStore(objectStoreID);
+		int size = store.feed(toIngest, true);
+		touch(objectStoreID, size);
+	}
+
+	/**
+	 * Feed object record.
+	 *
+	 * @param objectStoreID
+	 *            the object store id
+	 * @param record
+	 *            the record
+	 * @return the string
+	 * @throws ObjectStoreServiceException
+	 */
+	public String feedObjectRecord(final String objectStoreID, final ObjectStoreRecord record) throws ObjectStoreServiceException {
+		ObjectStore store = dao.getObjectStore(objectStoreID);
+		return store.feedObjectRecord(record);
+	}
+
+	/**
+	 * Sets the last modified date in the profile.
+	 *
+	 * @param obsId
+	 *            the obs id
+	 * @param size
+	 *            the size
+	 */
+	public void touch(final String obsId, final int size) {
+		try {
+			final String now = DateUtils.now_ISO8601();
+
+			final String mdstoreXUpdate = "for $x in //RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '" + obsId + "']"
+					+ "return update value $x//LAST_STORAGE_DATE with '" + now + "'";
+
+			serviceLocator.getService(ISRegistryService.class, true).executeXUpdate(mdstoreXUpdate);
+
+			touchSize(obsId, size);
+		} catch (final Exception e) {
+			throw new IllegalStateException(e);
+		}
+	}
+
+	/**
+	 * Touch size.
+	 *
+	 * @param obsId
+	 *            the obs id
+	 * @param size
+	 *            the size
+	 */
+	public void touchSize(final String obsId, final int size) {
+		try {
+			final String mdstoreNumberXUpdate = "for $x in //RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '" + obsId + "']"
+					+ "return update value $x//COUNT_STORE with '" + size + "'";
+
+			serviceLocator.getService(ISRegistryService.class, true).executeXUpdate(mdstoreNumberXUpdate);
+		} catch (final Exception e) {
+			throw new IllegalStateException(e);
+		}
+	}
+
+	public UniqueServiceLocator getServiceLocator() {
+		return serviceLocator;
+	}
+
+	@Required
+	public void setServiceLocator(final UniqueServiceLocator serviceLocator) {
+		this.serviceLocator = serviceLocator;
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ModularObjectStoreService.java

@@ -0,0 +1,265 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import java.util.List;
+
+import javax.xml.ws.wsaddressing.W3CEndpointReference;
+
+import org.springframework.beans.factory.annotation.Required;
+
+import com.google.gson.Gson;
+
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreService;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.enabling.tools.AbstractBaseService;
+import eu.dnetlib.enabling.tools.blackboard.NotificationHandler;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class ModularObjectStoreService is the implementation of the ObjectStoreService interface.
+ */
+public class ModularObjectStoreService extends AbstractBaseService implements ObjectStoreService {
+
+	/** The feeder. */
+	private ModularObjectStoreFeeder feeder;
+
+	/** The object store deliver. */
+	private ModularObjectStoreDeliver objectStoreDeliver;
+
+	/** The notification handler. */
+	private NotificationHandler notificationHandler;
+
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see eu.dnetlib.data.objectstore.rmi.ObjectStoreService#deliverObjects(java.lang.String, java.lang.Double, java.lang.Double)
+	 */
+	/**
+	 * Deliver objects.
+	 *
+	 * @param obsId
+	 *            the obs id
+	 * @param from
+	 *            the from
+	 * @param until
+	 *            the until
+	 * @return the w3 c endpoint reference
+	 * @throws ObjectStoreServiceException
+	 *             the object store service exception
+	 */
+	@Override
+	public W3CEndpointReference deliverObjects(final String obsId, final Long from, final Long until) throws ObjectStoreServiceException {
+
+		return objectStoreDeliver.deliver(obsId, from, until);
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see eu.dnetlib.data.objectstore.rmi.ObjectStoreService#deliverObjectsByIds(java.lang.String,
+	 * javax.xml.ws.wsaddressing.W3CEndpointReference)
+	 */
+	/**
+	 * Deliver objects by ids.
+	 *
+	 * @param obsId
+	 *            the obs id
+	 * @param eprId
+	 *            the epr id
+	 * @return the w3 c endpoint reference
+	 * @throws ObjectStoreServiceException
+	 *             the object store service exception
+	 */
+	@Override
+	public W3CEndpointReference deliverObjectsByIds(final String obsId, final W3CEndpointReference eprId) throws ObjectStoreServiceException {
+		return objectStoreDeliver.deliverIds(obsId, eprId);
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see eu.dnetlib.data.objectstore.rmi.ObjectStoreService#deliverRecord(java.lang.String, java.lang.String)
+	 */
+	/**
+	 * Deliver record.
+	 *
+	 * @param obsId
+	 *            the obs id
+	 * @param objectId
+	 *            the object id
+	 * @return the string
+	 * @throws ObjectStoreServiceException
+	 *             the object store service exception
+	 */
+	@Override
+	public String deliverRecord(final String obsId, final String objectId) throws ObjectStoreServiceException {
+		return objectStoreDeliver.deliverObject(obsId, objectId).toJSON();
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see eu.dnetlib.data.objectstore.rmi.ObjectStoreService#getListOfObjectStores()
+	 */
+	/**
+	 * Gets the list of object stores.
+	 *
+	 * @return the list of object stores
+	 */
+	@Override
+	public List<String> getListOfObjectStores() {
+		return objectStoreDeliver.getDao().listObjectStores();
+	}
+
+	/**
+	 * Gets the feeder.
+	 *
+	 * @return the feeder
+	 */
+	public ModularObjectStoreFeeder getFeeder() {
+		return feeder;
+	}
+
+	/**
+	 * Sets the feeder.
+	 *
+	 * @param feeder
+	 *            the new feeder
+	 */
+	@Required
+	public void setFeeder(final ModularObjectStoreFeeder feeder) {
+		this.feeder = feeder;
+	}
+
+	/**
+	 * Gets the notification handler.
+	 *
+	 * @return the notification handler
+	 */
+	public NotificationHandler getNotificationHandler() {
+		return notificationHandler;
+	}
+
+	/**
+	 * Sets the notification handler.
+	 *
+	 * @param notificationHandler
+	 *            the new notification handler
+	 */
+	@Required
+	public void setNotificationHandler(final NotificationHandler notificationHandler) {
+		this.notificationHandler = notificationHandler;
+	}
+
+	/**
+	 * {@inheritDoc}
+	 *
+	 * @see eu.dnetlib.enabling.tools.AbstractBaseService#notify(java.lang.String, java.lang.String, java.lang.String, java.lang.String)
+	 */
+	@Override
+	public void notify(final String subscriptionId, final String topic, final String isId, final String message) {
+		getNotificationHandler().notified(subscriptionId, topic, isId, message);
+	}
+
+	/**
+	 * Gets the object store deliver.
+	 *
+	 * @return the object store deliver
+	 */
+	public ModularObjectStoreDeliver getObjectStoreDeliver() {
+		return objectStoreDeliver;
+	}
+
+	/**
+	 * Sets the object store deliver.
+	 *
+	 * @param objectStoreDeliver
+	 *            the new object store deliver
+	 */
+	@Required
+	public void setObjectStoreDeliver(final ModularObjectStoreDeliver objectStoreDeliver) {
+		this.objectStoreDeliver = objectStoreDeliver;
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see eu.dnetlib.data.objectstore.rmi.ObjectStoreService#feedObject(java.lang.String, java.lang.String)
+	 */
+	/**
+	 * Feed object.
+	 *
+	 * @param obsId
+	 *            the obs id
+	 * @param objectMetadata
+	 *            the object metadata
+	 * @throws ObjectStoreServiceException
+	 *             the object store service exception
+	 */
+	@Override
+	public void feedObject(final String obsId, final String objectMetadata) throws ObjectStoreServiceException {
+		Gson g = new Gson();
+		try {
+			ObjectStoreFile file = g.fromJson(objectMetadata, ObjectStoreFile.class);
+			feeder.feedObject(obsId, file.getObjectID(), file.getURI(), file.getAccessProtocol(), file.getUsernameAuth(), file.getPasswordAuth(),
+					file.getMimeType());
+
+		} catch (Exception e) {
+			throw new ObjectStoreServiceException(e.getMessage());
+		}
+
+	}
+
+	/**
+	 * Exist id starts with.
+	 *
+	 * @param obsId
+	 *            the obs id
+	 * @param startId
+	 *            the start id
+	 * @return true, if successful
+	 * @throws ObjectStoreServiceException
+	 *             the object store service exception
+	 */
+	public boolean existIDStartsWith(final String obsId, final String startId) throws ObjectStoreServiceException {
+		return objectStoreDeliver.existIDStartsWith(obsId, startId);
+	}
+
+	/**
+	 * Feed object.
+	 *
+	 * @param obsId
+	 *            the obs id
+	 * @param record
+	 *            the record
+	 * @return the string
+	 * @throws ObjectStoreServiceException
+	 *             the object store service exception
+	 */
+	public String feedObject(final String obsId, final ObjectStoreRecord record) throws ObjectStoreServiceException {
+
+		try {
+
+			return feeder.feedObjectRecord(obsId, record);
+
+		} catch (Exception e) {
+			throw new ObjectStoreServiceException(e.getMessage());
+		}
+
+	}
+
+	/**
+	 * Gets the size.
+	 *
+	 * @param obsId
+	 *            the obs id
+	 * @return the size
+	 * @throws ObjectStoreServiceException
+	 *             the object store service exception
+	 */
+	@Override
+	public int getSize(final String obsId) throws ObjectStoreServiceException {
+		return objectStoreDeliver.getDao().getObjectStore(obsId).getSize();
+
+	}
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectBroker.java

@@ -0,0 +1,240 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.SocketException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.net.ftp.FTPClient;
+
+import com.google.common.base.Function;
+
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
+import eu.dnetlib.data.objectstore.rmi.Protocols;
+
+
+
+/**
+ * The Class ObjectBroker is responsible to retrieve content from URL.
+ */
+public class ObjectBroker implements Function<String, ObjectStoreRecord> {
+	private static final Log log = LogFactory.getLog(ObjectBroker.class);
+
+	/** The protocol. */
+	private Protocols protocol;
+
+	/** The login. */
+	private String login;
+
+	/** The password. */
+	private String password;
+
+	/** The mime. */
+	private String mime;
+
+	/**
+	 * Instantiates a new object broker.
+	 * 
+	 * @param protocol
+	 *            the protocol
+	 * @param login
+	 *            the login
+	 * @param password
+	 *            the password
+	 * @param mime
+	 *            the mime
+	 */
+	public ObjectBroker(final Protocols protocol, final String login, final String password, final String mime) {
+		this.protocol = protocol;
+		this.login = login;
+		this.password = password;
+		this.mime = mime;
+	}
+
+	/**
+	 * Gets the protocol.
+	 * 
+	 * @return the protocol
+	 */
+	public Protocols getProtocol() {
+		return protocol;
+	}
+
+	/**
+	 * Sets the protocol.
+	 * 
+	 * @param protocol
+	 *            the new protocol
+	 */
+	public void setProtocol(final Protocols protocol) {
+		this.protocol = protocol;
+	}
+
+	/**
+	 * Gets the login.
+	 * 
+	 * @return the login
+	 */
+	public String getLogin() {
+		return login;
+	}
+
+	/**
+	 * Sets the login.
+	 * 
+	 * @param login
+	 *            the new login
+	 */
+	public void setLogin(final String login) {
+		this.login = login;
+	}
+
+	/**
+	 * Gets the password.
+	 * 
+	 * @return the password
+	 */
+	public String getPassword() {
+		return password;
+	}
+
+	/**
+	 * Sets the password.
+	 * 
+	 * @param password
+	 *            the new password
+	 */
+	public void setPassword(final String password) {
+		this.password = password;
+	}
+
+	/**
+	 * Gets the mime.
+	 * 
+	 * @return the mime
+	 */
+	public String getMime() {
+		return mime;
+	}
+
+	/**
+	 * Sets the mime.
+	 * 
+	 * @param mime
+	 *            the new mime
+	 */
+	public void setMime(final String mime) {
+		this.mime = mime;
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see com.google.common.base.Function#apply(java.lang.Object)
+	 */
+	@Override
+	public ObjectStoreRecord apply(final String jsonInput) {
+		if (jsonInput == null) return null;
+		ObjectStoreRecord objectStorerecord = new ObjectStoreRecord();
+
+		objectStorerecord.setFileMetadata(ObjectStoreFile.createObject(jsonInput));
+		Protocols currentProtocol = (objectStorerecord.getFileMetadata().getAccessProtocol() == Protocols.None) ? protocol : objectStorerecord
+				.getFileMetadata().getAccessProtocol();
+		objectStorerecord.getFileMetadata().setAccessProtocol(currentProtocol);
+		if ((objectStorerecord.getFileMetadata().getMimeType() == null) || (objectStorerecord.getFileMetadata().getMimeType().length() == 0)) {
+			objectStorerecord.getFileMetadata().setMimeType(mime);
+		}
+
+		switch (currentProtocol) {
+		case FTP:
+			FTPClient client = new FTPClient();
+			try {
+				URI uri = new URI(objectStorerecord.getFileMetadata().getURI());
+				client.connect(uri.getHost());
+				if ((objectStorerecord.getFileMetadata().getUsernameAuth() != null) && (objectStorerecord.getFileMetadata().getUsernameAuth().length() > 0)) {
+					client.login(objectStorerecord.getFileMetadata().getUsernameAuth(), objectStorerecord.getFileMetadata().getPasswordAuth());
+				} else {
+					client.login("ftp", "a@a");
+
+				}
+
+				final InputStream stream = client.retrieveFileStream(uri.getPath());
+				objectStorerecord.setInputStream(stream);
+				return objectStorerecord;
+
+			} catch (URISyntaxException e2) {
+				log.error(e2);
+				return null;
+			} catch (SocketException e) {
+				log.error(e);
+			} catch (IOException e) {
+				log.error(e);
+				return null;
+			}
+
+		case HTTP:
+			try {
+				HttpURLConnection conn = (HttpURLConnection) new URL(objectStorerecord.getFileMetadata().getURI()).openConnection();
+				InputStream in;
+				int http_status;
+				try {
+
+					http_status = conn.getResponseCode();
+
+					if ((http_status / 100) == 3) {
+						String newURL = conn.getHeaderField("Location");
+						conn.disconnect();
+						conn = (HttpURLConnection) new URL(newURL).openConnection();
+						http_status = conn.getResponseCode();
+						if ((http_status / 100) != 2) return null;
+					}
+					in = conn.getInputStream();
+					objectStorerecord.setInputStream(in);
+					return objectStorerecord;
+				} catch (Exception e) {
+					log.error(e);
+					return null;
+				}
+			} catch (MalformedURLException e1) {
+				log.error(e1);
+				return null;
+			} catch (IOException e1) {
+				log.error(e1);
+				return null;
+			}
+
+		case File_System:
+			File f = new File(objectStorerecord.getFileMetadata().getURI());
+			try {
+				InputStream myiInputStream = new FileInputStream(f);
+				objectStorerecord.setInputStream(myiInputStream);
+				return objectStorerecord;
+			} catch (FileNotFoundException e) {
+				try {
+					Thread.sleep(5000);
+					InputStream myiInputStream = new FileInputStream(f);
+					objectStorerecord.setInputStream(myiInputStream);
+					return objectStorerecord;
+				} catch (Exception e1) {
+					log.error(e1);
+					return null;
+				}
+			}
+		default:
+			break;
+		}
+
+		return null;
+
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectStoreActions.java

@@ -0,0 +1,22 @@
+package eu.dnetlib.data.objectstore.modular;
+
+/**
+ * The Enumeration of the blacboard action of the  ObjectStore.
+ */
+public enum ObjectStoreActions {
+	
+	/** The create. */
+	CREATE,
+	/**
+	 * The delete.
+	 */
+	DELETE,
+	/**
+	 * The feed.
+	 */
+	FEED,
+	/**
+	 * The feedobject.
+	 */
+	FEEDOBJECT, DROP_CONTENT
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectStoreConsistency.java

@@ -0,0 +1,102 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import java.util.List;
+import java.util.Set;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Controller;
+import org.springframework.ui.ModelMap;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.ResponseBody;
+
+
+@Controller
+public class ObjectStoreConsistency {
+
+    private static final Log log = LogFactory.getLog(ObjectStoreConsistency.class);
+
+    @Autowired
+    private ObjectStoreDao objectStoreDao;
+
+    @Autowired
+    private UniqueServiceLocator serviceLocator;
+
+    private ObjectStoreIntegrityInfo integrityInfo;
+
+
+    @RequestMapping(value = "/ui/objectStore/infoConsistency.do")
+    @ResponseBody
+    public ObjectStoreIntegrityInfo getConsistencyInfo(final ModelMap map) {
+        return getIntegrityInfo();
+    }
+
+    public void refreshConsistency() {
+        try {
+            Set<String> profiles = listProfileIds();
+            Set<String> objectStore = listObjectStoreIds();
+            Set<String> objectStoreNotInProfile = Sets.newHashSet(objectStore);
+            objectStoreNotInProfile.removeAll(profiles);
+            Set<String> profilesNotInObjectStore = Sets.newHashSet(profiles);
+            profilesNotInObjectStore.removeAll(objectStore);
+            ObjectStoreIntegrityInfo info = new ObjectStoreIntegrityInfo();
+            info.setObjectStoreWithoutProfile(objectStoreNotInProfile);
+            info.setProfileWithoutObjectStore(profilesNotInObjectStore);
+            this.integrityInfo = info;
+        } catch (ISLookUpException e) {
+            log.error("Error on refreshing consistency of objectStore ", e);
+        }
+
+
+    }
+
+
+    private Set<String> listObjectStoreIds() {
+        return Sets.newHashSet(
+                Lists.transform(objectStoreDao.listObjectStores(), new Function<String, String>() {
+                    @Override
+                    public String apply(String input) {
+                        return StringUtils.substringBefore(input, "_");
+                    }
+                }));
+    }
+
+
+    private Set<String> listProfileIds() throws ISLookUpException {
+
+        final ISLookUpService lookupService = serviceLocator.getService(ISLookUpService.class);
+        final String query = "for $x in collection('/db/DRIVER/ObjectStoreDSResources/ObjectStoreDSResourceType') return $x//RESOURCE_IDENTIFIER/@value/string()";
+
+        List<String> resultList = lookupService.quickSearchProfile(query);
+
+
+        return Sets.newHashSet(Lists.transform(resultList, new Function<String, String>() {
+            @Override
+            public String apply(String input) {
+                return StringUtils.substringBefore(input, "_");
+            }
+        }));
+
+    }
+
+
+    public ObjectStoreIntegrityInfo getIntegrityInfo() {
+        if (integrityInfo == null) {
+            refreshConsistency();
+        }
+        return integrityInfo;
+    }
+
+    public void setIntegrityInfo(ObjectStoreIntegrityInfo integrityInfo) {
+        this.integrityInfo = integrityInfo;
+    }
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectStoreIntegrityInfo.java

@@ -0,0 +1,30 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import java.util.Set;
+
+/**
+ * Created by Sandro La Bruzzo on 10/16/15.
+ */
+public class ObjectStoreIntegrityInfo {
+
+    private Set<String> objectStoreWithoutProfile;
+
+    private Set<String> profileWithoutObjectStore;
+
+
+    public Set<String> getObjectStoreWithoutProfile() {
+        return objectStoreWithoutProfile;
+    }
+
+    public void setObjectStoreWithoutProfile(Set<String> objectStoreWithoutProfile) {
+        this.objectStoreWithoutProfile = objectStoreWithoutProfile;
+    }
+
+    public Set<String> getProfileWithoutObjectStore() {
+        return profileWithoutObjectStore;
+    }
+
+    public void setProfileWithoutObjectStore(Set<String> profileWithoutObjectStore) {
+        this.profileWithoutObjectStore = profileWithoutObjectStore;
+    }
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectStoreProfileCreator.java

@@ -0,0 +1,127 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.soap.EndpointReferenceBuilder;
+import org.antlr.stringtemplate.StringTemplate;
+import org.springframework.beans.factory.annotation.Required;
+
+import javax.xml.ws.Endpoint;
+
+// TODO: Auto-generated Javadoc
+
+/**
+ * The Class ObjectStoreProfileCreator is responsible of creating profile of the ObjectStore
+ */
+public class ObjectStoreProfileCreator {
+
+    /**
+     * service locator.
+     */
+    private UniqueServiceLocator serviceLocator;
+
+    /**
+     * objectstore ds template.
+     */
+    private StringTemplate objectstoreDsTemplate;
+
+    /**
+     * service endpoint.
+     */
+    private Endpoint endpoint;
+
+    /**
+     * endpoint builder.
+     */
+    private EndpointReferenceBuilder<Endpoint> eprBuilder;
+
+    /**
+     * Register profile.
+     *
+     * @param interpretation the interpretation
+     * @return the string
+     * @throws ISRegistryException the IS registry exception
+     */
+    public String registerProfile(final String interpretation)
+            throws ISRegistryException {
+        // XXX: mini hack
+        StringTemplate template = new StringTemplate(
+                objectstoreDsTemplate.getTemplate());
+        template.setAttribute("serviceUri", eprBuilder.getAddress(endpoint));
+        template.setAttribute("interpretation", interpretation);
+
+        return serviceLocator.getService(ISRegistryService.class, true).registerProfile(template.toString());
+    }
+
+    public void deleteProfile(final String profileId) throws ISRegistryException {
+        serviceLocator.getService(ISRegistryService.class).deleteProfile(profileId);
+    }
+
+    /**
+     * Gets the endpoint.
+     *
+     * @return the endpoint
+     */
+    public Endpoint getEndpoint() {
+        return endpoint;
+    }
+
+    /**
+     * Sets the endpoint.
+     *
+     * @param endpoint the new endpoint
+     */
+    @Required
+    public void setEndpoint(final Endpoint endpoint) {
+        this.endpoint = endpoint;
+    }
+
+    /**
+     * Gets the epr builder.
+     *
+     * @return the epr builder
+     */
+    public EndpointReferenceBuilder<Endpoint> getEprBuilder() {
+        return eprBuilder;
+    }
+
+    /**
+     * Sets the epr builder.
+     *
+     * @param eprBuilder the new epr builder
+     */
+    @Required
+    public void setEprBuilder(final EndpointReferenceBuilder<Endpoint> eprBuilder) {
+        this.eprBuilder = eprBuilder;
+    }
+
+    /**
+     * Gets the objectstore ds template.
+     *
+     * @return the objectstore ds template
+     */
+    public StringTemplate getObjectstoreDsTemplate() {
+        return objectstoreDsTemplate;
+    }
+
+    /**
+     * Sets the objectstore ds template.
+     *
+     * @param objectstoreDsTemplate the new objectstore ds template
+     */
+    @Required
+    public void setObjectstoreDsTemplate(final StringTemplate objectstoreDsTemplate) {
+        this.objectstoreDsTemplate = objectstoreDsTemplate;
+    }
+
+    public UniqueServiceLocator getServiceLocator() {
+        return serviceLocator;
+    }
+
+    @Required
+    public void setServiceLocator(final UniqueServiceLocator serviceLocator) {
+        this.serviceLocator = serviceLocator;
+    }
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectStoreRecord.java

@@ -0,0 +1,55 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import java.io.InputStream;
+
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
+
+/**
+ * The Class ObjectStoreRecord is a serialization of the object to be ingested in the objectStore
+ * metadata + inputstream of the data.
+ */
+public class ObjectStoreRecord {
+
+	/** The file metadata. */
+	private ObjectStoreFile fileMetadata;
+
+	/** The input stream. */
+	private InputStream inputStream;
+
+	/**
+	 * Gets the input stream.
+	 *
+	 * @return the input stream
+	 */
+	public InputStream getInputStream() {
+		return inputStream;
+	}
+
+	/**
+	 * Sets the input stream.
+	 *
+	 * @param inputStream the new input stream
+	 */
+	public void setInputStream(final InputStream inputStream) {
+		this.inputStream = inputStream;
+	}
+
+	/**
+	 * Gets the file metadata.
+	 *
+	 * @return the file metadata
+	 */
+	public ObjectStoreFile getFileMetadata() {
+		return fileMetadata;
+	}
+
+	/**
+	 * Sets the file metadata.
+	 *
+	 * @param fileMetadata the new file metadata
+	 */
+	public void setFileMetadata(final ObjectStoreFile fileMetadata) {
+		this.fileMetadata = fileMetadata;
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/connector/ObjectStore.java

@@ -0,0 +1,131 @@
+package eu.dnetlib.data.objectstore.modular.connector;
+
+import eu.dnetlib.data.objectstore.modular.ObjectStoreRecord;
+import eu.dnetlib.data.objectstore.rmi.MetadataObjectRecord;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.enabling.resultset.ResultSetListener;
+
+/**
+ * The Interface ObjectStore.
+ */
+public interface ObjectStore {
+
+	/**
+	 * Gets the id of the objectStore.
+	 *
+	 * @return the id
+	 */
+	String getId();
+
+	/**
+	 * Gets the interpretation the objectStore.
+	 *
+	 * @return the interpretation
+	 */
+	String getInterpretation();
+
+	/**
+	 * Feed record into the objectstore.
+	 *
+	 * @param records
+	 *            the records
+	 * @param incremental
+	 *            the incremental
+	 * @return the int
+	 */
+	int feed(Iterable<ObjectStoreRecord> records, boolean incremental) throws ObjectStoreServiceException;
+
+	/**
+	 * Feed metadata record into the objectStore.
+	 *
+	 * @param records
+	 *            the records
+	 * @param incremental
+	 *            the incremental
+	 * @return the int
+	 */
+	int feedMetadataRecord(Iterable<MetadataObjectRecord> records, boolean incremental) throws ObjectStoreServiceException;
+
+	/**
+	 * Feed a single object record into the objectStore.
+	 *
+	 * @param record
+	 *            the record
+	 * @return the string
+	 */
+	String feedObjectRecord(ObjectStoreRecord record) throws ObjectStoreServiceException;
+
+	/**
+	 * Deliver Object from the objectStore.
+	 *
+	 * @param from
+	 *            : start date which you want to filter
+	 * @param until
+	 *            : end date which you want to filter
+	 * @return the result set listener
+	 */
+	ResultSetListener deliver(Long from, Long until) throws ObjectStoreServiceException;
+
+	/**
+	 * Deliver ids.
+	 *
+	 * @param ids
+	 *            the ids
+	 * @return the result set listener
+	 */
+	ResultSetListener deliverIds(Iterable<String> ids) throws ObjectStoreServiceException;
+
+	/**
+	 * Deliver object.
+	 *
+	 * @param objectId
+	 *            the object id
+	 * @return the object store file
+	 * @throws ObjectStoreServiceException
+	 */
+	ObjectStoreFile deliverObject(String objectId) throws ObjectStoreServiceException;
+
+	/**
+	 * Gets the size.
+	 *
+	 * @return the size
+	 */
+	int getSize() throws ObjectStoreServiceException;
+
+	/**
+	 * Delete object.
+	 *
+	 * @param objectId
+	 *            the object id
+	 */
+	void deleteObject(String objectId) throws ObjectStoreServiceException;
+
+	/**
+	 * Gets the object.
+	 *
+	 * @param recordId
+	 *            the record id
+	 * @return the object
+	 */
+	String getObject(String recordId) throws ObjectStoreServiceException;
+
+	/**
+	 * Find if exist an ID startingwith the string startId.
+	 *
+	 * @param startId
+	 *            the start id
+	 * @return if exist or less an id
+	 */
+	boolean existIDStartsWith(String startId) throws ObjectStoreServiceException;
+
+	/**
+	 * drop the content of the ObjectStore
+	 * THIS METHOD DELETE ALL THE CONTENT INSIDE THE OBJECTSTORE PAY ATTENTION WHEN YOU CALL IT
+	 *
+	 * @return if the content has been refreshed
+	 * @throws ObjectStoreServiceException
+	 */
+	boolean dropContent() throws ObjectStoreServiceException;
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/connector/ObjectStoreDao.java

@@ -0,0 +1,72 @@
+package eu.dnetlib.data.objectstore.modular.connector;
+
+import java.util.List;
+
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+
+/**
+ * The Interface ObjectStoreDao.
+ */
+public interface ObjectStoreDao {
+
+	/**
+	 * Gets an Object Store with the given identifier.
+	 * <p>
+	 * If an Object Store with the given identifier does not exist, a new one is created.
+	 * </p>
+	 *
+	 * @param obsId
+	 *            the object store identifier
+	 * @return the object store
+	 * @throws ObjectStoreServiceException
+	 */
+	ObjectStore getObjectStore(String obsId) throws ObjectStoreServiceException;
+
+	/**
+	 * List all the Object stores.
+	 *
+	 * @return the list of object store ids
+	 */
+	List<String> listObjectStores();
+
+	/**
+	 * Creates an Object Store with the given identifier.
+	 * <p>
+	 * If an Object Store with the given identifier already exists this method does nothing.
+	 * </p>
+	 *
+	 * @param obsId            	the object store identifier
+	 * @param interpretation    the interpretation of the store
+	 * @param basePath 			the base path to store the object Store in case of file system implementation
+	 * @return true, if successful
+	 * @throws ObjectStoreServiceException
+	 */
+	boolean createObjectStore(String obsId, String interpretation, String basePath) throws ObjectStoreServiceException;
+
+	/**
+	 * Upddate an Object Store metadata with the given identifier.
+	 * <p>
+	 * If an Object Store with the given identifier does not exist, a new one is created.
+	 * </p>
+	 *
+	 * @param obsId
+	 *            the object store identifier
+	 * @param interpretation
+	 *            the interpretation of the store
+	 * @return true, if successful
+	 */
+	boolean updateObjectStore(String obsId, String interpretation);
+
+	/**
+	 * Delete object store.
+	 *
+	 * @param obsId
+	 *            the object store identifier
+	 * @return true, if successful
+	 * @throws ObjectStoreServiceException
+	 */
+	boolean deleteObjectStore(String obsId) throws ObjectStoreServiceException;
+
+	boolean dropContent(String obsId) throws ObjectStoreServiceException;
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/inspector/ObjectStoreInspector.java

@@ -0,0 +1,199 @@
+package eu.dnetlib.data.objectstore.modular.inspector;
+
+import static eu.dnetlib.miscutils.collections.MappedCollection.listMap;
+
+import java.util.List;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Controller;
+import org.springframework.ui.Model;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+
+import com.google.gson.Gson;
+
+import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;
+import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.enabling.inspector.AbstractInspectorController;
+import eu.dnetlib.enabling.resultset.ResultSetListener;
+import eu.dnetlib.miscutils.collections.MappedCollection;
+import eu.dnetlib.miscutils.functional.UnaryFunction;
+
+/**
+ * The Class ObjectStoreInspector is the inspector controller of the objectStore.
+ */
+@Controller
+public class ObjectStoreInspector extends AbstractInspectorController {
+
+	// private static final Log log = LogFactory.getLog(ObjectStoreInspector.class);
+
+	/** The object store dao. */
+	@Autowired
+	private ObjectStoreDao objectStoreDao;
+
+	/**
+	 * The Class ObjectStoreDescription.
+	 */
+	class ObjectStoreDescription {
+
+		/** The id. */
+		private String id;
+
+		/** The size. */
+		private int size;
+
+		/**
+		 * Instantiates a new object store description.
+		 *
+		 * @param id
+		 *            the id
+		 * @param size
+		 *            the size
+		 */
+		public ObjectStoreDescription(final String id, final int size) {
+
+			this.id = id;
+			this.size = size;
+		}
+
+		/**
+		 * Gets the interpretation.
+		 *
+		 * @return the interpretation
+		 * @throws ObjectStoreServiceException
+		 */
+		public String getInterpretation() throws ObjectStoreServiceException {
+
+			ObjectStore objectStore = objectStoreDao.getObjectStore(id);
+			return objectStore.getInterpretation();
+		}
+
+		/**
+		 * Gets the id.
+		 *
+		 * @return the id
+		 */
+		public String getId() {
+			return id;
+		}
+
+		/**
+		 * Sets the id.
+		 *
+		 * @param id
+		 *            the new id
+		 */
+		public void setId(final String id) {
+			this.id = id;
+		}
+
+		/**
+		 * Gets the size.
+		 *
+		 * @return the size
+		 */
+		public int getSize() {
+			return size;
+		}
+
+		/**
+		 * Sets the size.
+		 *
+		 * @param size
+		 *            the new size
+		 */
+		public void setSize(final int size) {
+			this.size = size;
+		}
+	}
+
+	/**
+	 * Object stores.
+	 *
+	 * @param model
+	 *            the model
+	 */
+	@RequestMapping(value = "/inspector/objectstores.do")
+	public void objectStores(final Model model) {
+		List<String> objectStores = objectStoreDao.listObjectStores();
+		model.addAttribute("objectstores", MappedCollection.listMap(objectStores, new UnaryFunction<ObjectStoreDescription, String>() {
+
+			@Override
+			public ObjectStoreDescription evaluate(final String id) {
+				try {
+					ObjectStore objStore = objectStoreDao.getObjectStore(id);
+					return new ObjectStoreDescription(id, objStore.getSize());
+				} catch (ObjectStoreServiceException e) {
+					return null;
+				}
+			}
+		}));
+	}
+
+	/**
+	 * Object store.
+	 *
+	 * @param model
+	 *            the model
+	 * @param id
+	 *            the id
+	 * @param startParam
+	 *            the start param
+	 * @param regex
+	 *            the regex
+	 * @throws ObjectStoreServiceException
+	 */
+	@RequestMapping(value = "/inspector/objectstore.do", method = RequestMethod.GET)
+	public void objectStore(final Model model,
+			@RequestParam("id") final String id,
+			@RequestParam(value = "start", required = false) final Integer startParam,
+			@RequestParam(value = "regex", required = false) final String regex) throws ObjectStoreServiceException {
+		int pageSize = 10;
+		int start = 0;
+
+		if (startParam != null) {
+			start = startParam;
+		}
+		ObjectStore objctStore = objectStoreDao.getObjectStore(id);
+		ResultSetListener rs = objctStore.deliver((long) 0, System.currentTimeMillis());
+		List<String> page = rs.getResult((1 + start), (start + pageSize));
+		final Gson g = new Gson();
+		model.addAttribute("id", id);
+		model.addAttribute("start", start);
+		model.addAttribute("regex", regex);
+		model.addAttribute("nextPage", start + pageSize);
+		model.addAttribute("prevPage", Math.max(0, start - pageSize));
+		model.addAttribute("size", rs.getSize());
+		model.addAttribute("page", listMap(page, new UnaryFunction<ObjectStoreFile, String>() {
+
+			@Override
+			public ObjectStoreFile evaluate(final String json) {
+				return g.fromJson(json, ObjectStoreFile.class);
+			}
+		}));
+
+	}
+
+	/**
+	 * Gets the object store dao.
+	 *
+	 * @return the object store dao
+	 */
+	public ObjectStoreDao getObjectStoreDao() {
+		return objectStoreDao;
+	}
+
+	/**
+	 * Sets the object store dao.
+	 *
+	 * @param objectStoreDao
+	 *            the new object store dao
+	 */
+	public void setObjectStoreDao(final ObjectStoreDao objectStoreDao) {
+		this.objectStoreDao = objectStoreDao;
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/MetadataObjectRecord.java

@@ -0,0 +1,61 @@
+package eu.dnetlib.data.objectstore.rmi;
+
+import java.io.Serializable;
+
+import com.google.gson.Gson;
+
+public class MetadataObjectRecord implements Serializable {
+
+	/** The Constant serialVersionUID. */
+	private static final long serialVersionUID = -5640196854633924754L;
+
+	public static MetadataObjectRecord initFromJson(final String input) {
+		return new Gson().fromJson(input, MetadataObjectRecord.class);
+	}
+
+	public MetadataObjectRecord() {
+
+	}
+
+	public MetadataObjectRecord(final String id, final String record, final String mime) {
+		super();
+		this.id = id;
+		this.record = record;
+		this.mime = mime;
+	}
+
+	private String id;
+
+	private String record;
+
+	private String mime;
+
+	public String getId() {
+		return id;
+	}
+
+	public String getRecord() {
+		return record;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public void setRecord(final String record) {
+		this.record = record;
+	}
+
+	public String toJSON() {
+		return new Gson().toJson(this);
+	}
+
+	public String getMime() {
+		return mime;
+	}
+
+	public void setMime(final String mime) {
+		this.mime = mime;
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/ObjectStoreFile.java

@@ -0,0 +1,258 @@
+package eu.dnetlib.data.objectstore.rmi;
+
+import java.io.Serializable;
+
+import com.google.gson.Gson;
+
+/**
+ * The Class ObjectStoreFile.
+ */
+public class ObjectStoreFile implements Serializable {
+
+	/** The Constant serialVersionUID. */
+	private static final long serialVersionUID = -6501291693572693712L;
+
+	/** The metadata related id. */
+	private String metadataRelatedID;
+
+	/** The object id. */
+	private String objectID;
+
+	/** The mime type. */
+	private String mimeType;
+
+	/** The access protocol. */
+	private Protocols accessProtocol;
+
+	/** The object uri . */
+	private String URI;
+
+	/** The username auth. */
+	private String usernameAuth;
+
+	/** The password auth. */
+	private String passwordAuth;
+
+	/** The downloaded url. */
+	private String downloadedURL;
+
+	/** The md5 sum of the file. */
+	private String md5Sum;
+
+	/** The file size. */
+	private long fileSizeKB;
+
+	/**
+	 * Gets the downloaded url.
+	 *
+	 * @return the downloaded url
+	 */
+	public String getDownloadedURL() {
+		return downloadedURL;
+	}
+
+	/**
+	 * Sets the downloaded url.
+	 *
+	 * @param downloadedURL
+	 *            the new downloaded url
+	 */
+	public void setDownloadedURL(final String downloadedURL) {
+		this.downloadedURL = downloadedURL.replace("\\u003d", "=").replace("\\u0026", "&");;
+	}
+
+	/**
+	 * Instantiates a new object store file.
+	 */
+	public ObjectStoreFile() {
+		this.accessProtocol = Protocols.None;
+
+	}
+
+	/**
+	 * Return a new instance of the object from a json String.
+	 *
+	 * @param jsonObject
+	 *            the json object
+	 * @return the object store file
+	 */
+	public static ObjectStoreFile createObject(final String jsonObject) {
+		Gson g = new Gson();
+		return g.fromJson(jsonObject, ObjectStoreFile.class);
+	}
+
+	/**
+	 * Gets the object id.
+	 *
+	 * @return the object id
+	 */
+	public String getObjectID() {
+		return objectID;
+	}
+
+	/**
+	 * Sets the object id.
+	 *
+	 * @param objectID
+	 *            the new object id
+	 */
+	public void setObjectID(final String objectID) {
+		this.objectID = objectID;
+	}
+
+	/**
+	 * Gets the mime type.
+	 *
+	 * @return the mime type
+	 */
+	public String getMimeType() {
+		return mimeType;
+	}
+
+	/**
+	 * Sets the mime type.
+	 *
+	 * @param mimeType
+	 *            the new mime type
+	 */
+	public void setMimeType(final String mimeType) {
+		this.mimeType = mimeType;
+	}
+
+	/**
+	 * Gets the access protocol.
+	 *
+	 * @return the access protocol
+	 */
+	public Protocols getAccessProtocol() {
+		return accessProtocol;
+	}
+
+	/**
+	 * Sets the access protocol.
+	 *
+	 * @param accessProtocol
+	 *            the new access protocol
+	 */
+	public void setAccessProtocol(final Protocols accessProtocol) {
+		this.accessProtocol = accessProtocol;
+	}
+
+	/**
+	 * Gets the username auth.
+	 *
+	 * @return the username auth
+	 */
+	public String getUsernameAuth() {
+		return usernameAuth;
+	}
+
+	/**
+	 * Sets the username auth.
+	 *
+	 * @param usernameAuth
+	 *            the new username auth
+	 */
+	public void setUsernameAuth(final String usernameAuth) {
+		this.usernameAuth = usernameAuth;
+	}
+
+	/**
+	 * Gets the password auth.
+	 *
+	 * @return the password auth
+	 */
+	public String getPasswordAuth() {
+		return passwordAuth;
+	}
+
+	/**
+	 * Sets the password auth.
+	 *
+	 * @param passwordAuth
+	 *            the new password auth
+	 */
+	public void setPasswordAuth(final String passwordAuth) {
+		this.passwordAuth = passwordAuth;
+	}
+
+	/**
+	 * Gets the uri.
+	 *
+	 * @return the uri
+	 */
+	public String getURI() {
+		return URI != null ? URI.replace("\\u003d", "=").replace("\\u0026", "&") : "";
+	}
+
+	/**
+	 * Sets the uri.
+	 *
+	 * @param uri
+	 *            the new uri
+	 */
+	public void setURI(final String uri) {
+		if (uri != null) {
+			URI = uri.replace("\\u003d", "=").replace("\\u0026", "&");
+		}
+	}
+
+	/**
+	 * Convert the object into a json String.
+	 *
+	 * @return the string
+	 */
+	public String toJSON() {
+		Gson g = new Gson();
+		return g.toJson(this).replace("\\u003d", "=").replace("\\u0026", "&");
+	}
+
+	/**
+	 * Gets the related metadata id.
+	 *
+	 * @return the related metadata id.
+	 */
+	public String getMetadataRelatedID() {
+		return metadataRelatedID;
+	}
+
+	/**
+	 * Sets the related metadata id.
+	 *
+	 * @param metadataRelatedID
+	 *            the related metadata id.
+	 */
+	public void setMetadataRelatedID(final String metadataRelatedID) {
+		this.metadataRelatedID = metadataRelatedID;
+	}
+
+	/**
+	 * @return the md5Sum
+	 */
+	public String getMd5Sum() {
+		return md5Sum;
+	}
+
+	/**
+	 * @param md5Sum
+	 *            the md5Sum to set
+	 */
+	public void setMd5Sum(final String md5Sum) {
+		this.md5Sum = md5Sum;
+	}
+
+	/**
+	 * @return the fileSizeKB
+	 */
+	public long getFileSizeKB() {
+		return fileSizeKB;
+	}
+
+	/**
+	 * @param fileSizeKB the fileSizeKB to set
+	 */
+	public void setFileSizeKB(long fileSizeKB) {
+		this.fileSizeKB = fileSizeKB;
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/ObjectStoreFileNotFoundException.java

@@ -0,0 +1,22 @@
+package eu.dnetlib.data.objectstore.rmi;
+
+/**
+ * The Class ObjectStoreFileNotFoundException.
+ */
+public class ObjectStoreFileNotFoundException extends ObjectStoreServiceException {
+
+	/** The Constant serialVersionUID. */
+	private static final long serialVersionUID = -5418923557898374219L;
+
+	/**
+	 * Instantiates a new object store file not found exception.
+	 *
+	 * @param string
+	 *            the string
+	 */
+	public ObjectStoreFileNotFoundException(final String string) {
+		super(string);
+
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/ObjectStoreService.java

@@ -0,0 +1,109 @@
+package eu.dnetlib.data.objectstore.rmi;
+
+import java.util.List;
+
+import javax.jws.WebMethod;
+import javax.jws.WebParam;
+import javax.jws.WebService;
+import javax.xml.ws.wsaddressing.W3CEndpointReference;
+
+import eu.dnetlib.common.rmi.BaseService;
+
+/**
+ * Main ObjectStore Service interface.
+ *
+ * @author <a href="mailto:sandro.labruzzo at isti.cnr.it">Sandro La Bruzzo</a>
+ * @version 0.0.1
+ */
+@WebService(targetNamespace = "http://services.dnetlib.eu/")
+public interface ObjectStoreService extends BaseService {
+
+	/**
+	 * Returns ResultSet EPR for delivered ObjectStore records in a particular range date.
+	 * <p>
+	 * Please check service implementations for details on the expected format of the records in the result set epr.
+	 * </p>
+	 * <p>
+	 * This method could be used for a bulk deliver of all objects in the store
+	 * </p>
+	 *
+	 * @param obsId
+	 *            identifier of the ObjectStore
+	 * @param from
+	 *            the minimum date of the object
+	 * @param until
+	 *            the maximum date of the object
+	 * @return a ResultSet EPR. Each element of the result set contains the objIdentifier of a record and its URL for retrieve the
+	 *         inputstream of the file.
+	 * @throws ObjectStoreServiceException
+	 *             the object store service exception
+	 */
+	@WebMethod(operationName = "deliverObjects", action = "deliverObjects")
+	public W3CEndpointReference deliverObjects(@WebParam(name = "obsId") String obsId, @WebParam(name = "from") Long from, @WebParam(name = "until") Long until)
+			throws ObjectStoreServiceException;
+
+	/**
+	 * Returns ResultSet EPR for delivered ObjectStore records.
+	 * <p>
+	 * Please check service implementations for details on the expected format of the records in the result set epr.
+	 * </p>
+	 *
+	 * @param obsId
+	 *            identifier of the ObjectStore
+	 * @param eprId
+	 *            id of a ResultSet EPR with the identifiers of the interesting objects. Each element of the result set contains the
+	 *            objIdentifier of a record
+	 * @return a ResultSet EPR. Each element of the result set contains the objIdentifier of a record and its URL for retrieve the
+	 *         inputstream of the file.
+	 * @throws ObjectStoreServiceException
+	 *             the object store service exception
+	 */
+	@WebMethod(operationName = "deliverObjectsByIds", action = "deliverObjectsByIds")
+	public W3CEndpointReference deliverObjectsByIds(@WebParam(name = "obsId") String obsId, @WebParam(name = "eprId") W3CEndpointReference eprId)
+			throws ObjectStoreServiceException;
+
+	/**
+	 * Returns an URL to retrieve the ObjectStore record.
+	 *
+	 * @param obsId
+	 *            identifier of the ObjectStore
+	 * @param objectId
+	 *            the id of the object
+	 * @return the URL for retrieve the record
+	 * @throws ObjectStoreServiceException
+	 *             the object store service exception
+	 */
+	@WebMethod(operationName = "deliverObject", action = "deliverObject")
+	public String deliverRecord(@WebParam(name = "obsId") String obsId, @WebParam(name = "objectId") String objectId) throws ObjectStoreServiceException;
+
+	/**
+	 * Feed the object in the objectStore
+	 *
+	 * @param obsId
+	 *            identifier of the ObjectStore
+	 * @param objectMetadata
+	 *            the String serialized of the JSON object ObjectStoreFile
+	 * @throws ObjectStoreServiceException
+	 */
+
+	@WebMethod(operationName = "feedObject", action = "feedObject")
+	public void feedObject(@WebParam(name = "obsId") String obsId, @WebParam(name = "objectMetadata") String objectMetadata) throws ObjectStoreServiceException;
+
+	/**
+	 * Returns list of all stored indices.
+	 *
+	 * @return list of all stored indices
+	 */
+	@WebMethod(operationName = "getListOfObjectStores", action = "getListOfObjectStores")
+	public List<String> getListOfObjectStores();
+
+	/**
+	 * Gets the size of the objectStore ID.
+	 *
+	 * @param obsId
+	 *            the obs id
+	 * @return the size
+	 */
+	@WebMethod(operationName = "getSize", action = "getSize")
+	public int getSize(@WebParam(name = "obsId") String obsId) throws ObjectStoreServiceException;
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/ObjectStoreServiceException.java

@@ -0,0 +1,45 @@
+package eu.dnetlib.data.objectstore.rmi;
+
+import eu.dnetlib.common.rmi.RMIException;
+
+/**
+ * The Class ObjectStoreServiceException.
+ */
+public class ObjectStoreServiceException extends RMIException {
+
+	/** The Constant serialVersionUID. */
+	private static final long serialVersionUID = -7073846285219543315L;
+
+	/**
+	 * Instantiates a new object store service exception.
+	 *
+	 * @param string
+	 *            the string
+	 */
+	public ObjectStoreServiceException(final String string) {
+		super(string);
+	}
+
+	/**
+	 * Instantiates a new object store service exception.
+	 *
+	 * @param string
+	 *            the string
+	 * @param exception
+	 *            the exception
+	 */
+	public ObjectStoreServiceException(final String string, final Throwable exception) {
+		super(string, exception);
+	}
+
+	/**
+	 * Instantiates a new object store service exception.
+	 *
+	 * @param exception
+	 *            the exception
+	 */
+	public ObjectStoreServiceException(final Throwable exception) {
+		super(exception);
+	}
+
+}

dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/Protocols.java

@@ -0,0 +1,13 @@
+package eu.dnetlib.data.objectstore.rmi;
+
+/**
+  * The Enum Protocols.
+  */
+ public enum Protocols {
+	 None,
+	 HTTP, 
+	 HTTPS,
+	 FTP,
+	 File_System,
+	 FTPS
+}

dnet-data-services/src/main/resources/eu/dnetlib/applicationContext-dnet-objectstore-rmi.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns="http://www.springframework.org/schema/beans"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:jaxws="http://cxf.apache.org/jaxws"
+	xmlns:sec="http://cxf.apache.org/configuration/security" xmlns:wsa="http://cxf.apache.org/ws/addressing"
+	xmlns:p="http://www.springframework.org/schema/p" xmlns:http="http://cxf.apache.org/transports/http/configuration"
+	xmlns:t="http://dnetlib.eu/springbeans/t" xmlns:template="http://dnetlib.eu/springbeans/template"
+	xmlns:util="http://www.springframework.org/schema/util"
+	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+                                    http://cxf.apache.org/ws/addressing http://cxf.apache.org/schemas/ws-addr-conf.xsd
+                                    http://cxf.apache.org/configuration/security http://cxf.apache.org/schemas/configuration/security.xsd
+                                    http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd
+                            http://cxf.apache.org/jaxws http://cxf.apache.org/schemas/jaxws.xsd
+                            http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.0.xsd
+                            http://dnetlib.eu/springbeans/template http://dnetlib.eu/springbeans/template.xsd">
+
+
+
+
+</beans>

dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/applicationContext-dnet-modular-objectstore-service.properties

@@ -0,0 +1,2 @@
+services.objectstore.dao=gridFSObjectstoreDao
+services.objectstore.rsfactory.pagesize=20

dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/applicationContext-dnet-modular-objectstore-service.xml

@@ -0,0 +1,111 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns:jaxws="http://cxf.apache.org/jaxws"
+       xmlns:p="http://www.springframework.org/schema/p"
+       xmlns:http="http://cxf.apache.org/transports/http/configuration" xmlns:t="http://dnetlib.eu/springbeans/t"
+       xmlns:template="http://dnetlib.eu/springbeans/template"
+       xmlns="http://www.springframework.org/schema/beans"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+
+
+                                    http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd
+                            http://cxf.apache.org/jaxws http://cxf.apache.org/schemas/jaxws.xsd
+
+                            http://dnetlib.eu/springbeans/template http://dnetlib.eu/springbeans/template.xsd">
+
+
+    <bean id="objectStoreService"
+          class="eu.dnetlib.data.objectstore.modular.ModularObjectStoreService"
+          init-method="start" destroy-method="stop"
+          p:feeder-ref="objectstoreFeeder"
+          p:objectStoreDeliver-ref="objectstoreDeliver"
+          p:notificationHandler-ref="objectstoreNotificationHandler"
+            />
+
+    <bean id="objectstoreFeeder"
+          class="eu.dnetlib.data.objectstore.modular.ModularObjectStoreFeeder"
+          p:dao-ref="${services.objectstore.dao}"
+          p:resultSetClientFactory-ref="objectstoreResultSetClientFactory"
+          p:serviceLocator-ref="uniqueServiceLocator"
+            />
+
+    <bean id="objectstoreDeliver"
+          class="eu.dnetlib.data.objectstore.modular.ModularObjectStoreDeliver"
+          p:dao-ref="${services.objectstore.dao}"
+          p:resultSetClientFactory-ref="objectstoreResultSetClientFactory"
+
+            />
+
+
+    <bean id="objectstoreResultSetClientFactory" parent="resultSetClientFactory"
+          p:pageSize="${services.objectstore.rsfactory.pagesize}"/>
+
+
+    <bean id="objectstoreProfileCreator" class="eu.dnetlib.data.objectstore.modular.ObjectStoreProfileCreator"
+          p:serviceLocator-ref="uniqueServiceLocator" p:objectstoreDsTemplate-ref="objectstoreDsTemplate"
+          p:endpoint-ref="objectStoreServiceEndpoint" p:eprBuilder-ref="jaxwsEndpointReferenceBuilder"/>
+
+    <bean id="objectstoreNotificationHandler"
+          class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerExecutorNotificationHandler"
+          p:blackboardExecutor-ref="objectstoreBlackboardExecutor"/>
+
+    <bean id="objectstoreBlackboardExecutor"
+          class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerActionExecutor"
+          p:blackboardHandler-ref="blackboardHandler"
+          p:actionType="eu.dnetlib.data.objectstore.modular.ObjectStoreActions"
+          p:incomplete="false">
+        <property name="actionMap">
+            <map>
+                <entry key="CREATE">
+                    <bean class="eu.dnetlib.data.objectstore.modular.CreateObjectStoreAction"
+                          p:profileCreator-ref="objectstoreProfileCreator" p:dao-ref="${services.objectstore.dao}"/>
+                </entry>
+                <entry key="DELETE">
+                    <bean class="eu.dnetlib.data.objectstore.modular.DeleteObjectStoreAction"
+                          p:profileCreator-ref="objectstoreProfileCreator" p:dao-ref="${services.objectstore.dao}"/>
+                </entry>
+                <entry key="DROP_CONTENT">
+                    <bean class="eu.dnetlib.data.objectstore.modular.DropContentObjectStoreAction"
+                          p:dao-ref="${services.objectstore.dao}"/>
+                </entry>
+
+                <entry key="FEED">
+                    <bean class="eu.dnetlib.data.objectstore.modular.FeedObjectStoreAction"
+                          p:profileCreator-ref="objectstoreProfileCreator"
+                          p:storeFeeder-ref="objectstoreFeeder"
+                          p:dao-ref="${services.objectstore.dao}"/>
+                </entry>
+
+                <entry key="FEEDOBJECT">
+                    <bean class="eu.dnetlib.data.objectstore.modular.FeedCompleteObjectAction"
+                          p:storeFeeder-ref="objectstoreFeeder"
+                          p:dao-ref="${services.objectstore.dao}"/>
+                </entry>
+            </map>
+        </property>
+    </bean>
+
+
+    <bean id="objectStoreLocator" class="eu.dnetlib.enabling.tools.StaticServiceLocator"
+          p:service-ref="objectStoreService"/>
+
+
+    <bean id="objectstoreDsTemplate"
+          class="eu.dnetlib.springutils.stringtemplate.StringTemplateFactory"
+          p:template="classpath:/eu/dnetlib/data/objectstore/objectstoreds-template.xml"
+          scope="prototype"/>
+
+
+    <!-- endpoints -->
+    <jaxws:endpoint id="objectStoreServiceEndpoint"
+                    implementor="#objectStoreService" implementorClass="eu.dnetlib.data.objectstore.rmi.ObjectStoreService"
+                    address="/objectStore"/>
+
+    <template:instance name="serviceRegistrationManager"
+                       t:serviceRegistrationManagerClass="eu.dnetlib.enabling.tools.registration.ValidatingServiceRegistrationManagerImpl"
+                       t:name="objectStoreServiceRegistrationManager" t:service="objectStoreService"
+                       t:endpoint="objectStoreServiceEndpoint" t:jobScheduler="jobScheduler"
+                       t:serviceRegistrator="blackboardServiceRegistrator"/>
+
+
+</beans>

dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/filesystem/applicationContext-filesystem-objectstore.properties

@@ -0,0 +1,8 @@
+services.objectStore.mongodb.host=localhost
+services.objectStore.mongodb.port=27017
+services.objectStore.mongodb.db=objectStore
+services.objectStore.mongodb.connectionsPerHost=20
+services.objectStore.mongodb.upsert=true
+services.objectstore.mongodb.integritycheck.cron=0 0 2 1/1 * ? *
+services.objectStore.RESTURI=http://${container.hostname}:${container.port}/${container.context}/mvc/objectStore/retrieve.do
+services.objectstore.dao=fSObjectstoreDao

dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/filesystem/applicationContext-filesystem-objectstore.xml

@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns:p="http://www.springframework.org/schema/p" xmlns:t="http://dnetlib.eu/springbeans/t"
+       xmlns="http://www.springframework.org/schema/beans"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
+
+	<bean id="fSObjectstoreDao"
+	      class="eu.dnetlib.data.objectstore.filesystem.FileSystemObjectStoreDao"
+	      p:db-ref="objectstoreMongoDB" p:objectStoreRESTURI-ref="restURIString"
+	      p:upsert="${services.objectStore.mongodb.upsert}"/>
+
+	<bean id="restURIString" class="java.lang.String">
+		<constructor-arg value="${services.objectStore.RESTURI}"/>
+	</bean>
+
+	<bean id="objectStoreFileSystemUtility" class="eu.dnetlib.data.objectstore.filesystem.FileSystemUtility"/>
+
+	<bean id="objectStoreMongoClient" class="com.mongodb.MongoClient">
+		<constructor-arg index="0" type="com.mongodb.ServerAddress">
+			<bean class="com.mongodb.ServerAddress">
+				<constructor-arg index="0" value="${services.objectStore.mongodb.host}"/>
+				<constructor-arg index="1" value="${services.objectStore.mongodb.port}"/>
+			</bean>
+		</constructor-arg>
+		<constructor-arg index="1" type="com.mongodb.MongoClientOptions">
+			<bean class="eu.dnetlib.data.objectstore.filesystem.MongoFSOptionsFactory"
+			      p:connectionsPerHost="${services.objectStore.mongodb.connectionsPerHost}"/>
+		</constructor-arg>
+	</bean>
+
+	<bean id="objectstoreMongoDB" factory-bean="objectStoreMongoClient" factory-method="getDatabase">
+		<constructor-arg index="0" type="java.lang.String" value="${services.objectStore.mongodb.db}"/>
+	</bean>
+
+	<bean id="objectStoreIntegrityChecker" class="eu.dnetlib.data.objectstore.filesystem.IndexIntegrityChecker"/>
+
+	<bean t:id="objectStoreJobSchedulerAccessor"
+	      class="org.springframework.scheduling.quartz.SchedulerAccessorBean"
+	      p:scheduler-ref="jobScheduler">
+		<property name="triggers">
+			<list>
+				<bean class="org.springframework.scheduling.quartz.CronTriggerFactoryBean">
+					<property name="jobDetail">
+						<bean class="org.springframework.scheduling.quartz.MethodInvokingJobDetailFactoryBean"
+						      p:targetObject-ref="objectStoreIntegrityChecker" p:targetMethod="check"/>
+					</property>
+					<property name="cronExpression" value="${services.objectstore.mongodb.integritycheck.cron}"/>
+				</bean>
+			</list>
+		</property>
+	</bean>
+
+</beans>

dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/inspector/webContext-objectstore-inspector.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns="http://www.springframework.org/schema/beans"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:p="http://www.springframework.org/schema/p"
+	xmlns:util="http://www.springframework.org/schema/util"
+	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
+                http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.5.xsd">
+
+	<bean id="objectstoreInspectorGroup"
+		class="eu.dnetlib.enabling.inspector.StaticEntryPointDescriptorGroup"
+		p:name="objectstore">
+		<property name="descriptors">
+			<list>
+				<bean class="eu.dnetlib.enabling.inspector.StaticEntryPointDescriptor"
+					p:name="objectstores" p:relativeUrl="objectstores.do" />
+			</list>
+		</property>
+	</bean>
+
+</beans>
+

dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/objectstoreds-template.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<RESOURCE_PROFILE>
+    <HEADER>
+        <RESOURCE_IDENTIFIER value=""/>
+        <RESOURCE_TYPE value="ObjectStoreDSResourceType"/>
+        <RESOURCE_KIND value="ObjectStoreDSResources"/>
+        <RESOURCE_URI value="$serviceUri$?wsdl"/>
+        <DATE_OF_CREATION value=""/>
+    </HEADER>
+    <BODY>
+        <CONFIGURATION>
+            <OBJECTSTORE_INTERPRETATION>$interpretation$</OBJECTSTORE_INTERPRETATION>
+        </CONFIGURATION>
+        <STATUS>
+            <PENULTIMATE_STORAGE_DATE></PENULTIMATE_STORAGE_DATE>
+            <LAST_STORAGE_DATE></LAST_STORAGE_DATE>
+            <FETCHING_FREQUENCY/>
+            <COUNT_STORE>0</COUNT_STORE>
+            <STORE_SIZE>0</STORE_SIZE>
+            <STATISTICS_FIELDS/>
+        </STATUS>
+        <SECURITY_PARAMETERS></SECURITY_PARAMETERS>
+    </BODY>
+</RESOURCE_PROFILE>

dnet-data-services/src/test/java/eu/dnetlib/data/objectstore/filesystem/ConfigurationTestConfig.java

@@ -0,0 +1,51 @@
+package eu.dnetlib.data.objectstore.filesystem;
+
+import java.net.UnknownHostException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.MongoDatabase;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class ConfigurationTestConfig.
+ */
+@Configuration
+public class ConfigurationTestConfig {
+
+	/**
+	 * Mongo db.
+	 *
+	 * @return the db
+	 * @throws UnknownHostException the unknown host exception
+	 */
+	@Bean
+	public MongoDatabase objectstoreMongoDB() throws UnknownHostException {
+		MongoClient mongoClient = new MongoClient("localhost");
+		return mongoClient.getDatabase("objectStoreTest");
+
+	}
+
+
+	@Bean
+	public FileSystemObjectStoreDao fsObjectStoreDAO() {
+		FileSystemObjectStoreDao fileSystemObjectStoreDao = new FileSystemObjectStoreDao();
+
+		fileSystemObjectStoreDao.setObjectStoreRESTURI("http://www.objectstore.com");
+		return fileSystemObjectStoreDao;
+
+	}
+
+	/**
+	 * Fs utility.
+	 *
+	 * @return the file system utility
+	 */
+	@Bean
+	public FileSystemUtility fsUtility () {
+		return new FileSystemUtility();
+	}
+
+
+}

dnet-data-services/src/test/java/eu/dnetlib/data/objectstore/filesystem/InputIterator.java

@@ -0,0 +1,80 @@
+/**
+ *
+ */
+package eu.dnetlib.data.objectstore.filesystem;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import eu.dnetlib.data.objectstore.modular.ObjectStoreRecord;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
+import eu.dnetlib.data.objectstore.rmi.Protocols;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
+
+/**
+ * @author sandro
+ *
+ */
+public class InputIterator implements Iterable<ObjectStoreRecord>, Iterator<ObjectStoreRecord> {
+
+	private int counter = 0;
+	private String pdfResourcePath = "test.pdf";
+	private Resource testPDF = new ClassPathResource(pdfResourcePath);
+
+	/**
+	 * {@inheritDoc}
+	 * @see java.util.Iterator#hasNext()
+	 */
+	@Override
+	public boolean hasNext() {
+
+		return counter<100;
+	}
+
+	/**
+	 * {@inheritDoc}
+	 * @see java.util.Iterator#next()
+	 */
+	@Override
+	public ObjectStoreRecord next() {
+		try {
+			counter ++;
+			ObjectStoreRecord record = new ObjectStoreRecord();
+			ObjectStoreFile fileMetadata = new ObjectStoreFile();
+			fileMetadata.setAccessProtocol(Protocols.File_System);
+
+			fileMetadata.setDownloadedURL("file://" + pdfResourcePath);
+			fileMetadata.setURI("file://" + pdfResourcePath);
+			fileMetadata.setObjectID("Oggetto_"+counter);
+			fileMetadata.setMimeType("application/pdf");
+			System.out.println("Aggiungo Elemento "+counter);
+			record.setInputStream(testPDF.getInputStream());
+			record.setFileMetadata(fileMetadata );
+			return record;
+		} catch (IOException e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+	/**
+	 * {@inheritDoc}
+	 * @see java.util.Iterator#remove()
+	 */
+	@Override
+	public void remove() {
+		// TODO Auto-generated method stub
+
+	}
+
+	/**
+	 * {@inheritDoc}
+	 * @see java.lang.Iterable#iterator()
+	 */
+	@Override
+	public Iterator<ObjectStoreRecord> iterator() {
+		counter =0;
+		return this;
+	}
+
+}

dnet-data-services/src/test/java/eu/dnetlib/data/objectstore/filesystem/ObjectStoreServiceDAOTest.java

@@ -0,0 +1,117 @@
+package eu.dnetlib.data.objectstore.filesystem;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+import com.mongodb.client.MongoDatabase;
+import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.enabling.resultset.ResultSetListener;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * TODO: Test class set to ignored because it requires a mongo server running on localhost. Tests with mocks should be prepared.
+ * @author sandro
+ *
+ */
+@Ignore
+@RunWith(SpringJUnit4ClassRunner.class)
+@ContextConfiguration(classes = ConfigurationTestConfig.class)
+public class ObjectStoreServiceDAOTest {
+	private static final Log log = LogFactory.getLog(ObjectStoreServiceDAOTest.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+	@Autowired
+	private FileSystemObjectStoreDao objectStoreDAO;
+
+	@Autowired
+	private MongoDatabase objectstoreMongoDB;
+
+	private String interpretation = "interp";
+	private String basePath = "/tmp/basePath";
+	//must be 36 chars
+	private String obsID = "test56789111315171921232527293133350";
+	private String pdfPath = "./test.pdf";
+
+	@Before
+	public void setup() throws IOException, ObjectStoreServiceException {
+		Path baseDirPath = FileSystems.getDefault().getPath(basePath);
+		if (!Files.exists(baseDirPath)) {
+			Files.createDirectory(baseDirPath);
+		}
+		objectStoreDAO.createObjectStore(obsID + "_T2JqZWN0U3RvcmVEU1Jlc291cmNlcy9PYmplY3RTdG9yZURTUmVzb3VyY2VUeXBl", interpretation, basePath);
+	}
+
+	@After
+	public void tearDown() throws IOException, ObjectStoreServiceException {
+		objectstoreMongoDB.drop();
+		FileUtils.deleteDirectory(new File(basePath));
+	}
+
+	@Test
+	public void testList() {
+		final List<String> objectStores = this.objectStoreDAO.listObjectStores();
+		for (String o : objectStores)
+			System.out.println(o);
+		assertTrue(objectStores.contains(obsID + "_T2JqZWN0U3RvcmVEU1Jlc291cmNlcy9PYmplY3RTdG9yZURTUmVzb3VyY2VUeXBl"));
+	}
+
+	@Test
+	public void testGet() throws ObjectStoreServiceException {
+		ObjectStore o = objectStoreDAO.getObjectStore(obsID);
+		System.out.println(o.toString());
+		assertNotNull(o);
+	}
+
+	@Test
+	public void testFeed() throws ObjectStoreServiceException, FileNotFoundException {
+		objectStoreDAO.getObjectStore(obsID).feed(new InputIterator(),true);
+	}
+
+	@Test
+	public void testDeliverObject() throws ObjectStoreServiceException {
+		objectStoreDAO.getObjectStore(obsID).feed(new InputIterator(), true);
+		final ObjectStoreFile obj2 = objectStoreDAO.getObjectStore(obsID).deliverObject("Oggetto_2");
+		System.out.println(obj2.toJSON());
+		assertNotNull(obj2);
+	}
+
+	@Test
+	public void testGetStream() throws Exception {
+		objectStoreDAO.getObjectStore(obsID).feed(new InputIterator(), true);
+		ResultSetListener rs = objectStoreDAO.getObjectStore(obsID).deliver(0L, System.currentTimeMillis());
+		for (int i=1; i< (rs.getSize()/10); ) {
+
+			int from  = i;
+			int to = Math.max(rs.getSize(), i*10);
+
+			List<String> data = rs.getResult(from, to);
+
+			for (String s: data) {
+				System.out.println(s);
+			}
+
+			i= to ;
+		}
+	}
+
+
+}