addFileSystemObjectStoreResultSetListener method. When
+ * the fileSystemObjectStoreResultSet event occurs, that object's appropriate
+ * method is invoked.
+ *
+ * @author sandro
+ */
+public class FileSystemObjectStoreResultSetListener implements ResultSetListener, ResultSetAware {
+
+
+ /** The Constant log. */
+ private static final Log log = LogFactory.getLog(FileSystemObjectStoreResultSetListener.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+
+ /** The from date. */
+ private Long fromDate;
+
+ /** The until date. */
+ private Long untilDate;
+
+ /** The records. */
+ private List records;
+
+ /** The object store id. */
+ private String objectStoreID;
+
+
+ /** The mongo collection. */
+ private MongoCollection mongoCollection;
+
+ /** The base uri. */
+ private String baseURI;
+
+ /**
+ * The base path
+ */
+ private String basePath;
+
+ /** The current size. */
+ private int currentSize = -1;
+
+ /** The current cursor. */
+ private MongoCursor currentCursor;
+
+ /** The cursor position. */
+ private long cursorPosition;
+
+ /**
+ * {@inheritDoc}
+ * @see eu.dnetlib.enabling.resultset.TypedResultSetListener#getResult(int, int)
+ */
+ @Override
+ public List getResult(final int from, final int to) {
+ if (log.isDebugEnabled()) {
+ log.debug(String.format("ObjectStoreId :%s, from: %d, to: %d", objectStoreID, from, to));
+ }
+ if (records != null) {
+ List ids = Lists.newArrayList();
+ for (int i = from-1; i < Math.min(records.size(),to); i++) {
+ ids.add(records.get(i));
+ }
+ Bson q = Filters.in("id", ids);
+ FindIterable res = getMongoCollection().find(q);
+ return MappedCollection.listMap(res, ObjectStoreFileUtility.asJSON(getBaseURI(), getObjectStoreID(), getBasePath()));
+ } else if ((fromDate != null) && (untilDate != null)) {
+ if ((currentCursor == null) || (cursorPosition > from)) {
+ createCurrentCursor();
+ }
+ while (cursorPosition < from) {
+ currentCursor.next();
+ cursorPosition++;
+ }
+ List result = Lists.newArrayList();
+ for (int i = from; i <= to; i++) {
+ if (currentCursor.hasNext()) {
+ result.add(currentCursor.next());
+ cursorPosition++;
+ }
+ }
+ return MappedCollection.listMap(result, ObjectStoreFileUtility.asJSON(getBaseURI(), getObjectStoreID(), getBasePath()));
+ }
+
+ throw new IllegalArgumentException("Missing parameters on Delivery must provide either from, to, or ObjectStoreIDs");
+ }
+
+ /**
+ * Creates the current cursor.
+ */
+ private void createCurrentCursor() {
+ Bson timestampQuery = Filters.and(Filters.gt("timestamp", fromDate.doubleValue()), Filters.lt("timestamp", untilDate.doubleValue()));
+ if (currentCursor != null) {
+ currentCursor.close();
+ }
+ currentCursor = getMongoCollection().find(timestampQuery).sort(Sorts.orderBy(Filters.eq("_id", 1))).iterator();
+ cursorPosition = 1;
+
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see eu.dnetlib.enabling.resultset.TypedResultSetListener#getSize()
+ */
+ @Override
+ public int getSize() {
+ if (currentSize == -1) {
+ currentSize = calculateSize();
+ }
+ return Math.max(0, currentSize - 1);
+ }
+
+ /**
+ * Calculate size.
+ *
+ * @return the int
+ */
+ private int calculateSize() {
+ if (records != null) {
+ Bson query = Filters.in("id", records);
+ return (int) getMongoCollection().count(query);
+ } else if ((fromDate != null) && (untilDate != null)) {
+ Bson timestampQuery = Filters.and(Filters.gt("timestamp", fromDate.doubleValue()), Filters.lt("timestamp", untilDate.doubleValue()));
+ return (int) getMongoCollection().count(timestampQuery);
+ }
+ return 0;
+ }
+
+
+ /**
+ * {@inheritDoc}
+ * @see eu.dnetlib.enabling.resultset.ResultSetAware#setResultSet(eu.dnetlib.enabling.resultset.ResultSet)
+ */
+ @Override
+ public void setResultSet(final ResultSet resultSet) {
+ resultSet.close();
+ }
+
+
+ /**
+ * Gets the from date.
+ *
+ * @return the fromDate
+ */
+ public Long getFromDate() {
+ return fromDate;
+ }
+
+
+ /**
+ * Sets the from date.
+ *
+ * @param fromDate the fromDate to set
+ */
+ public FileSystemObjectStoreResultSetListener setFromDate(final Long fromDate) {
+ this.fromDate = fromDate;
+ return this;
+ }
+
+
+ /**
+ * Gets the until date.
+ *
+ * @return the untilDate
+ */
+ public Long getUntilDate() {
+ return untilDate;
+ }
+
+
+ /**
+ * Sets the until date.
+ *
+ * @param untilDate the untilDate to set
+ */
+ public FileSystemObjectStoreResultSetListener setUntilDate(final Long untilDate) {
+ this.untilDate = untilDate;
+ return this;
+ }
+
+
+ /**
+ * Gets the records.
+ *
+ * @return the records
+ */
+ public List getRecords() {
+ return records;
+ }
+
+
+ /**
+ * Sets the records.
+ *
+ * @param records the records to set
+ */
+ public void setRecords(final List records) {
+ this.records = records;
+ }
+
+
+ /**
+ * Gets the object store id.
+ *
+ * @return the objectStoreID
+ */
+ public String getObjectStoreID() {
+ return objectStoreID;
+ }
+
+
+ /**
+ * Sets the object store id.
+ *
+ * @param objectStoreID the objectStoreID to set
+ */
+ public void setObjectStoreID(final String objectStoreID) {
+ this.objectStoreID = objectStoreID;
+ }
+
+
+
+
+
+ /**
+ * Gets the base uri.
+ *
+ * @return the baseURI
+ */
+ public String getBaseURI() {
+ return baseURI;
+ }
+
+
+ /**
+ * Sets the base uri.
+ *
+ * @param baseURI the baseURI to set
+ */
+ public void setBaseURI(final String baseURI) {
+ this.baseURI = baseURI;
+ }
+
+
+ /**
+ * Gets the current size.
+ *
+ * @return the currentSize
+ */
+ public int getCurrentSize() {
+ return currentSize;
+ }
+
+
+ /**
+ * Sets the current size.
+ *
+ * @param currentSize the currentSize to set
+ */
+ public void setCurrentSize(final int currentSize) {
+ this.currentSize = currentSize;
+ }
+
+
+ /**
+ * Gets the current cursor.
+ *
+ * @return the currentCursor
+ */
+ public MongoCursor getCurrentCursor() {
+ return currentCursor;
+ }
+
+
+ /**
+ * Sets the current cursor.
+ *
+ * @param currentCursor the currentCursor to set
+ */
+ public void setCurrentCursor(final MongoCursor currentCursor) {
+ this.currentCursor = currentCursor;
+ }
+
+
+ /**
+ * Gets the cursor position.
+ *
+ * @return the cursorPosition
+ */
+ public long getCursorPosition() {
+ return cursorPosition;
+ }
+
+
+ /**
+ * Sets the cursor position.
+ *
+ * @param cursorPosition the cursorPosition to set
+ */
+ public void setCursorPosition(final long cursorPosition) {
+ this.cursorPosition = cursorPosition;
+ }
+
+ /**
+ * Gets the mongo collection.
+ *
+ * @return the mongo collection
+ */
+ public MongoCollection getMongoCollection() {
+ return mongoCollection;
+ }
+
+ /**
+ * Sets the mongo collection.
+ *
+ * @param mongoCollection the new mongo collection
+ */
+ public void setMongoCollection(final MongoCollection mongoCollection) {
+ this.mongoCollection = mongoCollection;
+ }
+
+ public String getBasePath() {
+ return basePath;
+ }
+
+ public void setBasePath(final String basePath) {
+ this.basePath = basePath;
+ }
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/FileSystemUtility.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/FileSystemUtility.java
new file mode 100644
index 0000000..0b3000a
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/FileSystemUtility.java
@@ -0,0 +1,78 @@
+package eu.dnetlib.data.objectstore.filesystem;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.*;
+import java.nio.file.attribute.BasicFileAttributes;
+
+import eu.dnetlib.miscutils.collections.Pair;
+import eu.dnetlib.miscutils.functional.xml.DnetXsltFunctions;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * The Class FileSystemUtility.
+ *
+ * @author sandro
+ */
+public class FileSystemUtility {
+
+ private static final Log log = LogFactory.getLog(FileSystemUtility.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+ public static Pair saveAndGenerateMD5(final InputStream inputStream, final Path filePath) {
+ if(inputStream==null) return null;
+
+ String md5 = null;
+ long size = 0;
+ try {
+ Files.copy(inputStream, filePath);
+
+ FileInputStream fis = new FileInputStream(filePath.toFile());
+ md5 = org.apache.commons.codec.digest.DigestUtils.md5Hex(fis);
+ fis.close();
+ size = Files.size(filePath);
+
+ } catch (IOException e1) {
+ log.error(e1);
+ }
+
+ return new Pair(md5, size);
+ }
+
+ /**
+ * Delete folder recursive.
+ *
+ * @param path the path
+ * @return true, if successful
+ * @throws IOException Signals that an I/O exception has occurred.
+ */
+ public static boolean deleteFolderRecursive(final Path path) throws IOException {
+
+ Files.walkFileTree(path, new SimpleFileVisitor() {
+ @Override
+ public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
+ Files.delete(file);
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult postVisitDirectory(final Path dir, final IOException exc) throws IOException {
+ Files.delete(dir);
+ return FileVisitResult.CONTINUE;
+ }
+ });
+ return true;
+ }
+
+ public static Path objectStoreFilePath(final String basePath, final String objectStoreId, final String objectIdentifier) {
+ final Path baseDirPath = FileSystems.getDefault().getPath(basePath).resolve(objectStoreId);
+ final String md5id = DnetXsltFunctions.md5(objectIdentifier);
+ final String firstLevel = StringUtils.substring(md5id, 0, 2);
+ final String secondLevel = StringUtils.substring(md5id, 2, 4);
+ final String fileName = StringUtils.substring(md5id, 4) + ".obj";
+ return baseDirPath.resolve(firstLevel).resolve(secondLevel).resolve(fileName);
+ }
+
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/IndexIntegrityChecker.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/IndexIntegrityChecker.java
new file mode 100644
index 0000000..1f9a272
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/IndexIntegrityChecker.java
@@ -0,0 +1,55 @@
+package eu.dnetlib.data.objectstore.filesystem;
+
+import com.mongodb.BasicDBObject;
+import com.mongodb.DBObject;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.model.IndexOptions;
+import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import static eu.dnetlib.data.objectstore.filesystem.FileSystemObjectStoreDao.OBJECTSTORE_ID_FIELD;
+import static eu.dnetlib.data.objectstore.filesystem.FileSystemObjectStoreDao.OBJECTSTORE_METADATA_NAME_FIELD;
+
+/**
+ * Created by claudio on 15/09/16.
+ */
+public class IndexIntegrityChecker {
+
+ private static final Log log = LogFactory.getLog(IndexIntegrityChecker.class);
+
+ @Autowired
+ private ObjectStoreDao dao;
+
+ public void check() {
+ checkIndexes();
+ }
+
+ private void checkIndexes() {
+ log.info("objectStore indexes integrity start");
+
+ final MongoDatabase db = ((FileSystemObjectStoreDao) dao).getDb();
+ final IndexOptions bg = new IndexOptions().background(true);
+
+ for (String objectStoreId : dao.listObjectStores()) {
+ final String id = StringUtils.substringBefore(objectStoreId, "_");
+ final MongoCollection objectStore = db.getCollection(id, DBObject.class);
+ if (log.isDebugEnabled()) {
+ log.debug(String.format("creating index (id, timestamp) on objectStore %s", id));
+ }
+ objectStore.createIndex(new BasicDBObject("id", 1), bg);
+ objectStore.createIndex(new BasicDBObject("timestamp", 1), bg);
+ }
+ if (log.isDebugEnabled()) {
+ log.debug(String.format("creating index (%s) on %s", OBJECTSTORE_ID_FIELD, OBJECTSTORE_METADATA_NAME_FIELD));
+ }
+ final MongoCollection metadata = db.getCollection(OBJECTSTORE_METADATA_NAME_FIELD, DBObject.class);
+ metadata.createIndex(new BasicDBObject(OBJECTSTORE_ID_FIELD, 1), bg);
+
+ log.info("objectStore indexes integrity completed");
+ }
+
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/ModularObjectStoreRESTService.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/ModularObjectStoreRESTService.java
new file mode 100644
index 0000000..3b13bc4
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/ModularObjectStoreRESTService.java
@@ -0,0 +1,79 @@
+package eu.dnetlib.data.objectstore.filesystem;
+
+import java.io.*;
+import java.net.URLEncoder;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import javax.servlet.http.HttpServletResponse;
+
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.miscutils.datetime.HumanTime;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+
+/**
+ * The Class ModularObjectStoreRESTService implement the controller REST of the object Store.
+ */
+@Controller
+public class ModularObjectStoreRESTService {
+
+ private static final Log log = LogFactory.getLog(ModularObjectStoreRESTService.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+ public static String retrieveURL(final String baseURI, final String basePath, final String objectStoreId, final String objectId)
+ throws UnsupportedEncodingException {
+ final StringBuilder sb = new StringBuilder(baseURI)
+ .append("?objectStore=" + encode(objectStoreId))
+ .append("&objectId=" + encode(objectId))
+ .append("&basePath=" + encode(basePath));
+ return sb.toString();
+ }
+
+ private static String encode(final String s) throws UnsupportedEncodingException {
+ return URLEncoder.encode(s, "UTF-8");
+ }
+
+ /**
+ *
+ * @param res
+ * @param basePath
+ * @param objectStoreId
+ * @param objectId
+ * @throws IOException
+ * @throws ObjectStoreServiceException
+ */
+ @RequestMapping(value = "/**/objectStore/retrieve.do")
+ public void retrieve(final HttpServletResponse res,
+ @RequestParam(value = "basePath", required = true) final String basePath,
+ @RequestParam(value = "objectStore", required = true) final String objectStoreId,
+ @RequestParam(value = "objectId", required = true) final String objectId) throws IOException, ObjectStoreServiceException {
+
+ final long start = System.currentTimeMillis();
+ final Path path = FileSystemUtility.objectStoreFilePath(basePath, objectStoreId, objectId);
+
+ if (!Files.exists(path) || !Files.isReadable(path)) {
+ final String msg = String.format("Object with identifier: %s not found the in %s", objectId, path);
+ res.sendError(HttpServletResponse.SC_NOT_FOUND, msg);
+ log.warn(msg);
+ } else {
+ try (final InputStream is = new BufferedInputStream(new FileInputStream(path.toFile()))) {
+
+ final long size = Files.size(path);
+
+ res.setHeader("Content-Length", String.valueOf(size));
+ IOUtils.copyLarge(is, res.getOutputStream());
+
+ if (log.isDebugEnabled()) {
+ log.debug(String.format("retrive.do completed in %s, objId: %s", HumanTime.exactly(System.currentTimeMillis() - start), objectId));
+ }
+ } catch (IOException e) {
+ final String msg = "unable to close input Stream";
+ res.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, msg);
+ log.error(msg, e);
+ }
+ }
+ }
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/MongoFSOptionsFactory.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/MongoFSOptionsFactory.java
new file mode 100644
index 0000000..1855627
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/MongoFSOptionsFactory.java
@@ -0,0 +1,34 @@
+package eu.dnetlib.data.objectstore.filesystem;
+
+import com.mongodb.MongoClientOptions;
+import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.FactoryBean;
+
+public class MongoFSOptionsFactory implements FactoryBean {
+
+ private int connectionsPerHost;
+
+ @Override
+ public MongoClientOptions getObject() throws BeansException {
+ return MongoClientOptions.builder().connectionsPerHost(connectionsPerHost).build();
+ }
+
+ @Override
+ public Class getObjectType() {
+ return MongoClientOptions.class;
+ }
+
+ @Override
+ public boolean isSingleton() {
+ return false;
+ }
+
+ public int getConnectionsPerHost() {
+ return connectionsPerHost;
+ }
+
+ public void setConnectionsPerHost(final int connectionsPerHost) {
+ this.connectionsPerHost = connectionsPerHost;
+ }
+
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/ObjectStoreFileUtility.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/ObjectStoreFileUtility.java
new file mode 100644
index 0000000..2316a66
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/ObjectStoreFileUtility.java
@@ -0,0 +1,57 @@
+package eu.dnetlib.data.objectstore.filesystem;
+
+import java.io.UnsupportedEncodingException;
+
+import com.mongodb.DBObject;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
+import eu.dnetlib.data.objectstore.rmi.Protocols;
+import eu.dnetlib.miscutils.functional.UnaryFunction;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * The Class ObjectStoreFileBuilder generates an objectStoreFile bean
+ *
+ */
+public class ObjectStoreFileUtility {
+
+ private static final int KB_SIZE = 1024;
+
+ /** The Constant log. */
+ private static final Log log = LogFactory.getLog(ObjectStoreFileUtility.class);
+
+ public static ObjectStoreFile build(final DBObject metadata, final String baseURI, final String objectStoreID, final String basePath) {
+
+ String originalFile = (String) metadata.get("originalObject");
+ ObjectStoreFile original = ObjectStoreFile.createObject(originalFile);
+ ObjectStoreFile newFile = new ObjectStoreFile();
+ newFile.setObjectID((String) metadata.get("id"));
+ newFile.setAccessProtocol(Protocols.HTTP);
+ newFile.setMimeType((String) metadata.get("mime"));
+ newFile.setMd5Sum((String) metadata.get("md5Sum"));
+ try {
+ newFile.setFileSizeKB(Long.parseLong(metadata.get("size").toString()) / KB_SIZE);
+ } catch (Throwable e) {
+ log.error("Error on getting file size", e);
+ }
+ if (originalFile != null) {
+ newFile.setMetadataRelatedID(original.getMetadataRelatedID());
+ if (StringUtils.isBlank(original.getDownloadedURL())) {
+ newFile.setDownloadedURL(original.getURI());
+ } else {
+ newFile.setDownloadedURL(original.getDownloadedURL());
+ }
+ }
+ try {
+ newFile.setURI(ModularObjectStoreRESTService.retrieveURL(baseURI, basePath, objectStoreID, newFile.getObjectID()));
+ } catch (UnsupportedEncodingException e) {
+ log.error("Error on Build objectStoreFile ", e);
+ }
+ return newFile;
+ }
+
+ public static UnaryFunction asJSON(final String baseURI, final String objectStoreID, final String basePath) {
+ return input -> build(input, baseURI, objectStoreID, basePath).toJSON();
+ }
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/AbstractObjectStoreAction.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/AbstractObjectStoreAction.java
new file mode 100644
index 0000000..b1d0e1b
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/AbstractObjectStoreAction.java
@@ -0,0 +1,77 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+
+import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardServerAction;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * The Class AbstractObjectStoreAction.
+ */
+public abstract class AbstractObjectStoreAction implements BlackboardServerAction {
+
+ /**
+ * Logger
+ */
+ private static final Log log = LogFactory.getLog(AbstractObjectStoreAction.class);
+ private final Executor executor = Executors.newCachedThreadPool();
+ /** The object store dao. */
+ private ObjectStoreDao dao;
+
+ protected abstract void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) throws ObjectStoreServiceException;
+
+ @Override
+ public void execute(final BlackboardServerHandler handler, final BlackboardJob job) {
+ executor.execute(new Runnable() {
+
+ @Override
+ public void run() {
+ try {
+ handler.ongoing(job);
+ executeAsync(handler, job);
+ } catch (ObjectStoreServiceException e) {
+ handler.failed(job, e);
+ }
+ }
+ });
+ }
+
+ protected void completeWithSuccess(final BlackboardServerHandler handler, final BlackboardJob job) {
+ // Don't change this synchronization rule
+ synchronized (this) {
+ handler.done(job);
+ }
+ }
+
+ protected void completeWithFail(final BlackboardServerHandler handler, final BlackboardJob job, final Throwable e) {
+ // Don't change this synchronization rule
+ synchronized (this) {
+ handler.failed(job, e);
+ }
+ }
+
+ /**
+ * Gets the dao.
+ *
+ * @return the dao
+ */
+ public ObjectStoreDao getDao() {
+ return dao;
+ }
+
+ /**
+ * Sets the dao.
+ *
+ * @param dao the new dao
+ */
+ public void setDao(ObjectStoreDao dao) {
+ this.dao = dao;
+ }
+
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/CreateObjectStoreAction.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/CreateObjectStoreAction.java
new file mode 100644
index 0000000..6a3c25d
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/CreateObjectStoreAction.java
@@ -0,0 +1,62 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * The Class CreateObjectStoreAction is responsible to execute the blacboard action of type CREATE.
+ */
+public class CreateObjectStoreAction extends AbstractObjectStoreAction {
+
+ private static final Log log = LogFactory.getLog(CreateObjectStoreAction.class);
+
+ /** The profile creator. */
+ private ObjectStoreProfileCreator profileCreator;
+
+ /**
+ * Gets the profile creator.
+ *
+ * @return the profile creator
+ */
+ public ObjectStoreProfileCreator getProfileCreator() {
+ return profileCreator;
+ }
+
+ /**
+ * Sets the profile creator.
+ *
+ * @param profileCreator the new profile creator
+ */
+ public void setProfileCreator(final ObjectStoreProfileCreator profileCreator) {
+ this.profileCreator = profileCreator;
+ }
+
+ @Override
+ protected void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) {
+ try {
+ final String interpretation = job.getParameters().get("interpretation");
+// final String basePath = job.getParameters().get("basePath");
+// if (StringUtils.isBlank(basePath)) {
+// throw new ObjectStoreServiceException("missing basePath param");
+// }
+
+ final String objID = profileCreator.registerProfile(interpretation);
+ try {
+ getDao().createObjectStore(objID, interpretation, null);
+ } catch (Throwable e) {
+ log.warn("cannot created objectStore, deleting profile");
+ profileCreator.deleteProfile(objID);
+ throw new ObjectStoreServiceException(e);
+ }
+ job.getParameters().put("objectStoreId", objID);
+ completeWithSuccess(handler, job);
+ } catch (Exception e) {
+ completeWithFail(handler, job, e);
+ }
+
+ }
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/DeleteObjectStoreAction.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/DeleteObjectStoreAction.java
new file mode 100644
index 0000000..3bec3cc
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/DeleteObjectStoreAction.java
@@ -0,0 +1,55 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;
+import org.springframework.beans.factory.annotation.Autowired;
+
+/**
+ * The Class DeleteObjectStoreAction is responsible to execute the blacboard action of type DELETE.
+ */
+public class DeleteObjectStoreAction extends AbstractObjectStoreAction {
+
+ /**
+ * The profile creator.
+ */
+
+ private ObjectStoreProfileCreator profileCreator;
+
+ @Autowired
+ private UniqueServiceLocator serviceLocator;
+
+ @Override
+ protected void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) throws ObjectStoreServiceException {
+ try {
+ final String objID = job.getParameters().get("obsID");
+ serviceLocator.getService(ISRegistryService.class).deleteProfile(objID);
+ getDao().deleteObjectStore(objID);
+ completeWithSuccess(handler, job);
+
+ } catch (Exception e) {
+ completeWithFail(handler, job, e);
+ }
+ }
+
+ /**
+ * Gets the profile creator.
+ *
+ * @return the profile creator
+ */
+ public ObjectStoreProfileCreator getProfileCreator() {
+ return profileCreator;
+ }
+
+ /**
+ * Sets the profile creator.
+ *
+ * @param profileCreator the new profile creator
+ */
+ public void setProfileCreator(ObjectStoreProfileCreator profileCreator) {
+ this.profileCreator = profileCreator;
+ }
+
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/DropContentObjectStoreAction.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/DropContentObjectStoreAction.java
new file mode 100644
index 0000000..0ded74a
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/DropContentObjectStoreAction.java
@@ -0,0 +1,25 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;
+
+/**
+ * Created by sandro on 2/26/16.
+ */
+public class DropContentObjectStoreAction extends AbstractObjectStoreAction {
+
+ @Override
+ protected void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) throws ObjectStoreServiceException {
+ try {
+ final String objID = job.getParameters().get("obsID");
+
+ final boolean status = getDao().dropContent(objID);
+ job.getParameters().put("dropStatus", "" + status);
+ completeWithSuccess(handler, job);
+ } catch (Exception e) {
+ completeWithFail(handler, job, e);
+ }
+ }
+
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/FeedCompleteObjectAction.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/FeedCompleteObjectAction.java
new file mode 100644
index 0000000..5d3e579
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/FeedCompleteObjectAction.java
@@ -0,0 +1,50 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;
+import org.springframework.beans.factory.annotation.Required;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class FeedCompleteObjectAction is responsible to execute the blacboard action of type FEED OBJECT which is Metadata created for WOS case of openaire.
+ */
+public class FeedCompleteObjectAction extends AbstractObjectStoreAction {
+
+ /** The store feeder. */
+ private ModularObjectStoreFeeder storeFeeder;
+
+ @Override
+ protected void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) throws ObjectStoreServiceException {
+ try {
+ final String objStoreID = job.getParameters().get("obsID");
+ final String eprRs = job.getParameters().get("epr");
+ final String mime = job.getParameters().get("mime");
+ int count = storeFeeder.feedMetadataObjectRecord(objStoreID, eprRs, mime);
+ job.getParameters().put("total", "" + count);
+ completeWithSuccess(handler, job);
+ } catch (Exception e) {
+ completeWithFail(handler, job, e);
+ }
+ }
+
+ /**
+ * Gets the store feeder.
+ *
+ * @return the store feeder
+ */
+ public ModularObjectStoreFeeder getStoreFeeder() {
+ return storeFeeder;
+ }
+
+ /**
+ * Sets the store feeder.
+ *
+ * @param storeFeeder the new store feeder
+ */
+ @Required
+ public void setStoreFeeder(final ModularObjectStoreFeeder storeFeeder) {
+ this.storeFeeder = storeFeeder;
+ }
+
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/FeedObjectStoreAction.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/FeedObjectStoreAction.java
new file mode 100644
index 0000000..51be588
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/FeedObjectStoreAction.java
@@ -0,0 +1,92 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.data.objectstore.rmi.Protocols;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
+import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;
+import org.springframework.beans.factory.annotation.Required;
+
+
+/**
+ * The Class FeedObjectStoreAction to execute the blacboard action of type FEED.
+ */
+public class FeedObjectStoreAction extends AbstractObjectStoreAction {
+
+ /** The profile creator. */
+ private ObjectStoreProfileCreator profileCreator;
+
+ /** The store feeder. */
+ private ModularObjectStoreFeeder storeFeeder;
+
+ @Override
+ protected void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) throws ObjectStoreServiceException {
+ try {
+ final String objStoreID = job.getParameters().get("obsID");
+ final String objID = job.getParameters().get("objID");
+ final String URI = job.getParameters().get("URI");
+ final String eprRs = job.getParameters().get("epr");
+ final String protocolString = job.getParameters().get("protocol");
+
+ final Protocols protocol;
+ if (protocolString == null)
+ protocol = Protocols.None;
+ else
+ protocol = Protocols.valueOf(job.getParameters().get("protocol"));
+ final String mime = job.getParameters().get("mime");
+ final String login = job.getParameters().get("login");
+ final String password = job.getParameters().get("password");
+
+ if (URI != null && URI.length() > 0) {
+ storeFeeder.feedObject(objStoreID, objID, URI, protocol, login,
+ password, mime);
+ } else if (eprRs != null && eprRs.length() > 0) {
+ storeFeeder
+ .feed(objStoreID, eprRs, protocol, login, password, mime);
+ }
+ completeWithSuccess(handler, job);
+ } catch (Exception e) {
+ completeWithFail(handler, job, e);
+ }
+ }
+
+
+
+ /**
+ * Gets the profile creator.
+ *
+ * @return the profile creator
+ */
+ public ObjectStoreProfileCreator getProfileCreator() {
+ return profileCreator;
+ }
+
+ /**
+ * Sets the profile creator.
+ *
+ * @param profileCreator the new profile creator
+ */
+ @Required
+ public void setProfileCreator(ObjectStoreProfileCreator profileCreator) {
+ this.profileCreator = profileCreator;
+ }
+
+ /**
+ * Gets the store feeder.
+ *
+ * @return the store feeder
+ */
+ public ModularObjectStoreFeeder getStoreFeeder() {
+ return storeFeeder;
+ }
+
+ /**
+ * Sets the store feeder.
+ *
+ * @param storeFeeder the new store feeder
+ */
+ @Required
+ public void setStoreFeeder(ModularObjectStoreFeeder storeFeeder) {
+ this.storeFeeder = storeFeeder;
+ }
+
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ModularObjectStoreDeliver.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ModularObjectStoreDeliver.java
new file mode 100644
index 0000000..ccbe1e4
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ModularObjectStoreDeliver.java
@@ -0,0 +1,128 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import javax.annotation.Resource;
+import javax.xml.ws.wsaddressing.W3CEndpointReference;
+
+import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.enabling.resultset.ResultSetFactory;
+import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;
+import org.springframework.beans.factory.annotation.Required;
+
+/**
+ * The Class ModularObjectStoreDeliver is responsible of delivering data from the object Store.
+ */
+public class ModularObjectStoreDeliver {
+
+ /** The dao. */
+ private ObjectStoreDao dao;
+
+ /** The result set factory. */
+ @Resource
+ private ResultSetFactory resultSetFactory;
+
+ /** The result set client factory. */
+ private ResultSetClientFactory resultSetClientFactory;
+
+ /**
+ * Gets the dao.
+ *
+ * @return the dao
+ */
+ public ObjectStoreDao getDao() {
+ return dao;
+ }
+
+ /**
+ * Sets the dao.
+ *
+ * @param dao
+ * the new dao
+ */
+ @Required
+ public void setDao(final ObjectStoreDao dao) {
+ this.dao = dao;
+ }
+
+ /**
+ * Deliver.
+ *
+ * @param objectStoreID
+ * the object store id
+ * @param from
+ * the from
+ * @param until
+ * the until
+ * @return the w3 c endpoint reference
+ * @throws ObjectStoreServiceException
+ */
+ public W3CEndpointReference deliver(final String objectStoreID, final Long from, final Long until) throws ObjectStoreServiceException {
+ return resultSetFactory.createResultSet(dao.getObjectStore(objectStoreID).deliver(from, until));
+ }
+
+ /**
+ * Deliver ids.
+ *
+ * @param objectStoreID
+ * the object store id
+ * @param eprId
+ * the epr id
+ * @return the w3 c endpoint reference
+ * @throws ObjectStoreServiceException
+ */
+ public W3CEndpointReference deliverIds(final String objectStoreID, final W3CEndpointReference eprId) throws ObjectStoreServiceException {
+
+ Iterable ids = resultSetClientFactory.getClient(eprId);
+ return resultSetFactory.createResultSet(dao.getObjectStore(objectStoreID).deliverIds(ids));
+ }
+
+ /**
+ * Exist id starts with.
+ *
+ * @param obsId
+ * the obs id
+ * @param startId
+ * the start id
+ * @return true, if successful
+ * @throws ObjectStoreServiceException
+ */
+ public boolean existIDStartsWith(final String obsId, final String startId) throws ObjectStoreServiceException {
+ return dao.getObjectStore(obsId).existIDStartsWith(startId);
+ }
+
+ /**
+ * Deliver object.
+ *
+ * @param objectStoreID
+ * the object store id
+ * @param objectId
+ * the object id
+ * @return the object store file
+ * @throws ObjectStoreServiceException
+ */
+ public ObjectStoreFile deliverObject(final String objectStoreID, final String objectId) throws ObjectStoreServiceException {
+ return dao.getObjectStore(objectStoreID).deliverObject(objectId);
+ }
+
+ /**
+ * Gets the result set client factory.
+ *
+ * @return the result set client factory
+ */
+ public ResultSetClientFactory getResultSetClientFactory() {
+ return resultSetClientFactory;
+ }
+
+ /**
+ * Sets the result set client factory.
+ *
+ * @param resultSetClientFactory
+ * the new result set client factory
+ */
+ @Required
+ public void setResultSetClientFactory(final ResultSetClientFactory resultSetClientFactory) {
+ this.resultSetClientFactory = resultSetClientFactory;
+ }
+
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ModularObjectStoreFeeder.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ModularObjectStoreFeeder.java
new file mode 100644
index 0000000..b172e85
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ModularObjectStoreFeeder.java
@@ -0,0 +1,234 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;
+import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;
+import eu.dnetlib.data.objectstore.rmi.MetadataObjectRecord;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.data.objectstore.rmi.Protocols;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;
+import eu.dnetlib.miscutils.datetime.DateUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Required;
+
+/**
+ * The Class ModularObjectStoreFeeder, responsible to feed data into the object Store
+ */
+public class ModularObjectStoreFeeder {
+
+ private static final Log log = LogFactory.getLog(ModularObjectStoreFeeder.class);
+
+ /** The dao of the objectStore. */
+ private ObjectStoreDao dao;
+
+ /** The result set client factory. */
+ private ResultSetClientFactory resultSetClientFactory;
+
+ /** The service locator. */
+ private UniqueServiceLocator serviceLocator;
+
+ /**
+ * Gets the dao.
+ *
+ * @return the dao
+ */
+ public ObjectStoreDao getDao() {
+ return dao;
+ }
+
+ /**
+ * Sets the dao.
+ *
+ * @param dao
+ * the new dao
+ */
+ @Required
+ public void setDao(final ObjectStoreDao dao) {
+ this.dao = dao;
+ }
+
+ /**
+ * Gets the result set client factory.
+ *
+ * @return the result set client factory
+ */
+ public ResultSetClientFactory getResultSetClientFactory() {
+ return resultSetClientFactory;
+ }
+
+ /**
+ * Sets the result set client factory.
+ *
+ * @param resultSetClientFactory
+ * the new result set client factory
+ */
+
+ @Required
+ public void setResultSetClientFactory(final ResultSetClientFactory resultSetClientFactory) {
+ this.resultSetClientFactory = resultSetClientFactory;
+ }
+
+ /**
+ * Feed metadata object record.
+ *
+ * @param objectStoreID
+ * the object store id
+ * @param rsEpr
+ * the rs epr
+ * @param mime
+ * the mime
+ * @throws ObjectStoreServiceException
+ */
+ public int feedMetadataObjectRecord(final String objectStoreID, final String rsEpr, final String mime) throws ObjectStoreServiceException {
+
+ final Iterable records = resultSetClientFactory.getClient(rsEpr);
+ Iterable toIngest = Iterables.transform(records, input -> {
+ MetadataObjectRecord record = MetadataObjectRecord.initFromJson(input);
+ if (record != null) {
+ record.setMime(mime);
+ } else {
+ log.error("An input record is null :" + input);
+ }
+ return record;
+ });
+ ObjectStore store = dao.getObjectStore(objectStoreID);
+ int size = store.feedMetadataRecord(toIngest, true);
+ touch(objectStoreID, size);
+ return size;
+ }
+
+ /**
+ * Feed object in the object store starting from an EPR of objectMetadata
+ *
+ * @param obsId
+ * the objectStore id
+ * @param rsEpr
+ * the result set Endpoint-reference
+ * @param protocol
+ * the protocol
+ * @param login
+ * the login
+ * @param password
+ * the password
+ * @param mime
+ * the mime type
+ * @throws ObjectStoreServiceException
+ */
+ public void feed(final String obsId, final String rsEpr, final Protocols protocol, final String login, final String password, final String mime)
+ throws ObjectStoreServiceException {
+ final Iterable records = resultSetClientFactory.getClient(rsEpr);
+ ObjectBroker objectBroker = new ObjectBroker(protocol, login, password, mime);
+ Iterable toIngest = Iterables.transform(records, objectBroker);
+ ObjectStore store = dao.getObjectStore(obsId);
+ int size = store.feed(toIngest, true);
+ touch(obsId, size);
+ }
+
+ /**
+ * Feed a single object in the object Stroe.
+ *
+ * @param objectStoreID
+ * the object store id
+ * @param objectID
+ * the object id
+ * @param URIObject
+ * the URI of object
+ * @param protocol
+ * the protocol
+ * @param login
+ * the login
+ * @param password
+ * the password
+ * @param mime
+ * the mime type
+ * @throws ObjectStoreServiceException
+ */
+ public void feedObject(final String objectStoreID,
+ final String objectID,
+ final String URIObject,
+ final Protocols protocol,
+ final String login,
+ final String password,
+ final String mime) throws ObjectStoreServiceException {
+ ObjectStoreFile inputFile = new ObjectStoreFile();
+ inputFile.setURI(URIObject);
+ inputFile.setObjectID(objectID);
+ ObjectBroker objectBroker = new ObjectBroker(protocol, login, password, mime);
+ Iterable toIngest = Iterables.transform(Lists.newArrayList(inputFile.toJSON()), objectBroker);
+ ObjectStore store = dao.getObjectStore(objectStoreID);
+ int size = store.feed(toIngest, true);
+ touch(objectStoreID, size);
+ }
+
+ /**
+ * Feed object record.
+ *
+ * @param objectStoreID
+ * the object store id
+ * @param record
+ * the record
+ * @return the string
+ * @throws ObjectStoreServiceException
+ */
+ public String feedObjectRecord(final String objectStoreID, final ObjectStoreRecord record) throws ObjectStoreServiceException {
+ ObjectStore store = dao.getObjectStore(objectStoreID);
+ return store.feedObjectRecord(record);
+ }
+
+ /**
+ * Sets the last modified date in the profile.
+ *
+ * @param obsId
+ * the obs id
+ * @param size
+ * the size
+ */
+ public void touch(final String obsId, final int size) {
+ try {
+ final String now = DateUtils.now_ISO8601();
+
+ final String mdstoreXUpdate = "for $x in //RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '" + obsId + "']"
+ + "return update value $x//LAST_STORAGE_DATE with '" + now + "'";
+
+ serviceLocator.getService(ISRegistryService.class, true).executeXUpdate(mdstoreXUpdate);
+
+ touchSize(obsId, size);
+ } catch (final Exception e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ /**
+ * Touch size.
+ *
+ * @param obsId
+ * the obs id
+ * @param size
+ * the size
+ */
+ public void touchSize(final String obsId, final int size) {
+ try {
+ final String mdstoreNumberXUpdate = "for $x in //RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '" + obsId + "']"
+ + "return update value $x//COUNT_STORE with '" + size + "'";
+
+ serviceLocator.getService(ISRegistryService.class, true).executeXUpdate(mdstoreNumberXUpdate);
+ } catch (final Exception e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ public UniqueServiceLocator getServiceLocator() {
+ return serviceLocator;
+ }
+
+ @Required
+ public void setServiceLocator(final UniqueServiceLocator serviceLocator) {
+ this.serviceLocator = serviceLocator;
+ }
+
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ModularObjectStoreService.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ModularObjectStoreService.java
new file mode 100644
index 0000000..2690df4
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ModularObjectStoreService.java
@@ -0,0 +1,265 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import java.util.List;
+
+import javax.xml.ws.wsaddressing.W3CEndpointReference;
+
+import org.springframework.beans.factory.annotation.Required;
+
+import com.google.gson.Gson;
+
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreService;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.enabling.tools.AbstractBaseService;
+import eu.dnetlib.enabling.tools.blackboard.NotificationHandler;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class ModularObjectStoreService is the implementation of the ObjectStoreService interface.
+ */
+public class ModularObjectStoreService extends AbstractBaseService implements ObjectStoreService {
+
+ /** The feeder. */
+ private ModularObjectStoreFeeder feeder;
+
+ /** The object store deliver. */
+ private ModularObjectStoreDeliver objectStoreDeliver;
+
+ /** The notification handler. */
+ private NotificationHandler notificationHandler;
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see eu.dnetlib.data.objectstore.rmi.ObjectStoreService#deliverObjects(java.lang.String, java.lang.Double, java.lang.Double)
+ */
+ /**
+ * Deliver objects.
+ *
+ * @param obsId
+ * the obs id
+ * @param from
+ * the from
+ * @param until
+ * the until
+ * @return the w3 c endpoint reference
+ * @throws ObjectStoreServiceException
+ * the object store service exception
+ */
+ @Override
+ public W3CEndpointReference deliverObjects(final String obsId, final Long from, final Long until) throws ObjectStoreServiceException {
+
+ return objectStoreDeliver.deliver(obsId, from, until);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see eu.dnetlib.data.objectstore.rmi.ObjectStoreService#deliverObjectsByIds(java.lang.String,
+ * javax.xml.ws.wsaddressing.W3CEndpointReference)
+ */
+ /**
+ * Deliver objects by ids.
+ *
+ * @param obsId
+ * the obs id
+ * @param eprId
+ * the epr id
+ * @return the w3 c endpoint reference
+ * @throws ObjectStoreServiceException
+ * the object store service exception
+ */
+ @Override
+ public W3CEndpointReference deliverObjectsByIds(final String obsId, final W3CEndpointReference eprId) throws ObjectStoreServiceException {
+ return objectStoreDeliver.deliverIds(obsId, eprId);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see eu.dnetlib.data.objectstore.rmi.ObjectStoreService#deliverRecord(java.lang.String, java.lang.String)
+ */
+ /**
+ * Deliver record.
+ *
+ * @param obsId
+ * the obs id
+ * @param objectId
+ * the object id
+ * @return the string
+ * @throws ObjectStoreServiceException
+ * the object store service exception
+ */
+ @Override
+ public String deliverRecord(final String obsId, final String objectId) throws ObjectStoreServiceException {
+ return objectStoreDeliver.deliverObject(obsId, objectId).toJSON();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see eu.dnetlib.data.objectstore.rmi.ObjectStoreService#getListOfObjectStores()
+ */
+ /**
+ * Gets the list of object stores.
+ *
+ * @return the list of object stores
+ */
+ @Override
+ public List getListOfObjectStores() {
+ return objectStoreDeliver.getDao().listObjectStores();
+ }
+
+ /**
+ * Gets the feeder.
+ *
+ * @return the feeder
+ */
+ public ModularObjectStoreFeeder getFeeder() {
+ return feeder;
+ }
+
+ /**
+ * Sets the feeder.
+ *
+ * @param feeder
+ * the new feeder
+ */
+ @Required
+ public void setFeeder(final ModularObjectStoreFeeder feeder) {
+ this.feeder = feeder;
+ }
+
+ /**
+ * Gets the notification handler.
+ *
+ * @return the notification handler
+ */
+ public NotificationHandler getNotificationHandler() {
+ return notificationHandler;
+ }
+
+ /**
+ * Sets the notification handler.
+ *
+ * @param notificationHandler
+ * the new notification handler
+ */
+ @Required
+ public void setNotificationHandler(final NotificationHandler notificationHandler) {
+ this.notificationHandler = notificationHandler;
+ }
+
+ /**
+ * {@inheritDoc}
+ *
+ * @see eu.dnetlib.enabling.tools.AbstractBaseService#notify(java.lang.String, java.lang.String, java.lang.String, java.lang.String)
+ */
+ @Override
+ public void notify(final String subscriptionId, final String topic, final String isId, final String message) {
+ getNotificationHandler().notified(subscriptionId, topic, isId, message);
+ }
+
+ /**
+ * Gets the object store deliver.
+ *
+ * @return the object store deliver
+ */
+ public ModularObjectStoreDeliver getObjectStoreDeliver() {
+ return objectStoreDeliver;
+ }
+
+ /**
+ * Sets the object store deliver.
+ *
+ * @param objectStoreDeliver
+ * the new object store deliver
+ */
+ @Required
+ public void setObjectStoreDeliver(final ModularObjectStoreDeliver objectStoreDeliver) {
+ this.objectStoreDeliver = objectStoreDeliver;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see eu.dnetlib.data.objectstore.rmi.ObjectStoreService#feedObject(java.lang.String, java.lang.String)
+ */
+ /**
+ * Feed object.
+ *
+ * @param obsId
+ * the obs id
+ * @param objectMetadata
+ * the object metadata
+ * @throws ObjectStoreServiceException
+ * the object store service exception
+ */
+ @Override
+ public void feedObject(final String obsId, final String objectMetadata) throws ObjectStoreServiceException {
+ Gson g = new Gson();
+ try {
+ ObjectStoreFile file = g.fromJson(objectMetadata, ObjectStoreFile.class);
+ feeder.feedObject(obsId, file.getObjectID(), file.getURI(), file.getAccessProtocol(), file.getUsernameAuth(), file.getPasswordAuth(),
+ file.getMimeType());
+
+ } catch (Exception e) {
+ throw new ObjectStoreServiceException(e.getMessage());
+ }
+
+ }
+
+ /**
+ * Exist id starts with.
+ *
+ * @param obsId
+ * the obs id
+ * @param startId
+ * the start id
+ * @return true, if successful
+ * @throws ObjectStoreServiceException
+ * the object store service exception
+ */
+ public boolean existIDStartsWith(final String obsId, final String startId) throws ObjectStoreServiceException {
+ return objectStoreDeliver.existIDStartsWith(obsId, startId);
+ }
+
+ /**
+ * Feed object.
+ *
+ * @param obsId
+ * the obs id
+ * @param record
+ * the record
+ * @return the string
+ * @throws ObjectStoreServiceException
+ * the object store service exception
+ */
+ public String feedObject(final String obsId, final ObjectStoreRecord record) throws ObjectStoreServiceException {
+
+ try {
+
+ return feeder.feedObjectRecord(obsId, record);
+
+ } catch (Exception e) {
+ throw new ObjectStoreServiceException(e.getMessage());
+ }
+
+ }
+
+ /**
+ * Gets the size.
+ *
+ * @param obsId
+ * the obs id
+ * @return the size
+ * @throws ObjectStoreServiceException
+ * the object store service exception
+ */
+ @Override
+ public int getSize(final String obsId) throws ObjectStoreServiceException {
+ return objectStoreDeliver.getDao().getObjectStore(obsId).getSize();
+
+ }
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectBroker.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectBroker.java
new file mode 100644
index 0000000..aa26ff2
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectBroker.java
@@ -0,0 +1,240 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.SocketException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.net.ftp.FTPClient;
+
+import com.google.common.base.Function;
+
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
+import eu.dnetlib.data.objectstore.rmi.Protocols;
+
+
+
+/**
+ * The Class ObjectBroker is responsible to retrieve content from URL.
+ */
+public class ObjectBroker implements Function {
+ private static final Log log = LogFactory.getLog(ObjectBroker.class);
+
+ /** The protocol. */
+ private Protocols protocol;
+
+ /** The login. */
+ private String login;
+
+ /** The password. */
+ private String password;
+
+ /** The mime. */
+ private String mime;
+
+ /**
+ * Instantiates a new object broker.
+ *
+ * @param protocol
+ * the protocol
+ * @param login
+ * the login
+ * @param password
+ * the password
+ * @param mime
+ * the mime
+ */
+ public ObjectBroker(final Protocols protocol, final String login, final String password, final String mime) {
+ this.protocol = protocol;
+ this.login = login;
+ this.password = password;
+ this.mime = mime;
+ }
+
+ /**
+ * Gets the protocol.
+ *
+ * @return the protocol
+ */
+ public Protocols getProtocol() {
+ return protocol;
+ }
+
+ /**
+ * Sets the protocol.
+ *
+ * @param protocol
+ * the new protocol
+ */
+ public void setProtocol(final Protocols protocol) {
+ this.protocol = protocol;
+ }
+
+ /**
+ * Gets the login.
+ *
+ * @return the login
+ */
+ public String getLogin() {
+ return login;
+ }
+
+ /**
+ * Sets the login.
+ *
+ * @param login
+ * the new login
+ */
+ public void setLogin(final String login) {
+ this.login = login;
+ }
+
+ /**
+ * Gets the password.
+ *
+ * @return the password
+ */
+ public String getPassword() {
+ return password;
+ }
+
+ /**
+ * Sets the password.
+ *
+ * @param password
+ * the new password
+ */
+ public void setPassword(final String password) {
+ this.password = password;
+ }
+
+ /**
+ * Gets the mime.
+ *
+ * @return the mime
+ */
+ public String getMime() {
+ return mime;
+ }
+
+ /**
+ * Sets the mime.
+ *
+ * @param mime
+ * the new mime
+ */
+ public void setMime(final String mime) {
+ this.mime = mime;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see com.google.common.base.Function#apply(java.lang.Object)
+ */
+ @Override
+ public ObjectStoreRecord apply(final String jsonInput) {
+ if (jsonInput == null) return null;
+ ObjectStoreRecord objectStorerecord = new ObjectStoreRecord();
+
+ objectStorerecord.setFileMetadata(ObjectStoreFile.createObject(jsonInput));
+ Protocols currentProtocol = (objectStorerecord.getFileMetadata().getAccessProtocol() == Protocols.None) ? protocol : objectStorerecord
+ .getFileMetadata().getAccessProtocol();
+ objectStorerecord.getFileMetadata().setAccessProtocol(currentProtocol);
+ if ((objectStorerecord.getFileMetadata().getMimeType() == null) || (objectStorerecord.getFileMetadata().getMimeType().length() == 0)) {
+ objectStorerecord.getFileMetadata().setMimeType(mime);
+ }
+
+ switch (currentProtocol) {
+ case FTP:
+ FTPClient client = new FTPClient();
+ try {
+ URI uri = new URI(objectStorerecord.getFileMetadata().getURI());
+ client.connect(uri.getHost());
+ if ((objectStorerecord.getFileMetadata().getUsernameAuth() != null) && (objectStorerecord.getFileMetadata().getUsernameAuth().length() > 0)) {
+ client.login(objectStorerecord.getFileMetadata().getUsernameAuth(), objectStorerecord.getFileMetadata().getPasswordAuth());
+ } else {
+ client.login("ftp", "a@a");
+
+ }
+
+ final InputStream stream = client.retrieveFileStream(uri.getPath());
+ objectStorerecord.setInputStream(stream);
+ return objectStorerecord;
+
+ } catch (URISyntaxException e2) {
+ log.error(e2);
+ return null;
+ } catch (SocketException e) {
+ log.error(e);
+ } catch (IOException e) {
+ log.error(e);
+ return null;
+ }
+
+ case HTTP:
+ try {
+ HttpURLConnection conn = (HttpURLConnection) new URL(objectStorerecord.getFileMetadata().getURI()).openConnection();
+ InputStream in;
+ int http_status;
+ try {
+
+ http_status = conn.getResponseCode();
+
+ if ((http_status / 100) == 3) {
+ String newURL = conn.getHeaderField("Location");
+ conn.disconnect();
+ conn = (HttpURLConnection) new URL(newURL).openConnection();
+ http_status = conn.getResponseCode();
+ if ((http_status / 100) != 2) return null;
+ }
+ in = conn.getInputStream();
+ objectStorerecord.setInputStream(in);
+ return objectStorerecord;
+ } catch (Exception e) {
+ log.error(e);
+ return null;
+ }
+ } catch (MalformedURLException e1) {
+ log.error(e1);
+ return null;
+ } catch (IOException e1) {
+ log.error(e1);
+ return null;
+ }
+
+ case File_System:
+ File f = new File(objectStorerecord.getFileMetadata().getURI());
+ try {
+ InputStream myiInputStream = new FileInputStream(f);
+ objectStorerecord.setInputStream(myiInputStream);
+ return objectStorerecord;
+ } catch (FileNotFoundException e) {
+ try {
+ Thread.sleep(5000);
+ InputStream myiInputStream = new FileInputStream(f);
+ objectStorerecord.setInputStream(myiInputStream);
+ return objectStorerecord;
+ } catch (Exception e1) {
+ log.error(e1);
+ return null;
+ }
+ }
+ default:
+ break;
+ }
+
+ return null;
+
+ }
+
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectStoreActions.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectStoreActions.java
new file mode 100644
index 0000000..7a201e4
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectStoreActions.java
@@ -0,0 +1,22 @@
+package eu.dnetlib.data.objectstore.modular;
+
+/**
+ * The Enumeration of the blacboard action of the ObjectStore.
+ */
+public enum ObjectStoreActions {
+
+ /** The create. */
+ CREATE,
+ /**
+ * The delete.
+ */
+ DELETE,
+ /**
+ * The feed.
+ */
+ FEED,
+ /**
+ * The feedobject.
+ */
+ FEEDOBJECT, DROP_CONTENT
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectStoreConsistency.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectStoreConsistency.java
new file mode 100644
index 0000000..1e8e2a7
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectStoreConsistency.java
@@ -0,0 +1,102 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import java.util.List;
+import java.util.Set;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Controller;
+import org.springframework.ui.ModelMap;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.ResponseBody;
+
+
+@Controller
+public class ObjectStoreConsistency {
+
+ private static final Log log = LogFactory.getLog(ObjectStoreConsistency.class);
+
+ @Autowired
+ private ObjectStoreDao objectStoreDao;
+
+ @Autowired
+ private UniqueServiceLocator serviceLocator;
+
+ private ObjectStoreIntegrityInfo integrityInfo;
+
+
+ @RequestMapping(value = "/ui/objectStore/infoConsistency.do")
+ @ResponseBody
+ public ObjectStoreIntegrityInfo getConsistencyInfo(final ModelMap map) {
+ return getIntegrityInfo();
+ }
+
+ public void refreshConsistency() {
+ try {
+ Set profiles = listProfileIds();
+ Set objectStore = listObjectStoreIds();
+ Set objectStoreNotInProfile = Sets.newHashSet(objectStore);
+ objectStoreNotInProfile.removeAll(profiles);
+ Set profilesNotInObjectStore = Sets.newHashSet(profiles);
+ profilesNotInObjectStore.removeAll(objectStore);
+ ObjectStoreIntegrityInfo info = new ObjectStoreIntegrityInfo();
+ info.setObjectStoreWithoutProfile(objectStoreNotInProfile);
+ info.setProfileWithoutObjectStore(profilesNotInObjectStore);
+ this.integrityInfo = info;
+ } catch (ISLookUpException e) {
+ log.error("Error on refreshing consistency of objectStore ", e);
+ }
+
+
+ }
+
+
+ private Set listObjectStoreIds() {
+ return Sets.newHashSet(
+ Lists.transform(objectStoreDao.listObjectStores(), new Function() {
+ @Override
+ public String apply(String input) {
+ return StringUtils.substringBefore(input, "_");
+ }
+ }));
+ }
+
+
+ private Set listProfileIds() throws ISLookUpException {
+
+ final ISLookUpService lookupService = serviceLocator.getService(ISLookUpService.class);
+ final String query = "for $x in collection('/db/DRIVER/ObjectStoreDSResources/ObjectStoreDSResourceType') return $x//RESOURCE_IDENTIFIER/@value/string()";
+
+ List resultList = lookupService.quickSearchProfile(query);
+
+
+ return Sets.newHashSet(Lists.transform(resultList, new Function() {
+ @Override
+ public String apply(String input) {
+ return StringUtils.substringBefore(input, "_");
+ }
+ }));
+
+ }
+
+
+ public ObjectStoreIntegrityInfo getIntegrityInfo() {
+ if (integrityInfo == null) {
+ refreshConsistency();
+ }
+ return integrityInfo;
+ }
+
+ public void setIntegrityInfo(ObjectStoreIntegrityInfo integrityInfo) {
+ this.integrityInfo = integrityInfo;
+ }
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectStoreIntegrityInfo.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectStoreIntegrityInfo.java
new file mode 100644
index 0000000..2998520
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectStoreIntegrityInfo.java
@@ -0,0 +1,30 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import java.util.Set;
+
+/**
+ * Created by Sandro La Bruzzo on 10/16/15.
+ */
+public class ObjectStoreIntegrityInfo {
+
+ private Set objectStoreWithoutProfile;
+
+ private Set profileWithoutObjectStore;
+
+
+ public Set getObjectStoreWithoutProfile() {
+ return objectStoreWithoutProfile;
+ }
+
+ public void setObjectStoreWithoutProfile(Set objectStoreWithoutProfile) {
+ this.objectStoreWithoutProfile = objectStoreWithoutProfile;
+ }
+
+ public Set getProfileWithoutObjectStore() {
+ return profileWithoutObjectStore;
+ }
+
+ public void setProfileWithoutObjectStore(Set profileWithoutObjectStore) {
+ this.profileWithoutObjectStore = profileWithoutObjectStore;
+ }
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectStoreProfileCreator.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectStoreProfileCreator.java
new file mode 100644
index 0000000..1f3dffd
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectStoreProfileCreator.java
@@ -0,0 +1,127 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException;
+import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
+import eu.dnetlib.enabling.locators.UniqueServiceLocator;
+import eu.dnetlib.soap.EndpointReferenceBuilder;
+import org.antlr.stringtemplate.StringTemplate;
+import org.springframework.beans.factory.annotation.Required;
+
+import javax.xml.ws.Endpoint;
+
+// TODO: Auto-generated Javadoc
+
+/**
+ * The Class ObjectStoreProfileCreator is responsible of creating profile of the ObjectStore
+ */
+public class ObjectStoreProfileCreator {
+
+ /**
+ * service locator.
+ */
+ private UniqueServiceLocator serviceLocator;
+
+ /**
+ * objectstore ds template.
+ */
+ private StringTemplate objectstoreDsTemplate;
+
+ /**
+ * service endpoint.
+ */
+ private Endpoint endpoint;
+
+ /**
+ * endpoint builder.
+ */
+ private EndpointReferenceBuilder eprBuilder;
+
+ /**
+ * Register profile.
+ *
+ * @param interpretation the interpretation
+ * @return the string
+ * @throws ISRegistryException the IS registry exception
+ */
+ public String registerProfile(final String interpretation)
+ throws ISRegistryException {
+ // XXX: mini hack
+ StringTemplate template = new StringTemplate(
+ objectstoreDsTemplate.getTemplate());
+ template.setAttribute("serviceUri", eprBuilder.getAddress(endpoint));
+ template.setAttribute("interpretation", interpretation);
+
+ return serviceLocator.getService(ISRegistryService.class, true).registerProfile(template.toString());
+ }
+
+ public void deleteProfile(final String profileId) throws ISRegistryException {
+ serviceLocator.getService(ISRegistryService.class).deleteProfile(profileId);
+ }
+
+ /**
+ * Gets the endpoint.
+ *
+ * @return the endpoint
+ */
+ public Endpoint getEndpoint() {
+ return endpoint;
+ }
+
+ /**
+ * Sets the endpoint.
+ *
+ * @param endpoint the new endpoint
+ */
+ @Required
+ public void setEndpoint(final Endpoint endpoint) {
+ this.endpoint = endpoint;
+ }
+
+ /**
+ * Gets the epr builder.
+ *
+ * @return the epr builder
+ */
+ public EndpointReferenceBuilder getEprBuilder() {
+ return eprBuilder;
+ }
+
+ /**
+ * Sets the epr builder.
+ *
+ * @param eprBuilder the new epr builder
+ */
+ @Required
+ public void setEprBuilder(final EndpointReferenceBuilder eprBuilder) {
+ this.eprBuilder = eprBuilder;
+ }
+
+ /**
+ * Gets the objectstore ds template.
+ *
+ * @return the objectstore ds template
+ */
+ public StringTemplate getObjectstoreDsTemplate() {
+ return objectstoreDsTemplate;
+ }
+
+ /**
+ * Sets the objectstore ds template.
+ *
+ * @param objectstoreDsTemplate the new objectstore ds template
+ */
+ @Required
+ public void setObjectstoreDsTemplate(final StringTemplate objectstoreDsTemplate) {
+ this.objectstoreDsTemplate = objectstoreDsTemplate;
+ }
+
+ public UniqueServiceLocator getServiceLocator() {
+ return serviceLocator;
+ }
+
+ @Required
+ public void setServiceLocator(final UniqueServiceLocator serviceLocator) {
+ this.serviceLocator = serviceLocator;
+ }
+
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectStoreRecord.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectStoreRecord.java
new file mode 100644
index 0000000..08beadb
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectStoreRecord.java
@@ -0,0 +1,55 @@
+package eu.dnetlib.data.objectstore.modular;
+
+import java.io.InputStream;
+
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
+
+/**
+ * The Class ObjectStoreRecord is a serialization of the object to be ingested in the objectStore
+ * metadata + inputstream of the data.
+ */
+public class ObjectStoreRecord {
+
+ /** The file metadata. */
+ private ObjectStoreFile fileMetadata;
+
+ /** The input stream. */
+ private InputStream inputStream;
+
+ /**
+ * Gets the input stream.
+ *
+ * @return the input stream
+ */
+ public InputStream getInputStream() {
+ return inputStream;
+ }
+
+ /**
+ * Sets the input stream.
+ *
+ * @param inputStream the new input stream
+ */
+ public void setInputStream(final InputStream inputStream) {
+ this.inputStream = inputStream;
+ }
+
+ /**
+ * Gets the file metadata.
+ *
+ * @return the file metadata
+ */
+ public ObjectStoreFile getFileMetadata() {
+ return fileMetadata;
+ }
+
+ /**
+ * Sets the file metadata.
+ *
+ * @param fileMetadata the new file metadata
+ */
+ public void setFileMetadata(final ObjectStoreFile fileMetadata) {
+ this.fileMetadata = fileMetadata;
+ }
+
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/connector/ObjectStore.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/connector/ObjectStore.java
new file mode 100644
index 0000000..339bbc5
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/connector/ObjectStore.java
@@ -0,0 +1,131 @@
+package eu.dnetlib.data.objectstore.modular.connector;
+
+import eu.dnetlib.data.objectstore.modular.ObjectStoreRecord;
+import eu.dnetlib.data.objectstore.rmi.MetadataObjectRecord;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.enabling.resultset.ResultSetListener;
+
+/**
+ * The Interface ObjectStore.
+ */
+public interface ObjectStore {
+
+ /**
+ * Gets the id of the objectStore.
+ *
+ * @return the id
+ */
+ String getId();
+
+ /**
+ * Gets the interpretation the objectStore.
+ *
+ * @return the interpretation
+ */
+ String getInterpretation();
+
+ /**
+ * Feed record into the objectstore.
+ *
+ * @param records
+ * the records
+ * @param incremental
+ * the incremental
+ * @return the int
+ */
+ int feed(Iterable records, boolean incremental) throws ObjectStoreServiceException;
+
+ /**
+ * Feed metadata record into the objectStore.
+ *
+ * @param records
+ * the records
+ * @param incremental
+ * the incremental
+ * @return the int
+ */
+ int feedMetadataRecord(Iterable records, boolean incremental) throws ObjectStoreServiceException;
+
+ /**
+ * Feed a single object record into the objectStore.
+ *
+ * @param record
+ * the record
+ * @return the string
+ */
+ String feedObjectRecord(ObjectStoreRecord record) throws ObjectStoreServiceException;
+
+ /**
+ * Deliver Object from the objectStore.
+ *
+ * @param from
+ * : start date which you want to filter
+ * @param until
+ * : end date which you want to filter
+ * @return the result set listener
+ */
+ ResultSetListener deliver(Long from, Long until) throws ObjectStoreServiceException;
+
+ /**
+ * Deliver ids.
+ *
+ * @param ids
+ * the ids
+ * @return the result set listener
+ */
+ ResultSetListener deliverIds(Iterable ids) throws ObjectStoreServiceException;
+
+ /**
+ * Deliver object.
+ *
+ * @param objectId
+ * the object id
+ * @return the object store file
+ * @throws ObjectStoreServiceException
+ */
+ ObjectStoreFile deliverObject(String objectId) throws ObjectStoreServiceException;
+
+ /**
+ * Gets the size.
+ *
+ * @return the size
+ */
+ int getSize() throws ObjectStoreServiceException;
+
+ /**
+ * Delete object.
+ *
+ * @param objectId
+ * the object id
+ */
+ void deleteObject(String objectId) throws ObjectStoreServiceException;
+
+ /**
+ * Gets the object.
+ *
+ * @param recordId
+ * the record id
+ * @return the object
+ */
+ String getObject(String recordId) throws ObjectStoreServiceException;
+
+ /**
+ * Find if exist an ID startingwith the string startId.
+ *
+ * @param startId
+ * the start id
+ * @return if exist or less an id
+ */
+ boolean existIDStartsWith(String startId) throws ObjectStoreServiceException;
+
+ /**
+ * drop the content of the ObjectStore
+ * THIS METHOD DELETE ALL THE CONTENT INSIDE THE OBJECTSTORE PAY ATTENTION WHEN YOU CALL IT
+ *
+ * @return if the content has been refreshed
+ * @throws ObjectStoreServiceException
+ */
+ boolean dropContent() throws ObjectStoreServiceException;
+
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/connector/ObjectStoreDao.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/connector/ObjectStoreDao.java
new file mode 100644
index 0000000..3027541
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/connector/ObjectStoreDao.java
@@ -0,0 +1,72 @@
+package eu.dnetlib.data.objectstore.modular.connector;
+
+import java.util.List;
+
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+
+/**
+ * The Interface ObjectStoreDao.
+ */
+public interface ObjectStoreDao {
+
+ /**
+ * Gets an Object Store with the given identifier.
+ *
+ * If an Object Store with the given identifier does not exist, a new one is created.
+ *
+ *
+ * @param obsId
+ * the object store identifier
+ * @return the object store
+ * @throws ObjectStoreServiceException
+ */
+ ObjectStore getObjectStore(String obsId) throws ObjectStoreServiceException;
+
+ /**
+ * List all the Object stores.
+ *
+ * @return the list of object store ids
+ */
+ List listObjectStores();
+
+ /**
+ * Creates an Object Store with the given identifier.
+ *
+ * If an Object Store with the given identifier already exists this method does nothing.
+ *
+ *
+ * @param obsId the object store identifier
+ * @param interpretation the interpretation of the store
+ * @param basePath the base path to store the object Store in case of file system implementation
+ * @return true, if successful
+ * @throws ObjectStoreServiceException
+ */
+ boolean createObjectStore(String obsId, String interpretation, String basePath) throws ObjectStoreServiceException;
+
+ /**
+ * Upddate an Object Store metadata with the given identifier.
+ *
+ * If an Object Store with the given identifier does not exist, a new one is created.
+ *
+ *
+ * @param obsId
+ * the object store identifier
+ * @param interpretation
+ * the interpretation of the store
+ * @return true, if successful
+ */
+ boolean updateObjectStore(String obsId, String interpretation);
+
+ /**
+ * Delete object store.
+ *
+ * @param obsId
+ * the object store identifier
+ * @return true, if successful
+ * @throws ObjectStoreServiceException
+ */
+ boolean deleteObjectStore(String obsId) throws ObjectStoreServiceException;
+
+ boolean dropContent(String obsId) throws ObjectStoreServiceException;
+
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/inspector/ObjectStoreInspector.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/inspector/ObjectStoreInspector.java
new file mode 100644
index 0000000..d0a2c5d
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/inspector/ObjectStoreInspector.java
@@ -0,0 +1,199 @@
+package eu.dnetlib.data.objectstore.modular.inspector;
+
+import static eu.dnetlib.miscutils.collections.MappedCollection.listMap;
+
+import java.util.List;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Controller;
+import org.springframework.ui.Model;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+
+import com.google.gson.Gson;
+
+import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;
+import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.enabling.inspector.AbstractInspectorController;
+import eu.dnetlib.enabling.resultset.ResultSetListener;
+import eu.dnetlib.miscutils.collections.MappedCollection;
+import eu.dnetlib.miscutils.functional.UnaryFunction;
+
+/**
+ * The Class ObjectStoreInspector is the inspector controller of the objectStore.
+ */
+@Controller
+public class ObjectStoreInspector extends AbstractInspectorController {
+
+ // private static final Log log = LogFactory.getLog(ObjectStoreInspector.class);
+
+ /** The object store dao. */
+ @Autowired
+ private ObjectStoreDao objectStoreDao;
+
+ /**
+ * The Class ObjectStoreDescription.
+ */
+ class ObjectStoreDescription {
+
+ /** The id. */
+ private String id;
+
+ /** The size. */
+ private int size;
+
+ /**
+ * Instantiates a new object store description.
+ *
+ * @param id
+ * the id
+ * @param size
+ * the size
+ */
+ public ObjectStoreDescription(final String id, final int size) {
+
+ this.id = id;
+ this.size = size;
+ }
+
+ /**
+ * Gets the interpretation.
+ *
+ * @return the interpretation
+ * @throws ObjectStoreServiceException
+ */
+ public String getInterpretation() throws ObjectStoreServiceException {
+
+ ObjectStore objectStore = objectStoreDao.getObjectStore(id);
+ return objectStore.getInterpretation();
+ }
+
+ /**
+ * Gets the id.
+ *
+ * @return the id
+ */
+ public String getId() {
+ return id;
+ }
+
+ /**
+ * Sets the id.
+ *
+ * @param id
+ * the new id
+ */
+ public void setId(final String id) {
+ this.id = id;
+ }
+
+ /**
+ * Gets the size.
+ *
+ * @return the size
+ */
+ public int getSize() {
+ return size;
+ }
+
+ /**
+ * Sets the size.
+ *
+ * @param size
+ * the new size
+ */
+ public void setSize(final int size) {
+ this.size = size;
+ }
+ }
+
+ /**
+ * Object stores.
+ *
+ * @param model
+ * the model
+ */
+ @RequestMapping(value = "/inspector/objectstores.do")
+ public void objectStores(final Model model) {
+ List objectStores = objectStoreDao.listObjectStores();
+ model.addAttribute("objectstores", MappedCollection.listMap(objectStores, new UnaryFunction() {
+
+ @Override
+ public ObjectStoreDescription evaluate(final String id) {
+ try {
+ ObjectStore objStore = objectStoreDao.getObjectStore(id);
+ return new ObjectStoreDescription(id, objStore.getSize());
+ } catch (ObjectStoreServiceException e) {
+ return null;
+ }
+ }
+ }));
+ }
+
+ /**
+ * Object store.
+ *
+ * @param model
+ * the model
+ * @param id
+ * the id
+ * @param startParam
+ * the start param
+ * @param regex
+ * the regex
+ * @throws ObjectStoreServiceException
+ */
+ @RequestMapping(value = "/inspector/objectstore.do", method = RequestMethod.GET)
+ public void objectStore(final Model model,
+ @RequestParam("id") final String id,
+ @RequestParam(value = "start", required = false) final Integer startParam,
+ @RequestParam(value = "regex", required = false) final String regex) throws ObjectStoreServiceException {
+ int pageSize = 10;
+ int start = 0;
+
+ if (startParam != null) {
+ start = startParam;
+ }
+ ObjectStore objctStore = objectStoreDao.getObjectStore(id);
+ ResultSetListener rs = objctStore.deliver((long) 0, System.currentTimeMillis());
+ List page = rs.getResult((1 + start), (start + pageSize));
+ final Gson g = new Gson();
+ model.addAttribute("id", id);
+ model.addAttribute("start", start);
+ model.addAttribute("regex", regex);
+ model.addAttribute("nextPage", start + pageSize);
+ model.addAttribute("prevPage", Math.max(0, start - pageSize));
+ model.addAttribute("size", rs.getSize());
+ model.addAttribute("page", listMap(page, new UnaryFunction() {
+
+ @Override
+ public ObjectStoreFile evaluate(final String json) {
+ return g.fromJson(json, ObjectStoreFile.class);
+ }
+ }));
+
+ }
+
+ /**
+ * Gets the object store dao.
+ *
+ * @return the object store dao
+ */
+ public ObjectStoreDao getObjectStoreDao() {
+ return objectStoreDao;
+ }
+
+ /**
+ * Sets the object store dao.
+ *
+ * @param objectStoreDao
+ * the new object store dao
+ */
+ public void setObjectStoreDao(final ObjectStoreDao objectStoreDao) {
+ this.objectStoreDao = objectStoreDao;
+ }
+
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/MetadataObjectRecord.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/MetadataObjectRecord.java
new file mode 100644
index 0000000..3fa3615
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/MetadataObjectRecord.java
@@ -0,0 +1,61 @@
+package eu.dnetlib.data.objectstore.rmi;
+
+import java.io.Serializable;
+
+import com.google.gson.Gson;
+
+public class MetadataObjectRecord implements Serializable {
+
+ /** The Constant serialVersionUID. */
+ private static final long serialVersionUID = -5640196854633924754L;
+
+ public static MetadataObjectRecord initFromJson(final String input) {
+ return new Gson().fromJson(input, MetadataObjectRecord.class);
+ }
+
+ public MetadataObjectRecord() {
+
+ }
+
+ public MetadataObjectRecord(final String id, final String record, final String mime) {
+ super();
+ this.id = id;
+ this.record = record;
+ this.mime = mime;
+ }
+
+ private String id;
+
+ private String record;
+
+ private String mime;
+
+ public String getId() {
+ return id;
+ }
+
+ public String getRecord() {
+ return record;
+ }
+
+ public void setId(final String id) {
+ this.id = id;
+ }
+
+ public void setRecord(final String record) {
+ this.record = record;
+ }
+
+ public String toJSON() {
+ return new Gson().toJson(this);
+ }
+
+ public String getMime() {
+ return mime;
+ }
+
+ public void setMime(final String mime) {
+ this.mime = mime;
+ }
+
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/ObjectStoreFile.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/ObjectStoreFile.java
new file mode 100644
index 0000000..df0ca16
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/ObjectStoreFile.java
@@ -0,0 +1,258 @@
+package eu.dnetlib.data.objectstore.rmi;
+
+import java.io.Serializable;
+
+import com.google.gson.Gson;
+
+/**
+ * The Class ObjectStoreFile.
+ */
+public class ObjectStoreFile implements Serializable {
+
+ /** The Constant serialVersionUID. */
+ private static final long serialVersionUID = -6501291693572693712L;
+
+ /** The metadata related id. */
+ private String metadataRelatedID;
+
+ /** The object id. */
+ private String objectID;
+
+ /** The mime type. */
+ private String mimeType;
+
+ /** The access protocol. */
+ private Protocols accessProtocol;
+
+ /** The object uri . */
+ private String URI;
+
+ /** The username auth. */
+ private String usernameAuth;
+
+ /** The password auth. */
+ private String passwordAuth;
+
+ /** The downloaded url. */
+ private String downloadedURL;
+
+ /** The md5 sum of the file. */
+ private String md5Sum;
+
+ /** The file size. */
+ private long fileSizeKB;
+
+ /**
+ * Gets the downloaded url.
+ *
+ * @return the downloaded url
+ */
+ public String getDownloadedURL() {
+ return downloadedURL;
+ }
+
+ /**
+ * Sets the downloaded url.
+ *
+ * @param downloadedURL
+ * the new downloaded url
+ */
+ public void setDownloadedURL(final String downloadedURL) {
+ this.downloadedURL = downloadedURL.replace("\\u003d", "=").replace("\\u0026", "&");;
+ }
+
+ /**
+ * Instantiates a new object store file.
+ */
+ public ObjectStoreFile() {
+ this.accessProtocol = Protocols.None;
+
+ }
+
+ /**
+ * Return a new instance of the object from a json String.
+ *
+ * @param jsonObject
+ * the json object
+ * @return the object store file
+ */
+ public static ObjectStoreFile createObject(final String jsonObject) {
+ Gson g = new Gson();
+ return g.fromJson(jsonObject, ObjectStoreFile.class);
+ }
+
+ /**
+ * Gets the object id.
+ *
+ * @return the object id
+ */
+ public String getObjectID() {
+ return objectID;
+ }
+
+ /**
+ * Sets the object id.
+ *
+ * @param objectID
+ * the new object id
+ */
+ public void setObjectID(final String objectID) {
+ this.objectID = objectID;
+ }
+
+ /**
+ * Gets the mime type.
+ *
+ * @return the mime type
+ */
+ public String getMimeType() {
+ return mimeType;
+ }
+
+ /**
+ * Sets the mime type.
+ *
+ * @param mimeType
+ * the new mime type
+ */
+ public void setMimeType(final String mimeType) {
+ this.mimeType = mimeType;
+ }
+
+ /**
+ * Gets the access protocol.
+ *
+ * @return the access protocol
+ */
+ public Protocols getAccessProtocol() {
+ return accessProtocol;
+ }
+
+ /**
+ * Sets the access protocol.
+ *
+ * @param accessProtocol
+ * the new access protocol
+ */
+ public void setAccessProtocol(final Protocols accessProtocol) {
+ this.accessProtocol = accessProtocol;
+ }
+
+ /**
+ * Gets the username auth.
+ *
+ * @return the username auth
+ */
+ public String getUsernameAuth() {
+ return usernameAuth;
+ }
+
+ /**
+ * Sets the username auth.
+ *
+ * @param usernameAuth
+ * the new username auth
+ */
+ public void setUsernameAuth(final String usernameAuth) {
+ this.usernameAuth = usernameAuth;
+ }
+
+ /**
+ * Gets the password auth.
+ *
+ * @return the password auth
+ */
+ public String getPasswordAuth() {
+ return passwordAuth;
+ }
+
+ /**
+ * Sets the password auth.
+ *
+ * @param passwordAuth
+ * the new password auth
+ */
+ public void setPasswordAuth(final String passwordAuth) {
+ this.passwordAuth = passwordAuth;
+ }
+
+ /**
+ * Gets the uri.
+ *
+ * @return the uri
+ */
+ public String getURI() {
+ return URI != null ? URI.replace("\\u003d", "=").replace("\\u0026", "&") : "";
+ }
+
+ /**
+ * Sets the uri.
+ *
+ * @param uri
+ * the new uri
+ */
+ public void setURI(final String uri) {
+ if (uri != null) {
+ URI = uri.replace("\\u003d", "=").replace("\\u0026", "&");
+ }
+ }
+
+ /**
+ * Convert the object into a json String.
+ *
+ * @return the string
+ */
+ public String toJSON() {
+ Gson g = new Gson();
+ return g.toJson(this).replace("\\u003d", "=").replace("\\u0026", "&");
+ }
+
+ /**
+ * Gets the related metadata id.
+ *
+ * @return the related metadata id.
+ */
+ public String getMetadataRelatedID() {
+ return metadataRelatedID;
+ }
+
+ /**
+ * Sets the related metadata id.
+ *
+ * @param metadataRelatedID
+ * the related metadata id.
+ */
+ public void setMetadataRelatedID(final String metadataRelatedID) {
+ this.metadataRelatedID = metadataRelatedID;
+ }
+
+ /**
+ * @return the md5Sum
+ */
+ public String getMd5Sum() {
+ return md5Sum;
+ }
+
+ /**
+ * @param md5Sum
+ * the md5Sum to set
+ */
+ public void setMd5Sum(final String md5Sum) {
+ this.md5Sum = md5Sum;
+ }
+
+ /**
+ * @return the fileSizeKB
+ */
+ public long getFileSizeKB() {
+ return fileSizeKB;
+ }
+
+ /**
+ * @param fileSizeKB the fileSizeKB to set
+ */
+ public void setFileSizeKB(long fileSizeKB) {
+ this.fileSizeKB = fileSizeKB;
+ }
+
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/ObjectStoreFileNotFoundException.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/ObjectStoreFileNotFoundException.java
new file mode 100644
index 0000000..699da54
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/ObjectStoreFileNotFoundException.java
@@ -0,0 +1,22 @@
+package eu.dnetlib.data.objectstore.rmi;
+
+/**
+ * The Class ObjectStoreFileNotFoundException.
+ */
+public class ObjectStoreFileNotFoundException extends ObjectStoreServiceException {
+
+ /** The Constant serialVersionUID. */
+ private static final long serialVersionUID = -5418923557898374219L;
+
+ /**
+ * Instantiates a new object store file not found exception.
+ *
+ * @param string
+ * the string
+ */
+ public ObjectStoreFileNotFoundException(final String string) {
+ super(string);
+
+ }
+
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/ObjectStoreService.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/ObjectStoreService.java
new file mode 100644
index 0000000..2aada19
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/ObjectStoreService.java
@@ -0,0 +1,109 @@
+package eu.dnetlib.data.objectstore.rmi;
+
+import java.util.List;
+
+import javax.jws.WebMethod;
+import javax.jws.WebParam;
+import javax.jws.WebService;
+import javax.xml.ws.wsaddressing.W3CEndpointReference;
+
+import eu.dnetlib.common.rmi.BaseService;
+
+/**
+ * Main ObjectStore Service interface.
+ *
+ * @author Sandro La Bruzzo
+ * @version 0.0.1
+ */
+@WebService(targetNamespace = "http://services.dnetlib.eu/")
+public interface ObjectStoreService extends BaseService {
+
+ /**
+ * Returns ResultSet EPR for delivered ObjectStore records in a particular range date.
+ *
+ * Please check service implementations for details on the expected format of the records in the result set epr.
+ *
+ *
+ * This method could be used for a bulk deliver of all objects in the store
+ *
+ *
+ * @param obsId
+ * identifier of the ObjectStore
+ * @param from
+ * the minimum date of the object
+ * @param until
+ * the maximum date of the object
+ * @return a ResultSet EPR. Each element of the result set contains the objIdentifier of a record and its URL for retrieve the
+ * inputstream of the file.
+ * @throws ObjectStoreServiceException
+ * the object store service exception
+ */
+ @WebMethod(operationName = "deliverObjects", action = "deliverObjects")
+ public W3CEndpointReference deliverObjects(@WebParam(name = "obsId") String obsId, @WebParam(name = "from") Long from, @WebParam(name = "until") Long until)
+ throws ObjectStoreServiceException;
+
+ /**
+ * Returns ResultSet EPR for delivered ObjectStore records.
+ *
+ * Please check service implementations for details on the expected format of the records in the result set epr.
+ *
+ *
+ * @param obsId
+ * identifier of the ObjectStore
+ * @param eprId
+ * id of a ResultSet EPR with the identifiers of the interesting objects. Each element of the result set contains the
+ * objIdentifier of a record
+ * @return a ResultSet EPR. Each element of the result set contains the objIdentifier of a record and its URL for retrieve the
+ * inputstream of the file.
+ * @throws ObjectStoreServiceException
+ * the object store service exception
+ */
+ @WebMethod(operationName = "deliverObjectsByIds", action = "deliverObjectsByIds")
+ public W3CEndpointReference deliverObjectsByIds(@WebParam(name = "obsId") String obsId, @WebParam(name = "eprId") W3CEndpointReference eprId)
+ throws ObjectStoreServiceException;
+
+ /**
+ * Returns an URL to retrieve the ObjectStore record.
+ *
+ * @param obsId
+ * identifier of the ObjectStore
+ * @param objectId
+ * the id of the object
+ * @return the URL for retrieve the record
+ * @throws ObjectStoreServiceException
+ * the object store service exception
+ */
+ @WebMethod(operationName = "deliverObject", action = "deliverObject")
+ public String deliverRecord(@WebParam(name = "obsId") String obsId, @WebParam(name = "objectId") String objectId) throws ObjectStoreServiceException;
+
+ /**
+ * Feed the object in the objectStore
+ *
+ * @param obsId
+ * identifier of the ObjectStore
+ * @param objectMetadata
+ * the String serialized of the JSON object ObjectStoreFile
+ * @throws ObjectStoreServiceException
+ */
+
+ @WebMethod(operationName = "feedObject", action = "feedObject")
+ public void feedObject(@WebParam(name = "obsId") String obsId, @WebParam(name = "objectMetadata") String objectMetadata) throws ObjectStoreServiceException;
+
+ /**
+ * Returns list of all stored indices.
+ *
+ * @return list of all stored indices
+ */
+ @WebMethod(operationName = "getListOfObjectStores", action = "getListOfObjectStores")
+ public List getListOfObjectStores();
+
+ /**
+ * Gets the size of the objectStore ID.
+ *
+ * @param obsId
+ * the obs id
+ * @return the size
+ */
+ @WebMethod(operationName = "getSize", action = "getSize")
+ public int getSize(@WebParam(name = "obsId") String obsId) throws ObjectStoreServiceException;
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/ObjectStoreServiceException.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/ObjectStoreServiceException.java
new file mode 100644
index 0000000..97ed419
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/ObjectStoreServiceException.java
@@ -0,0 +1,45 @@
+package eu.dnetlib.data.objectstore.rmi;
+
+import eu.dnetlib.common.rmi.RMIException;
+
+/**
+ * The Class ObjectStoreServiceException.
+ */
+public class ObjectStoreServiceException extends RMIException {
+
+ /** The Constant serialVersionUID. */
+ private static final long serialVersionUID = -7073846285219543315L;
+
+ /**
+ * Instantiates a new object store service exception.
+ *
+ * @param string
+ * the string
+ */
+ public ObjectStoreServiceException(final String string) {
+ super(string);
+ }
+
+ /**
+ * Instantiates a new object store service exception.
+ *
+ * @param string
+ * the string
+ * @param exception
+ * the exception
+ */
+ public ObjectStoreServiceException(final String string, final Throwable exception) {
+ super(string, exception);
+ }
+
+ /**
+ * Instantiates a new object store service exception.
+ *
+ * @param exception
+ * the exception
+ */
+ public ObjectStoreServiceException(final Throwable exception) {
+ super(exception);
+ }
+
+}
diff --git a/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/Protocols.java b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/Protocols.java
new file mode 100644
index 0000000..66a3911
--- /dev/null
+++ b/dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/Protocols.java
@@ -0,0 +1,13 @@
+package eu.dnetlib.data.objectstore.rmi;
+
+/**
+ * The Enum Protocols.
+ */
+ public enum Protocols {
+ None,
+ HTTP,
+ HTTPS,
+ FTP,
+ File_System,
+ FTPS
+}
diff --git a/dnet-data-services/src/main/resources/eu/dnetlib/applicationContext-dnet-objectstore-rmi.xml b/dnet-data-services/src/main/resources/eu/dnetlib/applicationContext-dnet-objectstore-rmi.xml
new file mode 100644
index 0000000..44b5322
--- /dev/null
+++ b/dnet-data-services/src/main/resources/eu/dnetlib/applicationContext-dnet-objectstore-rmi.xml
@@ -0,0 +1,19 @@
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/applicationContext-dnet-modular-objectstore-service.properties b/dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/applicationContext-dnet-modular-objectstore-service.properties
new file mode 100644
index 0000000..0b43727
--- /dev/null
+++ b/dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/applicationContext-dnet-modular-objectstore-service.properties
@@ -0,0 +1,2 @@
+services.objectstore.dao=gridFSObjectstoreDao
+services.objectstore.rsfactory.pagesize=20
\ No newline at end of file
diff --git a/dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/applicationContext-dnet-modular-objectstore-service.xml b/dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/applicationContext-dnet-modular-objectstore-service.xml
new file mode 100644
index 0000000..e8b2ade
--- /dev/null
+++ b/dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/applicationContext-dnet-modular-objectstore-service.xml
@@ -0,0 +1,111 @@
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/filesystem/applicationContext-filesystem-objectstore.properties b/dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/filesystem/applicationContext-filesystem-objectstore.properties
new file mode 100644
index 0000000..b51c409
--- /dev/null
+++ b/dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/filesystem/applicationContext-filesystem-objectstore.properties
@@ -0,0 +1,8 @@
+services.objectStore.mongodb.host=localhost
+services.objectStore.mongodb.port=27017
+services.objectStore.mongodb.db=objectStore
+services.objectStore.mongodb.connectionsPerHost=20
+services.objectStore.mongodb.upsert=true
+services.objectstore.mongodb.integritycheck.cron=0 0 2 1/1 * ? *
+services.objectStore.RESTURI=http://${container.hostname}:${container.port}/${container.context}/mvc/objectStore/retrieve.do
+services.objectstore.dao=fSObjectstoreDao
diff --git a/dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/filesystem/applicationContext-filesystem-objectstore.xml b/dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/filesystem/applicationContext-filesystem-objectstore.xml
new file mode 100644
index 0000000..c0fce46
--- /dev/null
+++ b/dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/filesystem/applicationContext-filesystem-objectstore.xml
@@ -0,0 +1,53 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/inspector/webContext-objectstore-inspector.xml b/dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/inspector/webContext-objectstore-inspector.xml
new file mode 100644
index 0000000..344de29
--- /dev/null
+++ b/dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/inspector/webContext-objectstore-inspector.xml
@@ -0,0 +1,20 @@
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/objectstoreds-template.xml b/dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/objectstoreds-template.xml
new file mode 100644
index 0000000..e11d5aa
--- /dev/null
+++ b/dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/objectstoreds-template.xml
@@ -0,0 +1,24 @@
+
+
+
+
+
+
+ $interpretation$
+
+
+ 0
+ 0
+
+
\ No newline at end of file
diff --git a/dnet-data-services/src/test/java/eu/dnetlib/data/objectstore/filesystem/ConfigurationTestConfig.java b/dnet-data-services/src/test/java/eu/dnetlib/data/objectstore/filesystem/ConfigurationTestConfig.java
new file mode 100644
index 0000000..f0469e3
--- /dev/null
+++ b/dnet-data-services/src/test/java/eu/dnetlib/data/objectstore/filesystem/ConfigurationTestConfig.java
@@ -0,0 +1,51 @@
+package eu.dnetlib.data.objectstore.filesystem;
+
+import java.net.UnknownHostException;
+
+import com.mongodb.MongoClient;
+import com.mongodb.client.MongoDatabase;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+// TODO: Auto-generated Javadoc
+/**
+ * The Class ConfigurationTestConfig.
+ */
+@Configuration
+public class ConfigurationTestConfig {
+
+ /**
+ * Mongo db.
+ *
+ * @return the db
+ * @throws UnknownHostException the unknown host exception
+ */
+ @Bean
+ public MongoDatabase objectstoreMongoDB() throws UnknownHostException {
+ MongoClient mongoClient = new MongoClient("localhost");
+ return mongoClient.getDatabase("objectStoreTest");
+
+ }
+
+
+ @Bean
+ public FileSystemObjectStoreDao fsObjectStoreDAO() {
+ FileSystemObjectStoreDao fileSystemObjectStoreDao = new FileSystemObjectStoreDao();
+
+ fileSystemObjectStoreDao.setObjectStoreRESTURI("http://www.objectstore.com");
+ return fileSystemObjectStoreDao;
+
+ }
+
+ /**
+ * Fs utility.
+ *
+ * @return the file system utility
+ */
+ @Bean
+ public FileSystemUtility fsUtility () {
+ return new FileSystemUtility();
+ }
+
+
+}
diff --git a/dnet-data-services/src/test/java/eu/dnetlib/data/objectstore/filesystem/InputIterator.java b/dnet-data-services/src/test/java/eu/dnetlib/data/objectstore/filesystem/InputIterator.java
new file mode 100644
index 0000000..12588ab
--- /dev/null
+++ b/dnet-data-services/src/test/java/eu/dnetlib/data/objectstore/filesystem/InputIterator.java
@@ -0,0 +1,80 @@
+/**
+ *
+ */
+package eu.dnetlib.data.objectstore.filesystem;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import eu.dnetlib.data.objectstore.modular.ObjectStoreRecord;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
+import eu.dnetlib.data.objectstore.rmi.Protocols;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
+
+/**
+ * @author sandro
+ *
+ */
+public class InputIterator implements Iterable, Iterator {
+
+ private int counter = 0;
+ private String pdfResourcePath = "test.pdf";
+ private Resource testPDF = new ClassPathResource(pdfResourcePath);
+
+ /**
+ * {@inheritDoc}
+ * @see java.util.Iterator#hasNext()
+ */
+ @Override
+ public boolean hasNext() {
+
+ return counter<100;
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see java.util.Iterator#next()
+ */
+ @Override
+ public ObjectStoreRecord next() {
+ try {
+ counter ++;
+ ObjectStoreRecord record = new ObjectStoreRecord();
+ ObjectStoreFile fileMetadata = new ObjectStoreFile();
+ fileMetadata.setAccessProtocol(Protocols.File_System);
+
+ fileMetadata.setDownloadedURL("file://" + pdfResourcePath);
+ fileMetadata.setURI("file://" + pdfResourcePath);
+ fileMetadata.setObjectID("Oggetto_"+counter);
+ fileMetadata.setMimeType("application/pdf");
+ System.out.println("Aggiungo Elemento "+counter);
+ record.setInputStream(testPDF.getInputStream());
+ record.setFileMetadata(fileMetadata );
+ return record;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see java.util.Iterator#remove()
+ */
+ @Override
+ public void remove() {
+ // TODO Auto-generated method stub
+
+ }
+
+ /**
+ * {@inheritDoc}
+ * @see java.lang.Iterable#iterator()
+ */
+ @Override
+ public Iterator iterator() {
+ counter =0;
+ return this;
+ }
+
+}
diff --git a/dnet-data-services/src/test/java/eu/dnetlib/data/objectstore/filesystem/ObjectStoreServiceDAOTest.java b/dnet-data-services/src/test/java/eu/dnetlib/data/objectstore/filesystem/ObjectStoreServiceDAOTest.java
new file mode 100644
index 0000000..fbd6178
--- /dev/null
+++ b/dnet-data-services/src/test/java/eu/dnetlib/data/objectstore/filesystem/ObjectStoreServiceDAOTest.java
@@ -0,0 +1,117 @@
+package eu.dnetlib.data.objectstore.filesystem;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+import com.mongodb.client.MongoDatabase;
+import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
+import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
+import eu.dnetlib.enabling.resultset.ResultSetListener;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * TODO: Test class set to ignored because it requires a mongo server running on localhost. Tests with mocks should be prepared.
+ * @author sandro
+ *
+ */
+@Ignore
+@RunWith(SpringJUnit4ClassRunner.class)
+@ContextConfiguration(classes = ConfigurationTestConfig.class)
+public class ObjectStoreServiceDAOTest {
+ private static final Log log = LogFactory.getLog(ObjectStoreServiceDAOTest.class); // NOPMD by marko on 11/24/08 5:02 PM
+
+ @Autowired
+ private FileSystemObjectStoreDao objectStoreDAO;
+
+ @Autowired
+ private MongoDatabase objectstoreMongoDB;
+
+ private String interpretation = "interp";
+ private String basePath = "/tmp/basePath";
+ //must be 36 chars
+ private String obsID = "test56789111315171921232527293133350";
+ private String pdfPath = "./test.pdf";
+
+ @Before
+ public void setup() throws IOException, ObjectStoreServiceException {
+ Path baseDirPath = FileSystems.getDefault().getPath(basePath);
+ if (!Files.exists(baseDirPath)) {
+ Files.createDirectory(baseDirPath);
+ }
+ objectStoreDAO.createObjectStore(obsID + "_T2JqZWN0U3RvcmVEU1Jlc291cmNlcy9PYmplY3RTdG9yZURTUmVzb3VyY2VUeXBl", interpretation, basePath);
+ }
+
+ @After
+ public void tearDown() throws IOException, ObjectStoreServiceException {
+ objectstoreMongoDB.drop();
+ FileUtils.deleteDirectory(new File(basePath));
+ }
+
+ @Test
+ public void testList() {
+ final List objectStores = this.objectStoreDAO.listObjectStores();
+ for (String o : objectStores)
+ System.out.println(o);
+ assertTrue(objectStores.contains(obsID + "_T2JqZWN0U3RvcmVEU1Jlc291cmNlcy9PYmplY3RTdG9yZURTUmVzb3VyY2VUeXBl"));
+ }
+
+ @Test
+ public void testGet() throws ObjectStoreServiceException {
+ ObjectStore o = objectStoreDAO.getObjectStore(obsID);
+ System.out.println(o.toString());
+ assertNotNull(o);
+ }
+
+ @Test
+ public void testFeed() throws ObjectStoreServiceException, FileNotFoundException {
+ objectStoreDAO.getObjectStore(obsID).feed(new InputIterator(),true);
+ }
+
+ @Test
+ public void testDeliverObject() throws ObjectStoreServiceException {
+ objectStoreDAO.getObjectStore(obsID).feed(new InputIterator(), true);
+ final ObjectStoreFile obj2 = objectStoreDAO.getObjectStore(obsID).deliverObject("Oggetto_2");
+ System.out.println(obj2.toJSON());
+ assertNotNull(obj2);
+ }
+
+ @Test
+ public void testGetStream() throws Exception {
+ objectStoreDAO.getObjectStore(obsID).feed(new InputIterator(), true);
+ ResultSetListener rs = objectStoreDAO.getObjectStore(obsID).deliver(0L, System.currentTimeMillis());
+ for (int i=1; i< (rs.getSize()/10); ) {
+
+ int from = i;
+ int to = Math.max(rs.getSize(), i*10);
+
+ List data = rs.getResult(from, to);
+
+ for (String s: data) {
+ System.out.println(s);
+ }
+
+ i= to ;
+ }
+ }
+
+
+}
diff --git a/dnet-data-services/src/test/resources/test.pdf b/dnet-data-services/src/test/resources/test.pdf
new file mode 100644
index 0000000..948e79a
Binary files /dev/null and b/dnet-data-services/src/test/resources/test.pdf differ