D-Net
/
dnet-core
6
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

imported dnet-objectstore-rmi, dnet-modular-objectstore-service, dnet-fs-objectstore in dnet-data-services

Browse Source
master
Claudio Atzori 5 years ago
parent 337b42e945
commit 7e202d3d79
44 changed files with 4211 additions and 0 deletions
Show Stats Download Patch File Download Diff File

4
dnet-data-services/pom.xml
Unescape Escape View File

@ -28,6 +28,10 @@
<artifactId>commons-beanutils</artifactId>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
</dependency>
<dependency>
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy</artifactId>

383
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/FileSystemObjectStore.java
Unescape Escape View File

@ -0,0 +1,383 @@
package eu.dnetlib.data.objectstore.filesystem;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.regex.Pattern;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.gson.Gson;
import com.mongodb.BasicDBObject;
import com.mongodb.DBObject;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.model.Filters;
import com.mongodb.client.result.DeleteResult;
import eu.dnetlib.data.objectstore.modular.ObjectStoreRecord;
import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;
import eu.dnetlib.data.objectstore.rmi.MetadataObjectRecord;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreFileNotFoundException;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
import eu.dnetlib.enabling.resultset.ResultSetListener;
import eu.dnetlib.miscutils.collections.Pair;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.bson.conversions.Bson;
/**
* The Class FileSystemObjectStore.
*
* @author sandro
*/
public class FileSystemObjectStore implements ObjectStore {
/**
*
*/
private static final String URI_FIELD = "uri";
/**
*
*/
private static final String FS_PATH_FIELD = "fsPath";
/** The Constant log. */
private static final Log log = LogFactory.getLog(FileSystemObjectStore.class); // NOPMD by marko on 11/24/08 5:02 PM
/** The id. */
private final String id;
/** The interpretation. */
private final String interpretation;
/** The base path. */
private final String basePath;
/** The base uri. */
private final String baseURI;
/** The mongo metadata. */
private final MongoCollection<DBObject> mongoMetadata;
/**
* Instantiates a new file system object store.
*
* @param identifier
* the identifier
* @param interpretation
* the interpretation
* @param basePath
* the base path
* @param mongoMetadata
* the mongo metadata
* @param baseURI
* the base uri
*/
public FileSystemObjectStore(final String identifier, final String interpretation, final String basePath, final MongoCollection<DBObject> mongoMetadata,
final String baseURI) {
this.id = identifier;
this.basePath = basePath;
this.interpretation = interpretation;
this.mongoMetadata = mongoMetadata;
this.baseURI = baseURI;
}
/**
* {@inheritDoc}
*
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#getId()
*/
@Override
public String getId() {
return this.id;
}
/**
* {@inheritDoc}
*
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#getInterpretation()
*/
@Override
public String getInterpretation() {
return this.interpretation;
}
/**
* {@inheritDoc}
*
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#feed(java.lang.Iterable, boolean)
*/
@Override
public int feed(final Iterable<ObjectStoreRecord> records, final boolean incremental) throws ObjectStoreServiceException {
if (records == null)
return 0;
Path baseDirPath = FileSystems.getDefault().getPath(getBasePath()).resolve(getId());
if (!Files.exists(baseDirPath))
throw new ObjectStoreServiceException("Error can't feed objects because the folder " + baseDirPath + " does not exist");
int addedCounter = 0;
int nulls = 0;
for (ObjectStoreRecord record : records) {
String url = feedObject(record);
if (StringUtils.isNotBlank(url)) {
addedCounter++;
} else {
nulls++;
}
}
if (nulls > 0) {
log.warn(String.format("Found %s null records", nulls));
}
return addedCounter;
}
/**
* {@inheritDoc}
*
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#feedMetadataRecord(java.lang.Iterable, boolean)
*
* This method handles the case of web crawl files and other cases when the metadata in mdstores are also the objects to put into the objectstores.
*/
@Override
public int feedMetadataRecord(final Iterable<MetadataObjectRecord> records, final boolean incremental) throws ObjectStoreServiceException {
Iterable<ObjectStoreRecord> it = Iterables.transform(records, mor -> {
ObjectStoreRecord r = new ObjectStoreRecord();
r.setInputStream(new ByteArrayInputStream(mor.getRecord().getBytes()));
ObjectStoreFile fileMetadata = new ObjectStoreFile();
fileMetadata.setObjectID(mor.getId());
fileMetadata.setMimeType(mor.getMime());
r.setFileMetadata(fileMetadata);
return r;
});
return feed(it, incremental);
}
/**
* {@inheritDoc}
*
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#feedObjectRecord(eu.dnetlib.data.objectstore.modular.ObjectStoreRecord)
*/
@Override
public String feedObjectRecord(final ObjectStoreRecord record) throws ObjectStoreServiceException {
return feedObject(record);
}
private String feedObject(final ObjectStoreRecord record) {
if (record != null) {
String objectIdentifier = record.getFileMetadata().getObjectID();
if (StringUtils.isNotBlank(objectIdentifier)) {
final Path objResolvedPath = FileSystemUtility.objectStoreFilePath(basePath, id, objectIdentifier);
if (Files.notExists(objResolvedPath)) {
try {
log.debug("Creation of folder " + objResolvedPath.getParent());
Files.createDirectories(objResolvedPath.getParent());
log.debug("Folder " + objResolvedPath.getParent() + " created");
String md5Sum = null;
Long size = new Long(0);
if (record.getInputStream() != null) {
Pair<String, Long> infos = FileSystemUtility.saveAndGenerateMD5(record.getInputStream(), objResolvedPath);
md5Sum = infos.getKey();
size = infos.getValue();
}
final String url =
ModularObjectStoreRESTService.retrieveURL(getBaseURI(), getBasePath(), getId(), record.getFileMetadata().getObjectID());
if (StringUtils.isNotBlank(md5Sum)) {
double timestamp = System.currentTimeMillis();
BasicDBObject metadata = new BasicDBObject();
metadata.put("id", record.getFileMetadata().getObjectID());
metadata.put("mime", record.getFileMetadata().getMimeType());
metadata.put("originalObject", record.getFileMetadata().toJSON());
metadata.put("timestamp", timestamp);
metadata.put("md5Sum", md5Sum);
metadata.put("size", size);
metadata.put(FS_PATH_FIELD, objResolvedPath.toAbsolutePath().toString());
metadata.put(URI_FIELD, url);
log.debug("saving metadata object to the collection: " + metadata.toString());
mongoMetadata.insertOne(metadata);
}
return url;
} catch (Exception e) {
log.error("Something bad happen on inserting Record", e);
log.error("Record: " + new Gson().toJson(record.getFileMetadata()));
} finally {
if (record.getInputStream() != null) {
try {
record.getInputStream().close();
} catch (Exception e) {
log.error("Error on close inputStream", e);
}
}
}
} else {
log.debug("The File in the path" + objResolvedPath.getParent() + "exists ");
}
}
}
return null;
}
/**
* {@inheritDoc}
*
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#deliver(java.lang.Long, java.lang.Long)
*/
@Override
public ResultSetListener deliver(final Long from, final Long until) throws ObjectStoreServiceException {
FileSystemObjectStoreResultSetListener resultSet = new FileSystemObjectStoreResultSetListener();
resultSet.setBaseURI(getBaseURI());
resultSet.setMongoCollection(mongoMetadata);
resultSet.setObjectStoreID(getId());
resultSet.setFromDate(from);
resultSet.setUntilDate(until);
resultSet.setBasePath(getBasePath());
return resultSet;
}
/**
* {@inheritDoc}
*
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#deliverIds(java.lang.Iterable)
*/
@Override
public ResultSetListener deliverIds(final Iterable<String> ids) throws ObjectStoreServiceException {
FileSystemObjectStoreResultSetListener resultSet = new FileSystemObjectStoreResultSetListener();
resultSet.setBaseURI(getBaseURI());
resultSet.setMongoCollection(mongoMetadata);
resultSet.setObjectStoreID(getId());
resultSet.setRecords(Lists.newArrayList(ids));
resultSet.setBasePath(basePath);
return resultSet;
}
/**
* {@inheritDoc}
*
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#deliverObject(java.lang.String)
*/
@Override
public ObjectStoreFile deliverObject(final String objectId) throws ObjectStoreServiceException {
Bson query = Filters.eq("id", objectId);
DBObject resultQuery = mongoMetadata.find(query).first();
checkAndGetFsPathField(resultQuery, objectId);
return ObjectStoreFileUtility.build(resultQuery, getBaseURI(), getId(), basePath);
}
private String checkAndGetFsPathField(final DBObject resultQuery, final String objectId) throws ObjectStoreServiceException {
if (resultQuery == null || !resultQuery.containsField(FS_PATH_FIELD))
throw new ObjectStoreFileNotFoundException("Object with identifier :" + objectId + " not found or missing " + FS_PATH_FIELD + " field");
String pathStr = (String) resultQuery.get(FS_PATH_FIELD);
if (StringUtils.isBlank(pathStr))
throw new ObjectStoreFileNotFoundException("Object with identifier :" + objectId + " with blank " + FS_PATH_FIELD);
return pathStr;
}
/**
* {@inheritDoc}
*
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#getSize()
*/
@Override
public int getSize() throws ObjectStoreServiceException {
return (int) mongoMetadata.count();
}
/**
* {@inheritDoc}
*
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#deleteObject(java.lang.String)
*/
@Override
public void deleteObject(final String objectId) throws ObjectStoreServiceException {
Bson query = Filters.eq("id", objectId);
DBObject response = mongoMetadata.find(query).first();
String pathStr = checkAndGetFsPathField(response, objectId);
Path path = FileSystems.getDefault().getPath(pathStr);
if (Files.notExists(path))
throw new ObjectStoreFileNotFoundException("Object with identifier :" + objectId + " not found in the assigned path " + path);
try {
Files.delete(path);
} catch (IOException e) {
throw new ObjectStoreServiceException("An error occurs on delete file ", e);
}
mongoMetadata.deleteOne(query);
}
/**
* {@inheritDoc}
*
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#getObject(java.lang.String)
*/
@Override
public String getObject(final String recordId) throws ObjectStoreServiceException {
Bson query = Filters.eq("id", recordId);
DBObject response = mongoMetadata.find(query).first();
if (response == null || !response.containsField(URI_FIELD))
return null;
return (String) response.get(URI_FIELD);
}
/**
* {@inheritDoc}
*
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStore#existIDStartsWith(java.lang.String)
*/
@Override
public boolean existIDStartsWith(final String startId) throws ObjectStoreServiceException {
Bson query = Filters.regex("id", Pattern.compile(startId));
return mongoMetadata.count(query) > 0;
}
@Override
public boolean dropContent() throws ObjectStoreServiceException {
if (getBasePath() == null) {
throw new ObjectStoreServiceException("Error on dropping object store base_path required");
}
final Path baseDirPath = FileSystems.getDefault().getPath(getBasePath()).resolve(getId());
try {
FileSystemUtility.deleteFolderRecursive(baseDirPath);
} catch (IOException e) {
throw new ObjectStoreServiceException("Error on dropping store ", e);
}
log.info("Deleted folder" + baseDirPath.toString());
if (!Files.exists(baseDirPath)) {
log.info("Recreating folder " + baseDirPath);
try {
Files.createDirectory(baseDirPath);
} catch (IOException e) {
throw new ObjectStoreServiceException("Error on dropping store ", e);
}
}
final DeleteResult deleteResult = this.mongoMetadata.deleteMany(new BasicDBObject());
log.info("Dropped content for object store " + id + ". " + deleteResult.getDeletedCount() + " object(s) deleted.");
return true;
}
@Override
public String toString() {
return "FileSystemObjectStore{" +
"id='" + getId() + '\'' +
", interpretation='" + getInterpretation() + '\'' +
", basePath='" + getBasePath() + '\'' +
", baseURI='" + getBaseURI() + '\'' +
'}';
}
/**
* Gets the base uri.
*
* @return the baseURI
*/
public String getBaseURI() {
return baseURI;
}
public String getBasePath() {
return basePath;
}
}

212
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/FileSystemObjectStoreDao.java
Unescape Escape View File

@ -0,0 +1,212 @@
package eu.dnetlib.data.objectstore.filesystem;
import java.io.IOException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import javax.annotation.Resource;
import com.mongodb.BasicDBObject;
import com.mongodb.DBObject;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.Filters;
import com.mongodb.client.result.UpdateResult;
import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;
import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreFileNotFoundException;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
import eu.dnetlib.miscutils.collections.MappedCollection;
import eu.dnetlib.miscutils.functional.UnaryFunction;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.bson.conversions.Bson;
/**
* @author sandro
*
*/
public class FileSystemObjectStoreDao implements ObjectStoreDao {
public static final String INTERPRETATION_FIELD = "interpretation";
public final static String OBJECTSTORE_METADATA_NAME_FIELD = "metadataObjectStore";
public final static String OBJECTSTORE_ID_FIELD = "obsId";
public final static String BASE_PATH_FIELD = "basePath";
private static final Log log = LogFactory.getLog(FileSystemObjectStoreDao.class); // NOPMD by marko on 11/24/08 5:02 PM
private static final String OBJECTSTORE_PROFILE_SUFFIX = "_T2JqZWN0U3RvcmVEU1Jlc291cmNlcy9PYmplY3RTdG9yZURTUmVzb3VyY2VUeXBl";
@Resource(name="objectstoreMongoDB")
private MongoDatabase db;
private boolean upsert;
private String objectStoreRESTURI;
/**
* {@inheritDoc}
* @throws ObjectStoreServiceException
*
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao#getObjectStore(java.lang.String)
*/
@Override
public ObjectStore getObjectStore(final String obsId) throws ObjectStoreServiceException {
String currentId = obsId.substring(0, 36);
String find_id = obsId;
if (find_id.length() == 36) {
find_id += OBJECTSTORE_PROFILE_SUFFIX;
}
MongoCollection<DBObject> metadataObjectStore = getDb().getCollection(OBJECTSTORE_METADATA_NAME_FIELD, DBObject.class);
Bson query = Filters.eq(OBJECTSTORE_ID_FIELD, find_id);
log.debug("QUERY :" + query.toString());
DBObject resultQuery = metadataObjectStore.find(query).first();
log.debug("result " + resultQuery);
if ((resultQuery == null)) throw new ObjectStoreFileNotFoundException("the objectStore with identifier: "+obsId+" was not found");
final String basePath = resultQuery.get(BASE_PATH_FIELD).toString();
final String interpretation = resultQuery.get("interpretation").toString();
if (!resultQuery.containsField(BASE_PATH_FIELD) || StringUtils.isBlank(basePath))
throw new ObjectStoreServiceException("Can't Get Objectstore, the metadata doesn't contain info about the basepath");
final MongoCollection<DBObject> collection = getDb().getCollection(currentId, DBObject.class);
return new FileSystemObjectStore(currentId, interpretation, basePath, collection, objectStoreRESTURI);
}
/**
* {@inheritDoc}
*
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao#listObjectStores()
*/
@Override
public List<String> listObjectStores() {
MongoCollection<DBObject> metadata = getDb().getCollection(OBJECTSTORE_METADATA_NAME_FIELD, DBObject.class);
return MappedCollection.listMap(metadata.find(), new UnaryFunction<String, DBObject>() {
@Override
public String evaluate(final DBObject object) {
return (String) object.get(OBJECTSTORE_ID_FIELD);
}
});
}
/**
* {@inheritDoc}
*
* @throws ObjectStoreServiceException
*/
@Override
public boolean createObjectStore(final String obsId, final String interpretation, final String basePath) throws ObjectStoreServiceException {
log.debug(String.format("Create object Store method\n\t Id:%s Interpretation:%s BasePath : %s", obsId, interpretation, basePath) );
if (StringUtils.isBlank(basePath)) throw new ObjectStoreServiceException("Can't create the object store: the base path cannot be blank");
Path path = FileSystems.getDefault().getPath(basePath);
if (!Files.exists(path) || !Files.isDirectory(path))
throw new ObjectStoreServiceException("Can't create the object store: base path: '" + basePath + "' doesn't exist or it is not a folder");
try {
String currentObsId = obsId.substring(0, 36);
log.debug("Cleaned objectStore Id " + currentObsId);
if (Files.exists(path.resolve(currentObsId)))
throw new ObjectStoreServiceException("Can't create the object store: base path: '" + path.resolve(currentObsId) + "' already exists");
Files.createDirectory(path.resolve(currentObsId));
MongoCollection<DBObject> coll = getDb().getCollection(OBJECTSTORE_METADATA_NAME_FIELD, DBObject.class);
final BasicDBObject obj = new BasicDBObject();
obj.put(OBJECTSTORE_ID_FIELD, obsId);
obj.put(INTERPRETATION_FIELD, interpretation);
obj.put(BASE_PATH_FIELD, basePath);
coll.insertOne(obj);
MongoCollection<DBObject> objectStore = getDb().getCollection(currentObsId, DBObject.class);
objectStore.createIndex(new BasicDBObject("id", 1));
objectStore.createIndex(new BasicDBObject("timestamp", 1));
return true;
} catch (Throwable e) {
throw new ObjectStoreServiceException("Can't Create the object Store id: '" + obsId, e);
}
}
/**
* {@inheritDoc}
*
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao#updateObjectStore(java.lang.String, java.lang.String)
*/
@Override
public boolean updateObjectStore(final String obsId, final String interpretation) {
MongoCollection<DBObject> coll = getDb().getCollection(OBJECTSTORE_METADATA_NAME_FIELD, DBObject.class);
final BasicDBObject obj = new BasicDBObject();
obj.put("$set", new BasicDBObject(INTERPRETATION_FIELD, interpretation));
final UpdateResult updateResult = coll.updateOne(Filters.eq(OBJECTSTORE_ID_FIELD, obsId), obj);
if (updateResult.isModifiedCountAvailable()) {
log.debug("Matched / Modified " + updateResult.getMatchedCount() + " / " + updateResult.getModifiedCount());
}
return true;
}
/**
* {@inheritDoc}
*
* @throws ObjectStoreServiceException
* @see eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao#deleteObjectStore(java.lang.String)
*/
@Override
public boolean deleteObjectStore(final String obsId) throws ObjectStoreServiceException {
MongoCollection<DBObject> coll = getDb().getCollection(OBJECTSTORE_METADATA_NAME_FIELD, DBObject.class);
Bson query = Filters.eq(OBJECTSTORE_ID_FIELD, obsId);
DBObject resultQuery = coll.find(query).first();
String basePath = checkAndGetFsPathField(resultQuery, obsId);
String currentObsId = obsId.substring(0, 36);
Path basePathFS = FileSystems.getDefault().getPath(basePath, currentObsId);
if (!Files.exists(basePathFS))
throw new ObjectStoreServiceException("Can't Delete ObjectStore " + obsId + ": the base path does not exist :" + basePathFS);
try {
FileSystemUtility.deleteFolderRecursive(basePathFS);
} catch (IOException e) {
throw new ObjectStoreServiceException("Can't Delete ObjectStore " + obsId, e);
}
coll.deleteOne(Filters.eq(OBJECTSTORE_ID_FIELD, obsId));
getDb().getCollection(obsId).drop();
return true;
}
@Override
public boolean dropContent(final String obsId) throws ObjectStoreServiceException {
return getObjectStore(obsId).dropContent();
}
private String checkAndGetFsPathField(final DBObject resultQuery, final String objectStoreID) throws ObjectStoreServiceException {
if (resultQuery == null || !resultQuery.containsField(BASE_PATH_FIELD))
throw new ObjectStoreServiceException("ObjectStore with identifier :" + objectStoreID + " not found or missing " + BASE_PATH_FIELD + " field");
String pathStr = (String) resultQuery.get(BASE_PATH_FIELD);
if (StringUtils.isBlank(pathStr))
throw new ObjectStoreServiceException("ObjectStore with identifier :" + objectStoreID + " with blank " + BASE_PATH_FIELD);
return pathStr;
}
public boolean isUpsert() {
return upsert;
}
public void setUpsert(final boolean upsert) {
this.upsert = upsert;
}
public String getObjectStoreRESTURI() {
return objectStoreRESTURI;
}
public void setObjectStoreRESTURI(final String objectStoreRESTURI) {
this.objectStoreRESTURI = objectStoreRESTURI;
}
public MongoDatabase getDb() {
return db;
}
public void setDb(final MongoDatabase db) {
this.db = db;
}
}

350
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/FileSystemObjectStoreResultSetListener.java
Unescape Escape View File

@ -0,0 +1,350 @@
package eu.dnetlib.data.objectstore.filesystem;
import java.util.List;
import com.google.common.collect.Lists;
import com.mongodb.DBObject;
import com.mongodb.client.FindIterable;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoCursor;
import com.mongodb.client.model.Filters;
import com.mongodb.client.model.Sorts;
import eu.dnetlib.enabling.resultset.ResultSet;
import eu.dnetlib.enabling.resultset.ResultSetAware;
import eu.dnetlib.enabling.resultset.ResultSetListener;
import eu.dnetlib.miscutils.collections.MappedCollection;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.bson.conversions.Bson;
/**
* The listener interface for receiving fileSystemObjectStoreResultSet events.
* The class that is interested in processing a fileSystemObjectStoreResultSet
* event implements this interface, and the object created
* with that class is registered with a component using the
* component's <code>addFileSystemObjectStoreResultSetListener<code> method. When
* the fileSystemObjectStoreResultSet event occurs, that object's appropriate
* method is invoked.
*
* @author sandro
*/
public class FileSystemObjectStoreResultSetListener implements ResultSetListener, ResultSetAware {
/** The Constant log. */
private static final Log log = LogFactory.getLog(FileSystemObjectStoreResultSetListener.class); // NOPMD by marko on 11/24/08 5:02 PM
/** The from date. */
private Long fromDate;
/** The until date. */
private Long untilDate;
/** The records. */
private List<String> records;
/** The object store id. */
private String objectStoreID;
/** The mongo collection. */
private MongoCollection<DBObject> mongoCollection;
/** The base uri. */
private String baseURI;
/**
* The base path
*/
private String basePath;
/** The current size. */
private int currentSize = -1;
/** The current cursor. */
private MongoCursor<DBObject> currentCursor;
/** The cursor position. */
private long cursorPosition;
/**
* {@inheritDoc}
* @see eu.dnetlib.enabling.resultset.TypedResultSetListener#getResult(int, int)
*/
@Override
public List<String> getResult(final int from, final int to) {
if (log.isDebugEnabled()) {
log.debug(String.format("ObjectStoreId :%s, from: %d, to: %d", objectStoreID, from, to));
}
if (records != null) {
List<String> ids = Lists.newArrayList();
for (int i = from-1; i < Math.min(records.size(),to); i++) {
ids.add(records.get(i));
}
Bson q = Filters.in("id", ids);
FindIterable<DBObject> res = getMongoCollection().find(q);
return MappedCollection.listMap(res, ObjectStoreFileUtility.asJSON(getBaseURI(), getObjectStoreID(), getBasePath()));
} else if ((fromDate != null) && (untilDate != null)) {
if ((currentCursor == null) || (cursorPosition > from)) {
createCurrentCursor();
}
while (cursorPosition < from) {
currentCursor.next();
cursorPosition++;
}
List<DBObject> result = Lists.newArrayList();
for (int i = from; i <= to; i++) {
if (currentCursor.hasNext()) {
result.add(currentCursor.next());
cursorPosition++;
}
}
return MappedCollection.listMap(result, ObjectStoreFileUtility.asJSON(getBaseURI(), getObjectStoreID(), getBasePath()));
}
throw new IllegalArgumentException("Missing parameters on Delivery must provide either from, to, or ObjectStoreIDs");
}
/**
* Creates the current cursor.
*/
private void createCurrentCursor() {
Bson timestampQuery = Filters.and(Filters.gt("timestamp", fromDate.doubleValue()), Filters.lt("timestamp", untilDate.doubleValue()));
if (currentCursor != null) {
currentCursor.close();
}
currentCursor = getMongoCollection().find(timestampQuery).sort(Sorts.orderBy(Filters.eq("_id", 1))).iterator();
cursorPosition = 1;
}
/**
* {@inheritDoc}
* @see eu.dnetlib.enabling.resultset.TypedResultSetListener#getSize()
*/
@Override
public int getSize() {
if (currentSize == -1) {
currentSize = calculateSize();
}
return Math.max(0, currentSize - 1);
}
/**
* Calculate size.
*
* @return the int
*/
private int calculateSize() {
if (records != null) {
Bson query = Filters.in("id", records);
return (int) getMongoCollection().count(query);
} else if ((fromDate != null) && (untilDate != null)) {
Bson timestampQuery = Filters.and(Filters.gt("timestamp", fromDate.doubleValue()), Filters.lt("timestamp", untilDate.doubleValue()));
return (int) getMongoCollection().count(timestampQuery);
}
return 0;
}
/**
* {@inheritDoc}
* @see eu.dnetlib.enabling.resultset.ResultSetAware#setResultSet(eu.dnetlib.enabling.resultset.ResultSet)
*/
@Override
public void setResultSet(final ResultSet resultSet) {
resultSet.close();
}
/**
* Gets the from date.
*
* @return the fromDate
*/
public Long getFromDate() {
return fromDate;
}
/**
* Sets the from date.
*
* @param fromDate the fromDate to set
*/
public FileSystemObjectStoreResultSetListener setFromDate(final Long fromDate) {
this.fromDate = fromDate;
return this;
}
/**
* Gets the until date.
*
* @return the untilDate
*/
public Long getUntilDate() {
return untilDate;
}
/**
* Sets the until date.
*
* @param untilDate the untilDate to set
*/
public FileSystemObjectStoreResultSetListener setUntilDate(final Long untilDate) {
this.untilDate = untilDate;
return this;
}
/**
* Gets the records.
*
* @return the records
*/
public List<String> getRecords() {
return records;
}
/**
* Sets the records.
*
* @param records the records to set
*/
public void setRecords(final List<String> records) {
this.records = records;
}
/**
* Gets the object store id.
*
* @return the objectStoreID
*/
public String getObjectStoreID() {
return objectStoreID;
}
/**
* Sets the object store id.
*
* @param objectStoreID the objectStoreID to set
*/
public void setObjectStoreID(final String objectStoreID) {
this.objectStoreID = objectStoreID;
}
/**
* Gets the base uri.
*
* @return the baseURI
*/
public String getBaseURI() {
return baseURI;
}
/**
* Sets the base uri.
*
* @param baseURI the baseURI to set
*/
public void setBaseURI(final String baseURI) {
this.baseURI = baseURI;
}
/**
* Gets the current size.
*
* @return the currentSize
*/
public int getCurrentSize() {
return currentSize;
}
/**
* Sets the current size.
*
* @param currentSize the currentSize to set
*/
public void setCurrentSize(final int currentSize) {
this.currentSize = currentSize;
}
/**
* Gets the current cursor.
*
* @return the currentCursor
*/
public MongoCursor<DBObject> getCurrentCursor() {
return currentCursor;
}
/**
* Sets the current cursor.
*
* @param currentCursor the currentCursor to set
*/
public void setCurrentCursor(final MongoCursor<DBObject> currentCursor) {
this.currentCursor = currentCursor;
}
/**
* Gets the cursor position.
*
* @return the cursorPosition
*/
public long getCursorPosition() {
return cursorPosition;
}
/**
* Sets the cursor position.
*
* @param cursorPosition the cursorPosition to set
*/
public void setCursorPosition(final long cursorPosition) {
this.cursorPosition = cursorPosition;
}
/**
* Gets the mongo collection.
*
* @return the mongo collection
*/
public MongoCollection<DBObject> getMongoCollection() {
return mongoCollection;
}
/**
* Sets the mongo collection.
*
* @param mongoCollection the new mongo collection
*/
public void setMongoCollection(final MongoCollection<DBObject> mongoCollection) {
this.mongoCollection = mongoCollection;
}
public String getBasePath() {
return basePath;
}
public void setBasePath(final String basePath) {
this.basePath = basePath;
}
}

78
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/FileSystemUtility.java
Unescape Escape View File

@ -0,0 +1,78 @@
package eu.dnetlib.data.objectstore.filesystem;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import eu.dnetlib.miscutils.collections.Pair;
import eu.dnetlib.miscutils.functional.xml.DnetXsltFunctions;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
/**
* The Class FileSystemUtility.
*
* @author sandro
*/
public class FileSystemUtility {
private static final Log log = LogFactory.getLog(FileSystemUtility.class); // NOPMD by marko on 11/24/08 5:02 PM
public static Pair<String, Long> saveAndGenerateMD5(final InputStream inputStream, final Path filePath) {
if(inputStream==null) return null;
String md5 = null;
long size = 0;
try {
Files.copy(inputStream, filePath);
FileInputStream fis = new FileInputStream(filePath.toFile());
md5 = org.apache.commons.codec.digest.DigestUtils.md5Hex(fis);
fis.close();
size = Files.size(filePath);
} catch (IOException e1) {
log.error(e1);
}
return new Pair<String, Long>(md5, size);
}
/**
* Delete folder recursive.
*
* @param path the path
* @return true, if successful
* @throws IOException Signals that an I/O exception has occurred.
*/
public static boolean deleteFolderRecursive(final Path path) throws IOException {
Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
Files.delete(file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(final Path dir, final IOException exc) throws IOException {
Files.delete(dir);
return FileVisitResult.CONTINUE;
}
});
return true;
}
public static Path objectStoreFilePath(final String basePath, final String objectStoreId, final String objectIdentifier) {
final Path baseDirPath = FileSystems.getDefault().getPath(basePath).resolve(objectStoreId);
final String md5id = DnetXsltFunctions.md5(objectIdentifier);
final String firstLevel = StringUtils.substring(md5id, 0, 2);
final String secondLevel = StringUtils.substring(md5id, 2, 4);
final String fileName = StringUtils.substring(md5id, 4) + ".obj";
return baseDirPath.resolve(firstLevel).resolve(secondLevel).resolve(fileName);
}
}

55
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/IndexIntegrityChecker.java
Unescape Escape View File

@ -0,0 +1,55 @@
package eu.dnetlib.data.objectstore.filesystem;
import com.mongodb.BasicDBObject;
import com.mongodb.DBObject;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.IndexOptions;
import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import static eu.dnetlib.data.objectstore.filesystem.FileSystemObjectStoreDao.OBJECTSTORE_ID_FIELD;
import static eu.dnetlib.data.objectstore.filesystem.FileSystemObjectStoreDao.OBJECTSTORE_METADATA_NAME_FIELD;
/**
* Created by claudio on 15/09/16.
*/
public class IndexIntegrityChecker {
private static final Log log = LogFactory.getLog(IndexIntegrityChecker.class);
@Autowired
private ObjectStoreDao dao;
public void check() {
checkIndexes();
}
private void checkIndexes() {
log.info("objectStore indexes integrity start");
final MongoDatabase db = ((FileSystemObjectStoreDao) dao).getDb();
final IndexOptions bg = new IndexOptions().background(true);
for (String objectStoreId : dao.listObjectStores()) {
final String id = StringUtils.substringBefore(objectStoreId, "_");
final MongoCollection<DBObject> objectStore = db.getCollection(id, DBObject.class);
if (log.isDebugEnabled()) {
log.debug(String.format("creating index (id, timestamp) on objectStore %s", id));
}
objectStore.createIndex(new BasicDBObject("id", 1), bg);
objectStore.createIndex(new BasicDBObject("timestamp", 1), bg);
}
if (log.isDebugEnabled()) {
log.debug(String.format("creating index (%s) on %s", OBJECTSTORE_ID_FIELD, OBJECTSTORE_METADATA_NAME_FIELD));
}
final MongoCollection<DBObject> metadata = db.getCollection(OBJECTSTORE_METADATA_NAME_FIELD, DBObject.class);
metadata.createIndex(new BasicDBObject(OBJECTSTORE_ID_FIELD, 1), bg);
log.info("objectStore indexes integrity completed");
}
}

79
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/ModularObjectStoreRESTService.java
Unescape Escape View File

@ -0,0 +1,79 @@
package eu.dnetlib.data.objectstore.filesystem;
import java.io.*;
import java.net.URLEncoder;
import java.nio.file.Files;
import java.nio.file.Path;
import javax.servlet.http.HttpServletResponse;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
import eu.dnetlib.miscutils.datetime.HumanTime;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
/**
* The Class ModularObjectStoreRESTService implement the controller REST of the object Store.
*/
@Controller
public class ModularObjectStoreRESTService {
private static final Log log = LogFactory.getLog(ModularObjectStoreRESTService.class); // NOPMD by marko on 11/24/08 5:02 PM
public static String retrieveURL(final String baseURI, final String basePath, final String objectStoreId, final String objectId)
throws UnsupportedEncodingException {
final StringBuilder sb = new StringBuilder(baseURI)
.append("?objectStore=" + encode(objectStoreId))
.append("&objectId=" + encode(objectId))
.append("&basePath=" + encode(basePath));
return sb.toString();
}
private static String encode(final String s) throws UnsupportedEncodingException {
return URLEncoder.encode(s, "UTF-8");
}
/**
*
* @param res
* @param basePath
* @param objectStoreId
* @param objectId
* @throws IOException
* @throws ObjectStoreServiceException
*/
@RequestMapping(value = "/**/objectStore/retrieve.do")
public void retrieve(final HttpServletResponse res,
@RequestParam(value = "basePath", required = true) final String basePath,
@RequestParam(value = "objectStore", required = true) final String objectStoreId,
@RequestParam(value = "objectId", required = true) final String objectId) throws IOException, ObjectStoreServiceException {
final long start = System.currentTimeMillis();
final Path path = FileSystemUtility.objectStoreFilePath(basePath, objectStoreId, objectId);
if (!Files.exists(path) || !Files.isReadable(path)) {
final String msg = String.format("Object with identifier: %s not found the in %s", objectId, path);
res.sendError(HttpServletResponse.SC_NOT_FOUND, msg);
log.warn(msg);
} else {
try (final InputStream is = new BufferedInputStream(new FileInputStream(path.toFile()))) {
final long size = Files.size(path);
res.setHeader("Content-Length", String.valueOf(size));
IOUtils.copyLarge(is, res.getOutputStream());
if (log.isDebugEnabled()) {
log.debug(String.format("retrive.do completed in %s, objId: %s", HumanTime.exactly(System.currentTimeMillis() - start), objectId));
}
} catch (IOException e) {
final String msg = "unable to close input Stream";
res.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, msg);
log.error(msg, e);
}
}
}
}

34
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/MongoFSOptionsFactory.java
Unescape Escape View File

@ -0,0 +1,34 @@
package eu.dnetlib.data.objectstore.filesystem;
import com.mongodb.MongoClientOptions;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.FactoryBean;
public class MongoFSOptionsFactory implements FactoryBean<MongoClientOptions> {
private int connectionsPerHost;
@Override
public MongoClientOptions getObject() throws BeansException {
return MongoClientOptions.builder().connectionsPerHost(connectionsPerHost).build();
}
@Override
public Class<MongoClientOptions> getObjectType() {
return MongoClientOptions.class;
}
@Override
public boolean isSingleton() {
return false;
}
public int getConnectionsPerHost() {
return connectionsPerHost;
}
public void setConnectionsPerHost(final int connectionsPerHost) {
this.connectionsPerHost = connectionsPerHost;
}
}

57
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/filesystem/ObjectStoreFileUtility.java
Unescape Escape View File

@ -0,0 +1,57 @@
package eu.dnetlib.data.objectstore.filesystem;
import java.io.UnsupportedEncodingException;
import com.mongodb.DBObject;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
import eu.dnetlib.data.objectstore.rmi.Protocols;
import eu.dnetlib.miscutils.functional.UnaryFunction;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
/**
* The Class ObjectStoreFileBuilder generates an objectStoreFile bean
*
*/
public class ObjectStoreFileUtility {
private static final int KB_SIZE = 1024;
/** The Constant log. */
private static final Log log = LogFactory.getLog(ObjectStoreFileUtility.class);
public static ObjectStoreFile build(final DBObject metadata, final String baseURI, final String objectStoreID, final String basePath) {
String originalFile = (String) metadata.get("originalObject");
ObjectStoreFile original = ObjectStoreFile.createObject(originalFile);
ObjectStoreFile newFile = new ObjectStoreFile();
newFile.setObjectID((String) metadata.get("id"));
newFile.setAccessProtocol(Protocols.HTTP);
newFile.setMimeType((String) metadata.get("mime"));
newFile.setMd5Sum((String) metadata.get("md5Sum"));
try {
newFile.setFileSizeKB(Long.parseLong(metadata.get("size").toString()) / KB_SIZE);
} catch (Throwable e) {
log.error("Error on getting file size", e);
}
if (originalFile != null) {
newFile.setMetadataRelatedID(original.getMetadataRelatedID());
if (StringUtils.isBlank(original.getDownloadedURL())) {
newFile.setDownloadedURL(original.getURI());
} else {
newFile.setDownloadedURL(original.getDownloadedURL());
}
}
try {
newFile.setURI(ModularObjectStoreRESTService.retrieveURL(baseURI, basePath, objectStoreID, newFile.getObjectID()));
} catch (UnsupportedEncodingException e) {
log.error("Error on Build objectStoreFile ", e);
}
return newFile;
}
public static UnaryFunction<String, DBObject> asJSON(final String baseURI, final String objectStoreID, final String basePath) {
return input -> build(input, baseURI, objectStoreID, basePath).toJSON();
}
}

77
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/AbstractObjectStoreAction.java
Unescape Escape View File

@ -0,0 +1,77 @@
package eu.dnetlib.data.objectstore.modular;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
import eu.dnetlib.enabling.tools.blackboard.BlackboardServerAction;
import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
/**
* The Class AbstractObjectStoreAction.
*/
public abstract class AbstractObjectStoreAction implements BlackboardServerAction<ObjectStoreActions> {
/**
* Logger
*/
private static final Log log = LogFactory.getLog(AbstractObjectStoreAction.class);
private final Executor executor = Executors.newCachedThreadPool();
/** The object store dao. */
private ObjectStoreDao dao;
protected abstract void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) throws ObjectStoreServiceException;
@Override
public void execute(final BlackboardServerHandler handler, final BlackboardJob job) {
executor.execute(new Runnable() {
@Override
public void run() {
try {
handler.ongoing(job);
executeAsync(handler, job);
} catch (ObjectStoreServiceException e) {
handler.failed(job, e);
}
}
});
}
protected void completeWithSuccess(final BlackboardServerHandler handler, final BlackboardJob job) {
// Don't change this synchronization rule
synchronized (this) {
handler.done(job);
}
}
protected void completeWithFail(final BlackboardServerHandler handler, final BlackboardJob job, final Throwable e) {
// Don't change this synchronization rule
synchronized (this) {
handler.failed(job, e);
}
}
/**
* Gets the dao.
*
* @return the dao
*/
public ObjectStoreDao getDao() {
return dao;
}
/**
* Sets the dao.
*
* @param dao the new dao
*/
public void setDao(ObjectStoreDao dao) {
this.dao = dao;
}
}

62
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/CreateObjectStoreAction.java
Unescape Escape View File

@ -0,0 +1,62 @@
package eu.dnetlib.data.objectstore.modular;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
/**
* The Class CreateObjectStoreAction is responsible to execute the blacboard action of type CREATE.
*/
public class CreateObjectStoreAction extends AbstractObjectStoreAction {
private static final Log log = LogFactory.getLog(CreateObjectStoreAction.class);
/** The profile creator. */
private ObjectStoreProfileCreator profileCreator;
/**
* Gets the profile creator.
*
* @return the profile creator
*/
public ObjectStoreProfileCreator getProfileCreator() {
return profileCreator;
}
/**
* Sets the profile creator.
*
* @param profileCreator the new profile creator
*/
public void setProfileCreator(final ObjectStoreProfileCreator profileCreator) {
this.profileCreator = profileCreator;
}
@Override
protected void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) {
try {
final String interpretation = job.getParameters().get("interpretation");
// final String basePath = job.getParameters().get("basePath");
// if (StringUtils.isBlank(basePath)) {
// throw new ObjectStoreServiceException("missing basePath param");
// }
final String objID = profileCreator.registerProfile(interpretation);
try {
getDao().createObjectStore(objID, interpretation, null);
} catch (Throwable e) {
log.warn("cannot created objectStore, deleting profile");
profileCreator.deleteProfile(objID);
throw new ObjectStoreServiceException(e);
}
job.getParameters().put("objectStoreId", objID);
completeWithSuccess(handler, job);
} catch (Exception e) {
completeWithFail(handler, job, e);
}
}
}

55
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/DeleteObjectStoreAction.java
Unescape Escape View File

@ -0,0 +1,55 @@
package eu.dnetlib.data.objectstore.modular;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
import eu.dnetlib.enabling.locators.UniqueServiceLocator;
import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;
import org.springframework.beans.factory.annotation.Autowired;
/**
* The Class DeleteObjectStoreAction is responsible to execute the blacboard action of type DELETE.
*/
public class DeleteObjectStoreAction extends AbstractObjectStoreAction {
/**
* The profile creator.
*/
private ObjectStoreProfileCreator profileCreator;
@Autowired
private UniqueServiceLocator serviceLocator;
@Override
protected void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) throws ObjectStoreServiceException {
try {
final String objID = job.getParameters().get("obsID");
serviceLocator.getService(ISRegistryService.class).deleteProfile(objID);
getDao().deleteObjectStore(objID);
completeWithSuccess(handler, job);
} catch (Exception e) {
completeWithFail(handler, job, e);
}
}
/**
* Gets the profile creator.
*
* @return the profile creator
*/
public ObjectStoreProfileCreator getProfileCreator() {
return profileCreator;
}
/**
* Sets the profile creator.
*
* @param profileCreator the new profile creator
*/
public void setProfileCreator(ObjectStoreProfileCreator profileCreator) {
this.profileCreator = profileCreator;
}
}

25
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/DropContentObjectStoreAction.java
Unescape Escape View File

@ -0,0 +1,25 @@
package eu.dnetlib.data.objectstore.modular;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;
/**
* Created by sandro on 2/26/16.
*/
public class DropContentObjectStoreAction extends AbstractObjectStoreAction {
@Override
protected void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) throws ObjectStoreServiceException {
try {
final String objID = job.getParameters().get("obsID");
final boolean status = getDao().dropContent(objID);
job.getParameters().put("dropStatus", "" + status);
completeWithSuccess(handler, job);
} catch (Exception e) {
completeWithFail(handler, job, e);
}
}
}

50
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/FeedCompleteObjectAction.java
Unescape Escape View File

@ -0,0 +1,50 @@
package eu.dnetlib.data.objectstore.modular;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;
import org.springframework.beans.factory.annotation.Required;
// TODO: Auto-generated Javadoc
/**
* The Class FeedCompleteObjectAction is responsible to execute the blacboard action of type FEED OBJECT which is Metadata created for WOS case of openaire.
*/
public class FeedCompleteObjectAction extends AbstractObjectStoreAction {
/** The store feeder. */
private ModularObjectStoreFeeder storeFeeder;
@Override
protected void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) throws ObjectStoreServiceException {
try {
final String objStoreID = job.getParameters().get("obsID");
final String eprRs = job.getParameters().get("epr");
final String mime = job.getParameters().get("mime");
int count = storeFeeder.feedMetadataObjectRecord(objStoreID, eprRs, mime);
job.getParameters().put("total", "" + count);
completeWithSuccess(handler, job);
} catch (Exception e) {
completeWithFail(handler, job, e);
}
}
/**
* Gets the store feeder.
*
* @return the store feeder
*/
public ModularObjectStoreFeeder getStoreFeeder() {
return storeFeeder;
}
/**
* Sets the store feeder.
*
* @param storeFeeder the new store feeder
*/
@Required
public void setStoreFeeder(final ModularObjectStoreFeeder storeFeeder) {
this.storeFeeder = storeFeeder;
}
}

92
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/FeedObjectStoreAction.java
Unescape Escape View File

@ -0,0 +1,92 @@
package eu.dnetlib.data.objectstore.modular;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
import eu.dnetlib.data.objectstore.rmi.Protocols;
import eu.dnetlib.enabling.tools.blackboard.BlackboardJob;
import eu.dnetlib.enabling.tools.blackboard.BlackboardServerHandler;
import org.springframework.beans.factory.annotation.Required;
/**
* The Class FeedObjectStoreAction to execute the blacboard action of type FEED.
*/
public class FeedObjectStoreAction extends AbstractObjectStoreAction {
/** The profile creator. */
private ObjectStoreProfileCreator profileCreator;
/** The store feeder. */
private ModularObjectStoreFeeder storeFeeder;
@Override
protected void executeAsync(final BlackboardServerHandler handler, final BlackboardJob job) throws ObjectStoreServiceException {
try {
final String objStoreID = job.getParameters().get("obsID");
final String objID = job.getParameters().get("objID");
final String URI = job.getParameters().get("URI");
final String eprRs = job.getParameters().get("epr");
final String protocolString = job.getParameters().get("protocol");
final Protocols protocol;
if (protocolString == null)
protocol = Protocols.None;
else
protocol = Protocols.valueOf(job.getParameters().get("protocol"));
final String mime = job.getParameters().get("mime");
final String login = job.getParameters().get("login");
final String password = job.getParameters().get("password");
if (URI != null && URI.length() > 0) {
storeFeeder.feedObject(objStoreID, objID, URI, protocol, login,
password, mime);
} else if (eprRs != null && eprRs.length() > 0) {
storeFeeder
.feed(objStoreID, eprRs, protocol, login, password, mime);
}
completeWithSuccess(handler, job);
} catch (Exception e) {
completeWithFail(handler, job, e);
}
}
/**
* Gets the profile creator.
*
* @return the profile creator
*/
public ObjectStoreProfileCreator getProfileCreator() {
return profileCreator;
}
/**
* Sets the profile creator.
*
* @param profileCreator the new profile creator
*/
@Required
public void setProfileCreator(ObjectStoreProfileCreator profileCreator) {
this.profileCreator = profileCreator;
}
/**
* Gets the store feeder.
*
* @return the store feeder
*/
public ModularObjectStoreFeeder getStoreFeeder() {
return storeFeeder;
}
/**
* Sets the store feeder.
*
* @param storeFeeder the new store feeder
*/
@Required
public void setStoreFeeder(ModularObjectStoreFeeder storeFeeder) {
this.storeFeeder = storeFeeder;
}
}

128
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ModularObjectStoreDeliver.java
Unescape Escape View File

@ -0,0 +1,128 @@
package eu.dnetlib.data.objectstore.modular;
import javax.annotation.Resource;
import javax.xml.ws.wsaddressing.W3CEndpointReference;
import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
import eu.dnetlib.enabling.resultset.ResultSetFactory;
import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;
import org.springframework.beans.factory.annotation.Required;
/**
* The Class ModularObjectStoreDeliver is responsible of delivering data from the object Store.
*/
public class ModularObjectStoreDeliver {
/** The dao. */
private ObjectStoreDao dao;
/** The result set factory. */
@Resource
private ResultSetFactory resultSetFactory;
/** The result set client factory. */
private ResultSetClientFactory resultSetClientFactory;
/**
* Gets the dao.
*
* @return the dao
*/
public ObjectStoreDao getDao() {
return dao;
}
/**
* Sets the dao.
*
* @param dao
* the new dao
*/
@Required
public void setDao(final ObjectStoreDao dao) {
this.dao = dao;
}
/**
* Deliver.
*
* @param objectStoreID
* the object store id
* @param from
* the from
* @param until
* the until
* @return the w3 c endpoint reference
* @throws ObjectStoreServiceException
*/
public W3CEndpointReference deliver(final String objectStoreID, final Long from, final Long until) throws ObjectStoreServiceException {
return resultSetFactory.createResultSet(dao.getObjectStore(objectStoreID).deliver(from, until));
}
/**
* Deliver ids.
*
* @param objectStoreID
* the object store id
* @param eprId
* the epr id
* @return the w3 c endpoint reference
* @throws ObjectStoreServiceException
*/
public W3CEndpointReference deliverIds(final String objectStoreID, final W3CEndpointReference eprId) throws ObjectStoreServiceException {
Iterable<String> ids = resultSetClientFactory.getClient(eprId);
return resultSetFactory.createResultSet(dao.getObjectStore(objectStoreID).deliverIds(ids));
}
/**
* Exist id starts with.
*
* @param obsId
* the obs id
* @param startId
* the start id
* @return true, if successful
* @throws ObjectStoreServiceException
*/
public boolean existIDStartsWith(final String obsId, final String startId) throws ObjectStoreServiceException {
return dao.getObjectStore(obsId).existIDStartsWith(startId);
}
/**
* Deliver object.
*
* @param objectStoreID
* the object store id
* @param objectId
* the object id
* @return the object store file
* @throws ObjectStoreServiceException
*/
public ObjectStoreFile deliverObject(final String objectStoreID, final String objectId) throws ObjectStoreServiceException {
return dao.getObjectStore(objectStoreID).deliverObject(objectId);
}
/**
* Gets the result set client factory.
*
* @return the result set client factory
*/
public ResultSetClientFactory getResultSetClientFactory() {
return resultSetClientFactory;
}
/**
* Sets the result set client factory.
*
* @param resultSetClientFactory
* the new result set client factory
*/
@Required
public void setResultSetClientFactory(final ResultSetClientFactory resultSetClientFactory) {
this.resultSetClientFactory = resultSetClientFactory;
}
}

234
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ModularObjectStoreFeeder.java
Unescape Escape View File

@ -0,0 +1,234 @@
package eu.dnetlib.data.objectstore.modular;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;
import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;
import eu.dnetlib.data.objectstore.rmi.MetadataObjectRecord;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
import eu.dnetlib.data.objectstore.rmi.Protocols;
import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
import eu.dnetlib.enabling.locators.UniqueServiceLocator;
import eu.dnetlib.enabling.resultset.client.ResultSetClientFactory;
import eu.dnetlib.miscutils.datetime.DateUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Required;
/**
* The Class ModularObjectStoreFeeder, responsible to feed data into the object Store
*/
public class ModularObjectStoreFeeder {
private static final Log log = LogFactory.getLog(ModularObjectStoreFeeder.class);
/** The dao of the objectStore. */
private ObjectStoreDao dao;
/** The result set client factory. */
private ResultSetClientFactory resultSetClientFactory;
/** The service locator. */
private UniqueServiceLocator serviceLocator;
/**
* Gets the dao.
*
* @return the dao
*/
public ObjectStoreDao getDao() {
return dao;
}
/**
* Sets the dao.
*
* @param dao
* the new dao
*/
@Required
public void setDao(final ObjectStoreDao dao) {
this.dao = dao;
}
/**
* Gets the result set client factory.
*
* @return the result set client factory
*/
public ResultSetClientFactory getResultSetClientFactory() {
return resultSetClientFactory;
}
/**
* Sets the result set client factory.
*
* @param resultSetClientFactory
* the new result set client factory
*/
@Required
public void setResultSetClientFactory(final ResultSetClientFactory resultSetClientFactory) {
this.resultSetClientFactory = resultSetClientFactory;
}
/**
* Feed metadata object record.
*
* @param objectStoreID
* the object store id
* @param rsEpr
* the rs epr
* @param mime
* the mime
* @throws ObjectStoreServiceException
*/
public int feedMetadataObjectRecord(final String objectStoreID, final String rsEpr, final String mime) throws ObjectStoreServiceException {
final Iterable<String> records = resultSetClientFactory.getClient(rsEpr);
Iterable<MetadataObjectRecord> toIngest = Iterables.transform(records, input -> {
MetadataObjectRecord record = MetadataObjectRecord.initFromJson(input);
if (record != null) {
record.setMime(mime);
} else {
log.error("An input record is null :" + input);
}
return record;
});
ObjectStore store = dao.getObjectStore(objectStoreID);
int size = store.feedMetadataRecord(toIngest, true);
touch(objectStoreID, size);
return size;
}
/**
* Feed object in the object store starting from an EPR of objectMetadata
*
* @param obsId
* the objectStore id
* @param rsEpr
* the result set Endpoint-reference
* @param protocol
* the protocol
* @param login
* the login
* @param password
* the password
* @param mime
* the mime type
* @throws ObjectStoreServiceException
*/
public void feed(final String obsId, final String rsEpr, final Protocols protocol, final String login, final String password, final String mime)
throws ObjectStoreServiceException {
final Iterable<String> records = resultSetClientFactory.getClient(rsEpr);
ObjectBroker objectBroker = new ObjectBroker(protocol, login, password, mime);
Iterable<ObjectStoreRecord> toIngest = Iterables.transform(records, objectBroker);
ObjectStore store = dao.getObjectStore(obsId);
int size = store.feed(toIngest, true);
touch(obsId, size);
}
/**
* Feed a single object in the object Stroe.
*
* @param objectStoreID
* the object store id
* @param objectID
* the object id
* @param URIObject
* the URI of object
* @param protocol
* the protocol
* @param login
* the login
* @param password
* the password
* @param mime
* the mime type
* @throws ObjectStoreServiceException
*/
public void feedObject(final String objectStoreID,
final String objectID,
final String URIObject,
final Protocols protocol,
final String login,
final String password,
final String mime) throws ObjectStoreServiceException {
ObjectStoreFile inputFile = new ObjectStoreFile();
inputFile.setURI(URIObject);
inputFile.setObjectID(objectID);
ObjectBroker objectBroker = new ObjectBroker(protocol, login, password, mime);
Iterable<ObjectStoreRecord> toIngest = Iterables.transform(Lists.newArrayList(inputFile.toJSON()), objectBroker);
ObjectStore store = dao.getObjectStore(objectStoreID);
int size = store.feed(toIngest, true);
touch(objectStoreID, size);
}
/**
* Feed object record.
*
* @param objectStoreID
* the object store id
* @param record
* the record
* @return the string
* @throws ObjectStoreServiceException
*/
public String feedObjectRecord(final String objectStoreID, final ObjectStoreRecord record) throws ObjectStoreServiceException {
ObjectStore store = dao.getObjectStore(objectStoreID);
return store.feedObjectRecord(record);
}
/**
* Sets the last modified date in the profile.
*
* @param obsId
* the obs id
* @param size
* the size
*/
public void touch(final String obsId, final int size) {
try {
final String now = DateUtils.now_ISO8601();
final String mdstoreXUpdate = "for $x in //RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '" + obsId + "']"
+ "return update value $x//LAST_STORAGE_DATE with '" + now + "'";
serviceLocator.getService(ISRegistryService.class, true).executeXUpdate(mdstoreXUpdate);
touchSize(obsId, size);
} catch (final Exception e) {
throw new IllegalStateException(e);
}
}
/**
* Touch size.
*
* @param obsId
* the obs id
* @param size
* the size
*/
public void touchSize(final String obsId, final int size) {
try {
final String mdstoreNumberXUpdate = "for $x in //RESOURCE_PROFILE[.//RESOURCE_IDENTIFIER/@value = '" + obsId + "']"
+ "return update value $x//COUNT_STORE with '" + size + "'";
serviceLocator.getService(ISRegistryService.class, true).executeXUpdate(mdstoreNumberXUpdate);
} catch (final Exception e) {
throw new IllegalStateException(e);
}
}
public UniqueServiceLocator getServiceLocator() {
return serviceLocator;
}
@Required
public void setServiceLocator(final UniqueServiceLocator serviceLocator) {
this.serviceLocator = serviceLocator;
}
}

265
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ModularObjectStoreService.java
Unescape Escape View File

@ -0,0 +1,265 @@
package eu.dnetlib.data.objectstore.modular;
import java.util.List;
import javax.xml.ws.wsaddressing.W3CEndpointReference;
import org.springframework.beans.factory.annotation.Required;
import com.google.gson.Gson;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreService;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
import eu.dnetlib.enabling.tools.AbstractBaseService;
import eu.dnetlib.enabling.tools.blackboard.NotificationHandler;
// TODO: Auto-generated Javadoc
/**
* The Class ModularObjectStoreService is the implementation of the ObjectStoreService interface.
*/
public class ModularObjectStoreService extends AbstractBaseService implements ObjectStoreService {
/** The feeder. */
private ModularObjectStoreFeeder feeder;
/** The object store deliver. */
private ModularObjectStoreDeliver objectStoreDeliver;
/** The notification handler. */
private NotificationHandler notificationHandler;
/*
* (non-Javadoc)
*
* @see eu.dnetlib.data.objectstore.rmi.ObjectStoreService#deliverObjects(java.lang.String, java.lang.Double, java.lang.Double)
*/
/**
* Deliver objects.
*
* @param obsId
* the obs id
* @param from
* the from
* @param until
* the until
* @return the w3 c endpoint reference
* @throws ObjectStoreServiceException
* the object store service exception
*/
@Override
public W3CEndpointReference deliverObjects(final String obsId, final Long from, final Long until) throws ObjectStoreServiceException {
return objectStoreDeliver.deliver(obsId, from, until);
}
/*
* (non-Javadoc)
*
* @see eu.dnetlib.data.objectstore.rmi.ObjectStoreService#deliverObjectsByIds(java.lang.String,
* javax.xml.ws.wsaddressing.W3CEndpointReference)
*/
/**
* Deliver objects by ids.
*
* @param obsId
* the obs id
* @param eprId
* the epr id
* @return the w3 c endpoint reference
* @throws ObjectStoreServiceException
* the object store service exception
*/
@Override
public W3CEndpointReference deliverObjectsByIds(final String obsId, final W3CEndpointReference eprId) throws ObjectStoreServiceException {
return objectStoreDeliver.deliverIds(obsId, eprId);
}
/*
* (non-Javadoc)
*
* @see eu.dnetlib.data.objectstore.rmi.ObjectStoreService#deliverRecord(java.lang.String, java.lang.String)
*/
/**
* Deliver record.
*
* @param obsId
* the obs id
* @param objectId
* the object id
* @return the string
* @throws ObjectStoreServiceException
* the object store service exception
*/
@Override
public String deliverRecord(final String obsId, final String objectId) throws ObjectStoreServiceException {
return objectStoreDeliver.deliverObject(obsId, objectId).toJSON();
}
/*
* (non-Javadoc)
*
* @see eu.dnetlib.data.objectstore.rmi.ObjectStoreService#getListOfObjectStores()
*/
/**
* Gets the list of object stores.
*
* @return the list of object stores
*/
@Override
public List<String> getListOfObjectStores() {
return objectStoreDeliver.getDao().listObjectStores();
}
/**
* Gets the feeder.
*
* @return the feeder
*/
public ModularObjectStoreFeeder getFeeder() {
return feeder;
}
/**
* Sets the feeder.
*
* @param feeder
* the new feeder
*/
@Required
public void setFeeder(final ModularObjectStoreFeeder feeder) {
this.feeder = feeder;
}
/**
* Gets the notification handler.
*
* @return the notification handler
*/
public NotificationHandler getNotificationHandler() {
return notificationHandler;
}
/**
* Sets the notification handler.
*
* @param notificationHandler
* the new notification handler
*/
@Required
public void setNotificationHandler(final NotificationHandler notificationHandler) {
this.notificationHandler = notificationHandler;
}
/**
* {@inheritDoc}
*
* @see eu.dnetlib.enabling.tools.AbstractBaseService#notify(java.lang.String, java.lang.String, java.lang.String, java.lang.String)
*/
@Override
public void notify(final String subscriptionId, final String topic, final String isId, final String message) {
getNotificationHandler().notified(subscriptionId, topic, isId, message);
}
/**
* Gets the object store deliver.
*
* @return the object store deliver
*/
public ModularObjectStoreDeliver getObjectStoreDeliver() {
return objectStoreDeliver;
}
/**
* Sets the object store deliver.
*
* @param objectStoreDeliver
* the new object store deliver
*/
@Required
public void setObjectStoreDeliver(final ModularObjectStoreDeliver objectStoreDeliver) {
this.objectStoreDeliver = objectStoreDeliver;
}
/*
* (non-Javadoc)
*
* @see eu.dnetlib.data.objectstore.rmi.ObjectStoreService#feedObject(java.lang.String, java.lang.String)
*/
/**
* Feed object.
*
* @param obsId
* the obs id
* @param objectMetadata
* the object metadata
* @throws ObjectStoreServiceException
* the object store service exception
*/
@Override
public void feedObject(final String obsId, final String objectMetadata) throws ObjectStoreServiceException {
Gson g = new Gson();
try {
ObjectStoreFile file = g.fromJson(objectMetadata, ObjectStoreFile.class);
feeder.feedObject(obsId, file.getObjectID(), file.getURI(), file.getAccessProtocol(), file.getUsernameAuth(), file.getPasswordAuth(),
file.getMimeType());
} catch (Exception e) {
throw new ObjectStoreServiceException(e.getMessage());
}
}
/**
* Exist id starts with.
*
* @param obsId
* the obs id
* @param startId
* the start id
* @return true, if successful
* @throws ObjectStoreServiceException
* the object store service exception
*/
public boolean existIDStartsWith(final String obsId, final String startId) throws ObjectStoreServiceException {
return objectStoreDeliver.existIDStartsWith(obsId, startId);
}
/**
* Feed object.
*
* @param obsId
* the obs id
* @param record
* the record
* @return the string
* @throws ObjectStoreServiceException
* the object store service exception
*/
public String feedObject(final String obsId, final ObjectStoreRecord record) throws ObjectStoreServiceException {
try {
return feeder.feedObjectRecord(obsId, record);
} catch (Exception e) {
throw new ObjectStoreServiceException(e.getMessage());
}
}
/**
* Gets the size.
*
* @param obsId
* the obs id
* @return the size
* @throws ObjectStoreServiceException
* the object store service exception
*/
@Override
public int getSize(final String obsId) throws ObjectStoreServiceException {
return objectStoreDeliver.getDao().getObjectStore(obsId).getSize();
}
}

240
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectBroker.java
Unescape Escape View File

@ -0,0 +1,240 @@
package eu.dnetlib.data.objectstore.modular;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.SocketException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.net.ftp.FTPClient;
import com.google.common.base.Function;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
import eu.dnetlib.data.objectstore.rmi.Protocols;
/**
* The Class ObjectBroker is responsible to retrieve content from URL.
*/
public class ObjectBroker implements Function<String, ObjectStoreRecord> {
private static final Log log = LogFactory.getLog(ObjectBroker.class);
/** The protocol. */
private Protocols protocol;
/** The login. */
private String login;
/** The password. */
private String password;
/** The mime. */
private String mime;
/**
* Instantiates a new object broker.
*
* @param protocol
* the protocol
* @param login
* the login
* @param password
* the password
* @param mime
* the mime
*/
public ObjectBroker(final Protocols protocol, final String login, final String password, final String mime) {
this.protocol = protocol;
this.login = login;
this.password = password;
this.mime = mime;
}
/**
* Gets the protocol.
*
* @return the protocol
*/
public Protocols getProtocol() {
return protocol;
}
/**
* Sets the protocol.
*
* @param protocol
* the new protocol
*/
public void setProtocol(final Protocols protocol) {
this.protocol = protocol;
}
/**
* Gets the login.
*
* @return the login
*/
public String getLogin() {
return login;
}
/**
* Sets the login.
*
* @param login
* the new login
*/
public void setLogin(final String login) {
this.login = login;
}
/**
* Gets the password.
*
* @return the password
*/
public String getPassword() {
return password;
}
/**
* Sets the password.
*
* @param password
* the new password
*/
public void setPassword(final String password) {
this.password = password;
}
/**
* Gets the mime.
*
* @return the mime
*/
public String getMime() {
return mime;
}
/**
* Sets the mime.
*
* @param mime
* the new mime
*/
public void setMime(final String mime) {
this.mime = mime;
}
/*
* (non-Javadoc)
*
* @see com.google.common.base.Function#apply(java.lang.Object)
*/
@Override
public ObjectStoreRecord apply(final String jsonInput) {
if (jsonInput == null) return null;
ObjectStoreRecord objectStorerecord = new ObjectStoreRecord();
objectStorerecord.setFileMetadata(ObjectStoreFile.createObject(jsonInput));
Protocols currentProtocol = (objectStorerecord.getFileMetadata().getAccessProtocol() == Protocols.None) ? protocol : objectStorerecord
.getFileMetadata().getAccessProtocol();
objectStorerecord.getFileMetadata().setAccessProtocol(currentProtocol);
if ((objectStorerecord.getFileMetadata().getMimeType() == null) || (objectStorerecord.getFileMetadata().getMimeType().length() == 0)) {
objectStorerecord.getFileMetadata().setMimeType(mime);
}
switch (currentProtocol) {
case FTP:
FTPClient client = new FTPClient();
try {
URI uri = new URI(objectStorerecord.getFileMetadata().getURI());
client.connect(uri.getHost());
if ((objectStorerecord.getFileMetadata().getUsernameAuth() != null) && (objectStorerecord.getFileMetadata().getUsernameAuth().length() > 0)) {
client.login(objectStorerecord.getFileMetadata().getUsernameAuth(), objectStorerecord.getFileMetadata().getPasswordAuth());
} else {
client.login("ftp", "a@a");
}
final InputStream stream = client.retrieveFileStream(uri.getPath());
objectStorerecord.setInputStream(stream);
return objectStorerecord;
} catch (URISyntaxException e2) {
log.error(e2);
return null;
} catch (SocketException e) {
log.error(e);
} catch (IOException e) {
log.error(e);
return null;
}
case HTTP:
try {
HttpURLConnection conn = (HttpURLConnection) new URL(objectStorerecord.getFileMetadata().getURI()).openConnection();
InputStream in;
int http_status;
try {
http_status = conn.getResponseCode();
if ((http_status / 100) == 3) {
String newURL = conn.getHeaderField("Location");
conn.disconnect();
conn = (HttpURLConnection) new URL(newURL).openConnection();
http_status = conn.getResponseCode();
if ((http_status / 100) != 2) return null;
}
in = conn.getInputStream();
objectStorerecord.setInputStream(in);
return objectStorerecord;
} catch (Exception e) {
log.error(e);
return null;
}
} catch (MalformedURLException e1) {
log.error(e1);
return null;
} catch (IOException e1) {
log.error(e1);
return null;
}
case File_System:
File f = new File(objectStorerecord.getFileMetadata().getURI());
try {
InputStream myiInputStream = new FileInputStream(f);
objectStorerecord.setInputStream(myiInputStream);
return objectStorerecord;
} catch (FileNotFoundException e) {
try {
Thread.sleep(5000);
InputStream myiInputStream = new FileInputStream(f);
objectStorerecord.setInputStream(myiInputStream);
return objectStorerecord;
} catch (Exception e1) {
log.error(e1);
return null;
}
}
default:
break;
}
return null;
}
}

22
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectStoreActions.java
Unescape Escape View File

@ -0,0 +1,22 @@
package eu.dnetlib.data.objectstore.modular;
/**
* The Enumeration of the blacboard action of the ObjectStore.
*/
public enum ObjectStoreActions {
/** The create. */
CREATE,
/**
* The delete.
*/
DELETE,
/**
* The feed.
*/
FEED,
/**
* The feedobject.
*/
FEEDOBJECT, DROP_CONTENT
}

102
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectStoreConsistency.java
Unescape Escape View File

@ -0,0 +1,102 @@
package eu.dnetlib.data.objectstore.modular;
import java.util.List;
import java.util.Set;
import com.google.common.base.Function;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
import eu.dnetlib.enabling.locators.UniqueServiceLocator;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.ui.ModelMap;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseBody;
@Controller
public class ObjectStoreConsistency {
private static final Log log = LogFactory.getLog(ObjectStoreConsistency.class);
@Autowired
private ObjectStoreDao objectStoreDao;
@Autowired
private UniqueServiceLocator serviceLocator;
private ObjectStoreIntegrityInfo integrityInfo;
@RequestMapping(value = "/ui/objectStore/infoConsistency.do")
@ResponseBody
public ObjectStoreIntegrityInfo getConsistencyInfo(final ModelMap map) {
return getIntegrityInfo();
}
public void refreshConsistency() {
try {
Set<String> profiles = listProfileIds();
Set<String> objectStore = listObjectStoreIds();
Set<String> objectStoreNotInProfile = Sets.newHashSet(objectStore);
objectStoreNotInProfile.removeAll(profiles);
Set<String> profilesNotInObjectStore = Sets.newHashSet(profiles);
profilesNotInObjectStore.removeAll(objectStore);
ObjectStoreIntegrityInfo info = new ObjectStoreIntegrityInfo();
info.setObjectStoreWithoutProfile(objectStoreNotInProfile);
info.setProfileWithoutObjectStore(profilesNotInObjectStore);
this.integrityInfo = info;
} catch (ISLookUpException e) {
log.error("Error on refreshing consistency of objectStore ", e);
}
}
private Set<String> listObjectStoreIds() {
return Sets.newHashSet(
Lists.transform(objectStoreDao.listObjectStores(), new Function<String, String>() {
@Override
public String apply(String input) {
return StringUtils.substringBefore(input, "_");
}
}));
}
private Set<String> listProfileIds() throws ISLookUpException {
final ISLookUpService lookupService = serviceLocator.getService(ISLookUpService.class);
final String query = "for $x in collection('/db/DRIVER/ObjectStoreDSResources/ObjectStoreDSResourceType') return $x//RESOURCE_IDENTIFIER/@value/string()";
List<String> resultList = lookupService.quickSearchProfile(query);
return Sets.newHashSet(Lists.transform(resultList, new Function<String, String>() {
@Override
public String apply(String input) {
return StringUtils.substringBefore(input, "_");
}
}));
}
public ObjectStoreIntegrityInfo getIntegrityInfo() {
if (integrityInfo == null) {
refreshConsistency();
}
return integrityInfo;
}
public void setIntegrityInfo(ObjectStoreIntegrityInfo integrityInfo) {
this.integrityInfo = integrityInfo;
}
}

30
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectStoreIntegrityInfo.java
Unescape Escape View File

@ -0,0 +1,30 @@
package eu.dnetlib.data.objectstore.modular;
import java.util.Set;
/**
* Created by Sandro La Bruzzo on 10/16/15.
*/
public class ObjectStoreIntegrityInfo {
private Set<String> objectStoreWithoutProfile;
private Set<String> profileWithoutObjectStore;
public Set<String> getObjectStoreWithoutProfile() {
return objectStoreWithoutProfile;
}
public void setObjectStoreWithoutProfile(Set<String> objectStoreWithoutProfile) {
this.objectStoreWithoutProfile = objectStoreWithoutProfile;
}
public Set<String> getProfileWithoutObjectStore() {
return profileWithoutObjectStore;
}
public void setProfileWithoutObjectStore(Set<String> profileWithoutObjectStore) {
this.profileWithoutObjectStore = profileWithoutObjectStore;
}
}

127
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectStoreProfileCreator.java
Unescape Escape View File

@ -0,0 +1,127 @@
package eu.dnetlib.data.objectstore.modular;
import eu.dnetlib.enabling.is.registry.rmi.ISRegistryException;
import eu.dnetlib.enabling.is.registry.rmi.ISRegistryService;
import eu.dnetlib.enabling.locators.UniqueServiceLocator;
import eu.dnetlib.soap.EndpointReferenceBuilder;
import org.antlr.stringtemplate.StringTemplate;
import org.springframework.beans.factory.annotation.Required;
import javax.xml.ws.Endpoint;
// TODO: Auto-generated Javadoc
/**
* The Class ObjectStoreProfileCreator is responsible of creating profile of the ObjectStore
*/
public class ObjectStoreProfileCreator {
/**
* service locator.
*/
private UniqueServiceLocator serviceLocator;
/**
* objectstore ds template.
*/
private StringTemplate objectstoreDsTemplate;
/**
* service endpoint.
*/
private Endpoint endpoint;
/**
* endpoint builder.
*/
private EndpointReferenceBuilder<Endpoint> eprBuilder;
/**
* Register profile.
*
* @param interpretation the interpretation
* @return the string
* @throws ISRegistryException the IS registry exception
*/
public String registerProfile(final String interpretation)
throws ISRegistryException {
// XXX: mini hack
StringTemplate template = new StringTemplate(
objectstoreDsTemplate.getTemplate());
template.setAttribute("serviceUri", eprBuilder.getAddress(endpoint));
template.setAttribute("interpretation", interpretation);
return serviceLocator.getService(ISRegistryService.class, true).registerProfile(template.toString());
}
public void deleteProfile(final String profileId) throws ISRegistryException {
serviceLocator.getService(ISRegistryService.class).deleteProfile(profileId);
}
/**
* Gets the endpoint.
*
* @return the endpoint
*/
public Endpoint getEndpoint() {
return endpoint;
}
/**
* Sets the endpoint.
*
* @param endpoint the new endpoint
*/
@Required
public void setEndpoint(final Endpoint endpoint) {
this.endpoint = endpoint;
}
/**
* Gets the epr builder.
*
* @return the epr builder
*/
public EndpointReferenceBuilder<Endpoint> getEprBuilder() {
return eprBuilder;
}
/**
* Sets the epr builder.
*
* @param eprBuilder the new epr builder
*/
@Required
public void setEprBuilder(final EndpointReferenceBuilder<Endpoint> eprBuilder) {
this.eprBuilder = eprBuilder;
}
/**
* Gets the objectstore ds template.
*
* @return the objectstore ds template
*/
public StringTemplate getObjectstoreDsTemplate() {
return objectstoreDsTemplate;
}
/**
* Sets the objectstore ds template.
*
* @param objectstoreDsTemplate the new objectstore ds template
*/
@Required
public void setObjectstoreDsTemplate(final StringTemplate objectstoreDsTemplate) {
this.objectstoreDsTemplate = objectstoreDsTemplate;
}
public UniqueServiceLocator getServiceLocator() {
return serviceLocator;
}
@Required
public void setServiceLocator(final UniqueServiceLocator serviceLocator) {
this.serviceLocator = serviceLocator;
}
}

55
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/ObjectStoreRecord.java
Unescape Escape View File

@ -0,0 +1,55 @@
package eu.dnetlib.data.objectstore.modular;
import java.io.InputStream;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
/**
* The Class ObjectStoreRecord is a serialization of the object to be ingested in the objectStore
* metadata + inputstream of the data.
*/
public class ObjectStoreRecord {
/** The file metadata. */
private ObjectStoreFile fileMetadata;
/** The input stream. */
private InputStream inputStream;
/**
* Gets the input stream.
*
* @return the input stream
*/
public InputStream getInputStream() {
return inputStream;
}
/**
* Sets the input stream.
*
* @param inputStream the new input stream
*/
public void setInputStream(final InputStream inputStream) {
this.inputStream = inputStream;
}
/**
* Gets the file metadata.
*
* @return the file metadata
*/
public ObjectStoreFile getFileMetadata() {
return fileMetadata;
}
/**
* Sets the file metadata.
*
* @param fileMetadata the new file metadata
*/
public void setFileMetadata(final ObjectStoreFile fileMetadata) {
this.fileMetadata = fileMetadata;
}
}

131
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/connector/ObjectStore.java
Unescape Escape View File

@ -0,0 +1,131 @@
package eu.dnetlib.data.objectstore.modular.connector;
import eu.dnetlib.data.objectstore.modular.ObjectStoreRecord;
import eu.dnetlib.data.objectstore.rmi.MetadataObjectRecord;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
import eu.dnetlib.enabling.resultset.ResultSetListener;
/**
* The Interface ObjectStore.
*/
public interface ObjectStore {
/**
* Gets the id of the objectStore.
*
* @return the id
*/
String getId();
/**
* Gets the interpretation the objectStore.
*
* @return the interpretation
*/
String getInterpretation();
/**
* Feed record into the objectstore.
*
* @param records
* the records
* @param incremental
* the incremental
* @return the int
*/
int feed(Iterable<ObjectStoreRecord> records, boolean incremental) throws ObjectStoreServiceException;
/**
* Feed metadata record into the objectStore.
*
* @param records
* the records
* @param incremental
* the incremental
* @return the int
*/
int feedMetadataRecord(Iterable<MetadataObjectRecord> records, boolean incremental) throws ObjectStoreServiceException;
/**
* Feed a single object record into the objectStore.
*
* @param record
* the record
* @return the string
*/
String feedObjectRecord(ObjectStoreRecord record) throws ObjectStoreServiceException;
/**
* Deliver Object from the objectStore.
*
* @param from
* : start date which you want to filter
* @param until
* : end date which you want to filter
* @return the result set listener
*/
ResultSetListener deliver(Long from, Long until) throws ObjectStoreServiceException;
/**
* Deliver ids.
*
* @param ids
* the ids
* @return the result set listener
*/
ResultSetListener deliverIds(Iterable<String> ids) throws ObjectStoreServiceException;
/**
* Deliver object.
*
* @param objectId
* the object id
* @return the object store file
* @throws ObjectStoreServiceException
*/
ObjectStoreFile deliverObject(String objectId) throws ObjectStoreServiceException;
/**
* Gets the size.
*
* @return the size
*/
int getSize() throws ObjectStoreServiceException;
/**
* Delete object.
*
* @param objectId
* the object id
*/
void deleteObject(String objectId) throws ObjectStoreServiceException;
/**
* Gets the object.
*
* @param recordId
* the record id
* @return the object
*/
String getObject(String recordId) throws ObjectStoreServiceException;
/**
* Find if exist an ID startingwith the string startId.
*
* @param startId
* the start id
* @return if exist or less an id
*/
boolean existIDStartsWith(String startId) throws ObjectStoreServiceException;
/**
* drop the content of the ObjectStore
* THIS METHOD DELETE ALL THE CONTENT INSIDE THE OBJECTSTORE PAY ATTENTION WHEN YOU CALL IT
*
* @return if the content has been refreshed
* @throws ObjectStoreServiceException
*/
boolean dropContent() throws ObjectStoreServiceException;
}

72
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/connector/ObjectStoreDao.java
Unescape Escape View File

@ -0,0 +1,72 @@
package eu.dnetlib.data.objectstore.modular.connector;
import java.util.List;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
/**
* The Interface ObjectStoreDao.
*/
public interface ObjectStoreDao {
/**
* Gets an Object Store with the given identifier.
* <p>
* If an Object Store with the given identifier does not exist, a new one is created.
* </p>
*
* @param obsId
* the object store identifier
* @return the object store
* @throws ObjectStoreServiceException
*/
ObjectStore getObjectStore(String obsId) throws ObjectStoreServiceException;
/**
* List all the Object stores.
*
* @return the list of object store ids
*/
List<String> listObjectStores();
/**
* Creates an Object Store with the given identifier.
* <p>
* If an Object Store with the given identifier already exists this method does nothing.
* </p>
*
* @param obsId the object store identifier
* @param interpretation the interpretation of the store
* @param basePath the base path to store the object Store in case of file system implementation
* @return true, if successful
* @throws ObjectStoreServiceException
*/
boolean createObjectStore(String obsId, String interpretation, String basePath) throws ObjectStoreServiceException;
/**
* Upddate an Object Store metadata with the given identifier.
* <p>
* If an Object Store with the given identifier does not exist, a new one is created.
* </p>
*
* @param obsId
* the object store identifier
* @param interpretation
* the interpretation of the store
* @return true, if successful
*/
boolean updateObjectStore(String obsId, String interpretation);
/**
* Delete object store.
*
* @param obsId
* the object store identifier
* @return true, if successful
* @throws ObjectStoreServiceException
*/
boolean deleteObjectStore(String obsId) throws ObjectStoreServiceException;
boolean dropContent(String obsId) throws ObjectStoreServiceException;
}

199
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/modular/inspector/ObjectStoreInspector.java
Unescape Escape View File

@ -0,0 +1,199 @@
package eu.dnetlib.data.objectstore.modular.inspector;
import static eu.dnetlib.miscutils.collections.MappedCollection.listMap;
import java.util.List;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import com.google.gson.Gson;
import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;
import eu.dnetlib.data.objectstore.modular.connector.ObjectStoreDao;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
import eu.dnetlib.enabling.inspector.AbstractInspectorController;
import eu.dnetlib.enabling.resultset.ResultSetListener;
import eu.dnetlib.miscutils.collections.MappedCollection;
import eu.dnetlib.miscutils.functional.UnaryFunction;
/**
* The Class ObjectStoreInspector is the inspector controller of the objectStore.
*/
@Controller
public class ObjectStoreInspector extends AbstractInspectorController {
// private static final Log log = LogFactory.getLog(ObjectStoreInspector.class);
/** The object store dao. */
@Autowired
private ObjectStoreDao objectStoreDao;
/**
* The Class ObjectStoreDescription.
*/
class ObjectStoreDescription {
/** The id. */
private String id;
/** The size. */
private int size;
/**
* Instantiates a new object store description.
*
* @param id
* the id
* @param size
* the size
*/
public ObjectStoreDescription(final String id, final int size) {
this.id = id;
this.size = size;
}
/**
* Gets the interpretation.
*
* @return the interpretation
* @throws ObjectStoreServiceException
*/
public String getInterpretation() throws ObjectStoreServiceException {
ObjectStore objectStore = objectStoreDao.getObjectStore(id);
return objectStore.getInterpretation();
}
/**
* Gets the id.
*
* @return the id
*/
public String getId() {
return id;
}
/**
* Sets the id.
*
* @param id
* the new id
*/
public void setId(final String id) {
this.id = id;
}
/**
* Gets the size.
*
* @return the size
*/
public int getSize() {
return size;
}
/**
* Sets the size.
*
* @param size
* the new size
*/
public void setSize(final int size) {
this.size = size;
}
}
/**
* Object stores.
*
* @param model
* the model
*/
@RequestMapping(value = "/inspector/objectstores.do")
public void objectStores(final Model model) {
List<String> objectStores = objectStoreDao.listObjectStores();
model.addAttribute("objectstores", MappedCollection.listMap(objectStores, new UnaryFunction<ObjectStoreDescription, String>() {
@Override
public ObjectStoreDescription evaluate(final String id) {
try {
ObjectStore objStore = objectStoreDao.getObjectStore(id);
return new ObjectStoreDescription(id, objStore.getSize());
} catch (ObjectStoreServiceException e) {
return null;
}
}
}));
}
/**
* Object store.
*
* @param model
* the model
* @param id
* the id
* @param startParam
* the start param
* @param regex
* the regex
* @throws ObjectStoreServiceException
*/
@RequestMapping(value = "/inspector/objectstore.do", method = RequestMethod.GET)
public void objectStore(final Model model,
@RequestParam("id") final String id,
@RequestParam(value = "start", required = false) final Integer startParam,
@RequestParam(value = "regex", required = false) final String regex) throws ObjectStoreServiceException {
int pageSize = 10;
int start = 0;
if (startParam != null) {
start = startParam;
}
ObjectStore objctStore = objectStoreDao.getObjectStore(id);
ResultSetListener rs = objctStore.deliver((long) 0, System.currentTimeMillis());
List<String> page = rs.getResult((1 + start), (start + pageSize));
final Gson g = new Gson();
model.addAttribute("id", id);
model.addAttribute("start", start);
model.addAttribute("regex", regex);
model.addAttribute("nextPage", start + pageSize);
model.addAttribute("prevPage", Math.max(0, start - pageSize));
model.addAttribute("size", rs.getSize());
model.addAttribute("page", listMap(page, new UnaryFunction<ObjectStoreFile, String>() {
@Override
public ObjectStoreFile evaluate(final String json) {
return g.fromJson(json, ObjectStoreFile.class);
}
}));
}
/**
* Gets the object store dao.
*
* @return the object store dao
*/
public ObjectStoreDao getObjectStoreDao() {
return objectStoreDao;
}
/**
* Sets the object store dao.
*
* @param objectStoreDao
* the new object store dao
*/
public void setObjectStoreDao(final ObjectStoreDao objectStoreDao) {
this.objectStoreDao = objectStoreDao;
}
}

61
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/MetadataObjectRecord.java
Unescape Escape View File

@ -0,0 +1,61 @@
package eu.dnetlib.data.objectstore.rmi;
import java.io.Serializable;
import com.google.gson.Gson;
public class MetadataObjectRecord implements Serializable {
/** The Constant serialVersionUID. */
private static final long serialVersionUID = -5640196854633924754L;
public static MetadataObjectRecord initFromJson(final String input) {
return new Gson().fromJson(input, MetadataObjectRecord.class);
}
public MetadataObjectRecord() {
}
public MetadataObjectRecord(final String id, final String record, final String mime) {
super();
this.id = id;
this.record = record;
this.mime = mime;
}
private String id;
private String record;
private String mime;
public String getId() {
return id;
}
public String getRecord() {
return record;
}
public void setId(final String id) {
this.id = id;
}
public void setRecord(final String record) {
this.record = record;
}
public String toJSON() {
return new Gson().toJson(this);
}
public String getMime() {
return mime;
}
public void setMime(final String mime) {
this.mime = mime;
}
}

258
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/ObjectStoreFile.java
Unescape Escape View File

@ -0,0 +1,258 @@
package eu.dnetlib.data.objectstore.rmi;
import java.io.Serializable;
import com.google.gson.Gson;
/**
* The Class ObjectStoreFile.
*/
public class ObjectStoreFile implements Serializable {
/** The Constant serialVersionUID. */
private static final long serialVersionUID = -6501291693572693712L;
/** The metadata related id. */
private String metadataRelatedID;
/** The object id. */
private String objectID;
/** The mime type. */
private String mimeType;
/** The access protocol. */
private Protocols accessProtocol;
/** The object uri . */
private String URI;
/** The username auth. */
private String usernameAuth;
/** The password auth. */
private String passwordAuth;
/** The downloaded url. */
private String downloadedURL;
/** The md5 sum of the file. */
private String md5Sum;
/** The file size. */
private long fileSizeKB;
/**
* Gets the downloaded url.
*
* @return the downloaded url
*/
public String getDownloadedURL() {
return downloadedURL;
}
/**
* Sets the downloaded url.
*
* @param downloadedURL
* the new downloaded url
*/
public void setDownloadedURL(final String downloadedURL) {
this.downloadedURL = downloadedURL.replace("\\u003d", "=").replace("\\u0026", "&");;
}
/**
* Instantiates a new object store file.
*/
public ObjectStoreFile() {
this.accessProtocol = Protocols.None;
}
/**
* Return a new instance of the object from a json String.
*
* @param jsonObject
* the json object
* @return the object store file
*/
public static ObjectStoreFile createObject(final String jsonObject) {
Gson g = new Gson();
return g.fromJson(jsonObject, ObjectStoreFile.class);
}
/**
* Gets the object id.
*
* @return the object id
*/
public String getObjectID() {
return objectID;
}
/**
* Sets the object id.
*
* @param objectID
* the new object id
*/
public void setObjectID(final String objectID) {
this.objectID = objectID;
}
/**
* Gets the mime type.
*
* @return the mime type
*/
public String getMimeType() {
return mimeType;
}
/**
* Sets the mime type.
*
* @param mimeType
* the new mime type
*/
public void setMimeType(final String mimeType) {
this.mimeType = mimeType;
}
/**
* Gets the access protocol.
*
* @return the access protocol
*/
public Protocols getAccessProtocol() {
return accessProtocol;
}
/**
* Sets the access protocol.
*
* @param accessProtocol
* the new access protocol
*/
public void setAccessProtocol(final Protocols accessProtocol) {
this.accessProtocol = accessProtocol;
}
/**
* Gets the username auth.
*
* @return the username auth
*/
public String getUsernameAuth() {
return usernameAuth;
}
/**
* Sets the username auth.
*
* @param usernameAuth
* the new username auth
*/
public void setUsernameAuth(final String usernameAuth) {
this.usernameAuth = usernameAuth;
}
/**
* Gets the password auth.
*
* @return the password auth
*/
public String getPasswordAuth() {
return passwordAuth;
}
/**
* Sets the password auth.
*
* @param passwordAuth
* the new password auth
*/
public void setPasswordAuth(final String passwordAuth) {
this.passwordAuth = passwordAuth;
}
/**
* Gets the uri.
*
* @return the uri
*/
public String getURI() {
return URI != null ? URI.replace("\\u003d", "=").replace("\\u0026", "&") : "";
}
/**
* Sets the uri.
*
* @param uri
* the new uri
*/
public void setURI(final String uri) {
if (uri != null) {
URI = uri.replace("\\u003d", "=").replace("\\u0026", "&");
}
}
/**
* Convert the object into a json String.
*
* @return the string
*/
public String toJSON() {
Gson g = new Gson();
return g.toJson(this).replace("\\u003d", "=").replace("\\u0026", "&");
}
/**
* Gets the related metadata id.
*
* @return the related metadata id.
*/
public String getMetadataRelatedID() {
return metadataRelatedID;
}
/**
* Sets the related metadata id.
*
* @param metadataRelatedID
* the related metadata id.
*/
public void setMetadataRelatedID(final String metadataRelatedID) {
this.metadataRelatedID = metadataRelatedID;
}
/**
* @return the md5Sum
*/
public String getMd5Sum() {
return md5Sum;
}
/**
* @param md5Sum
* the md5Sum to set
*/
public void setMd5Sum(final String md5Sum) {
this.md5Sum = md5Sum;
}
/**
* @return the fileSizeKB
*/
public long getFileSizeKB() {
return fileSizeKB;
}
/**
* @param fileSizeKB the fileSizeKB to set
*/
public void setFileSizeKB(long fileSizeKB) {
this.fileSizeKB = fileSizeKB;
}
}

22
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/ObjectStoreFileNotFoundException.java
Unescape Escape View File

@ -0,0 +1,22 @@
package eu.dnetlib.data.objectstore.rmi;
/**
* The Class ObjectStoreFileNotFoundException.
*/
public class ObjectStoreFileNotFoundException extends ObjectStoreServiceException {
/** The Constant serialVersionUID. */
private static final long serialVersionUID = -5418923557898374219L;
/**
* Instantiates a new object store file not found exception.
*
* @param string
* the string
*/
public ObjectStoreFileNotFoundException(final String string) {
super(string);
}
}

109
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/ObjectStoreService.java
Unescape Escape View File

@ -0,0 +1,109 @@
package eu.dnetlib.data.objectstore.rmi;
import java.util.List;
import javax.jws.WebMethod;
import javax.jws.WebParam;
import javax.jws.WebService;
import javax.xml.ws.wsaddressing.W3CEndpointReference;
import eu.dnetlib.common.rmi.BaseService;
/**
* Main ObjectStore Service interface.
*
* @author <a href="mailto:sandro.labruzzo at isti.cnr.it">Sandro La Bruzzo</a>
* @version 0.0.1
*/
@WebService(targetNamespace = "http://services.dnetlib.eu/")
public interface ObjectStoreService extends BaseService {
/**
* Returns ResultSet EPR for delivered ObjectStore records in a particular range date.
* <p>
* Please check service implementations for details on the expected format of the records in the result set epr.
* </p>
* <p>
* This method could be used for a bulk deliver of all objects in the store
* </p>
*
* @param obsId
* identifier of the ObjectStore
* @param from
* the minimum date of the object
* @param until
* the maximum date of the object
* @return a ResultSet EPR. Each element of the result set contains the objIdentifier of a record and its URL for retrieve the
* inputstream of the file.
* @throws ObjectStoreServiceException
* the object store service exception
*/
@WebMethod(operationName = "deliverObjects", action = "deliverObjects")
public W3CEndpointReference deliverObjects(@WebParam(name = "obsId") String obsId, @WebParam(name = "from") Long from, @WebParam(name = "until") Long until)
throws ObjectStoreServiceException;
/**
* Returns ResultSet EPR for delivered ObjectStore records.
* <p>
* Please check service implementations for details on the expected format of the records in the result set epr.
* </p>
*
* @param obsId
* identifier of the ObjectStore
* @param eprId
* id of a ResultSet EPR with the identifiers of the interesting objects. Each element of the result set contains the
* objIdentifier of a record
* @return a ResultSet EPR. Each element of the result set contains the objIdentifier of a record and its URL for retrieve the
* inputstream of the file.
* @throws ObjectStoreServiceException
* the object store service exception
*/
@WebMethod(operationName = "deliverObjectsByIds", action = "deliverObjectsByIds")
public W3CEndpointReference deliverObjectsByIds(@WebParam(name = "obsId") String obsId, @WebParam(name = "eprId") W3CEndpointReference eprId)
throws ObjectStoreServiceException;
/**
* Returns an URL to retrieve the ObjectStore record.
*
* @param obsId
* identifier of the ObjectStore
* @param objectId
* the id of the object
* @return the URL for retrieve the record
* @throws ObjectStoreServiceException
* the object store service exception
*/
@WebMethod(operationName = "deliverObject", action = "deliverObject")
public String deliverRecord(@WebParam(name = "obsId") String obsId, @WebParam(name = "objectId") String objectId) throws ObjectStoreServiceException;
/**
* Feed the object in the objectStore
*
* @param obsId
* identifier of the ObjectStore
* @param objectMetadata
* the String serialized of the JSON object ObjectStoreFile
* @throws ObjectStoreServiceException
*/
@WebMethod(operationName = "feedObject", action = "feedObject")
public void feedObject(@WebParam(name = "obsId") String obsId, @WebParam(name = "objectMetadata") String objectMetadata) throws ObjectStoreServiceException;
/**
* Returns list of all stored indices.
*
* @return list of all stored indices
*/
@WebMethod(operationName = "getListOfObjectStores", action = "getListOfObjectStores")
public List<String> getListOfObjectStores();
/**
* Gets the size of the objectStore ID.
*
* @param obsId
* the obs id
* @return the size
*/
@WebMethod(operationName = "getSize", action = "getSize")
public int getSize(@WebParam(name = "obsId") String obsId) throws ObjectStoreServiceException;
}

45
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/ObjectStoreServiceException.java
Unescape Escape View File

@ -0,0 +1,45 @@
package eu.dnetlib.data.objectstore.rmi;
import eu.dnetlib.common.rmi.RMIException;
/**
* The Class ObjectStoreServiceException.
*/
public class ObjectStoreServiceException extends RMIException {
/** The Constant serialVersionUID. */
private static final long serialVersionUID = -7073846285219543315L;
/**
* Instantiates a new object store service exception.
*
* @param string
* the string
*/
public ObjectStoreServiceException(final String string) {
super(string);
}
/**
* Instantiates a new object store service exception.
*
* @param string
* the string
* @param exception
* the exception
*/
public ObjectStoreServiceException(final String string, final Throwable exception) {
super(string, exception);
}
/**
* Instantiates a new object store service exception.
*
* @param exception
* the exception
*/
public ObjectStoreServiceException(final Throwable exception) {
super(exception);
}
}

13
dnet-data-services/src/main/java/eu/dnetlib/data/objectstore/rmi/Protocols.java
Unescape Escape View File

@ -0,0 +1,13 @@
package eu.dnetlib.data.objectstore.rmi;
/**
* The Enum Protocols.
*/
public enum Protocols {
None,
HTTP,
HTTPS,
FTP,
File_System,
FTPS
}

19
dnet-data-services/src/main/resources/eu/dnetlib/applicationContext-dnet-objectstore-rmi.xml
Unescape Escape View File

@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:jaxws="http://cxf.apache.org/jaxws"
xmlns:sec="http://cxf.apache.org/configuration/security" xmlns:wsa="http://cxf.apache.org/ws/addressing"
xmlns:p="http://www.springframework.org/schema/p" xmlns:http="http://cxf.apache.org/transports/http/configuration"
xmlns:t="http://dnetlib.eu/springbeans/t" xmlns:template="http://dnetlib.eu/springbeans/template"
xmlns:util="http://www.springframework.org/schema/util"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://cxf.apache.org/ws/addressing http://cxf.apache.org/schemas/ws-addr-conf.xsd
http://cxf.apache.org/configuration/security http://cxf.apache.org/schemas/configuration/security.xsd
http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd
http://cxf.apache.org/jaxws http://cxf.apache.org/schemas/jaxws.xsd
http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.0.xsd
http://dnetlib.eu/springbeans/template http://dnetlib.eu/springbeans/template.xsd">
</beans>

2
dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/applicationContext-dnet-modular-objectstore-service.properties
Unescape Escape View File

@ -0,0 +1,2 @@
services.objectstore.dao=gridFSObjectstoreDao
services.objectstore.rsfactory.pagesize=20

111
dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/applicationContext-dnet-modular-objectstore-service.xml
Unescape Escape View File

@ -0,0 +1,111 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:jaxws="http://cxf.apache.org/jaxws"
xmlns:p="http://www.springframework.org/schema/p"
xmlns:http="http://cxf.apache.org/transports/http/configuration" xmlns:t="http://dnetlib.eu/springbeans/t"
xmlns:template="http://dnetlib.eu/springbeans/template"
xmlns="http://www.springframework.org/schema/beans"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://cxf.apache.org/transports/http/configuration http://cxf.apache.org/schemas/configuration/http-conf.xsd
http://cxf.apache.org/jaxws http://cxf.apache.org/schemas/jaxws.xsd
http://dnetlib.eu/springbeans/template http://dnetlib.eu/springbeans/template.xsd">
<bean id="objectStoreService"
class="eu.dnetlib.data.objectstore.modular.ModularObjectStoreService"
init-method="start" destroy-method="stop"
p:feeder-ref="objectstoreFeeder"
p:objectStoreDeliver-ref="objectstoreDeliver"
p:notificationHandler-ref="objectstoreNotificationHandler"
/>
<bean id="objectstoreFeeder"
class="eu.dnetlib.data.objectstore.modular.ModularObjectStoreFeeder"
p:dao-ref="${services.objectstore.dao}"
p:resultSetClientFactory-ref="objectstoreResultSetClientFactory"
p:serviceLocator-ref="uniqueServiceLocator"
/>
<bean id="objectstoreDeliver"
class="eu.dnetlib.data.objectstore.modular.ModularObjectStoreDeliver"
p:dao-ref="${services.objectstore.dao}"
p:resultSetClientFactory-ref="objectstoreResultSetClientFactory"
/>
<bean id="objectstoreResultSetClientFactory" parent="resultSetClientFactory"
p:pageSize="${services.objectstore.rsfactory.pagesize}"/>
<bean id="objectstoreProfileCreator" class="eu.dnetlib.data.objectstore.modular.ObjectStoreProfileCreator"
p:serviceLocator-ref="uniqueServiceLocator" p:objectstoreDsTemplate-ref="objectstoreDsTemplate"
p:endpoint-ref="objectStoreServiceEndpoint" p:eprBuilder-ref="jaxwsEndpointReferenceBuilder"/>
<bean id="objectstoreNotificationHandler"
class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerExecutorNotificationHandler"
p:blackboardExecutor-ref="objectstoreBlackboardExecutor"/>
<bean id="objectstoreBlackboardExecutor"
class="eu.dnetlib.enabling.tools.blackboard.BlackboardServerActionExecutor"
p:blackboardHandler-ref="blackboardHandler"
p:actionType="eu.dnetlib.data.objectstore.modular.ObjectStoreActions"
p:incomplete="false">
<property name="actionMap">
<map>
<entry key="CREATE">
<bean class="eu.dnetlib.data.objectstore.modular.CreateObjectStoreAction"
p:profileCreator-ref="objectstoreProfileCreator" p:dao-ref="${services.objectstore.dao}"/>
</entry>
<entry key="DELETE">
<bean class="eu.dnetlib.data.objectstore.modular.DeleteObjectStoreAction"
p:profileCreator-ref="objectstoreProfileCreator" p:dao-ref="${services.objectstore.dao}"/>
</entry>
<entry key="DROP_CONTENT">
<bean class="eu.dnetlib.data.objectstore.modular.DropContentObjectStoreAction"
p:dao-ref="${services.objectstore.dao}"/>
</entry>
<entry key="FEED">
<bean class="eu.dnetlib.data.objectstore.modular.FeedObjectStoreAction"
p:profileCreator-ref="objectstoreProfileCreator"
p:storeFeeder-ref="objectstoreFeeder"
p:dao-ref="${services.objectstore.dao}"/>
</entry>
<entry key="FEEDOBJECT">
<bean class="eu.dnetlib.data.objectstore.modular.FeedCompleteObjectAction"
p:storeFeeder-ref="objectstoreFeeder"
p:dao-ref="${services.objectstore.dao}"/>
</entry>
</map>
</property>
</bean>
<bean id="objectStoreLocator" class="eu.dnetlib.enabling.tools.StaticServiceLocator"
p:service-ref="objectStoreService"/>
<bean id="objectstoreDsTemplate"
class="eu.dnetlib.springutils.stringtemplate.StringTemplateFactory"
p:template="classpath:/eu/dnetlib/data/objectstore/objectstoreds-template.xml"
scope="prototype"/>
<!-- endpoints -->
<jaxws:endpoint id="objectStoreServiceEndpoint"
implementor="#objectStoreService" implementorClass="eu.dnetlib.data.objectstore.rmi.ObjectStoreService"
address="/objectStore"/>
<template:instance name="serviceRegistrationManager"
t:serviceRegistrationManagerClass="eu.dnetlib.enabling.tools.registration.ValidatingServiceRegistrationManagerImpl"
t:name="objectStoreServiceRegistrationManager" t:service="objectStoreService"
t:endpoint="objectStoreServiceEndpoint" t:jobScheduler="jobScheduler"
t:serviceRegistrator="blackboardServiceRegistrator"/>
</beans>

8
dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/filesystem/applicationContext-filesystem-objectstore.properties
Unescape Escape View File

@ -0,0 +1,8 @@
services.objectStore.mongodb.host=localhost
services.objectStore.mongodb.port=27017
services.objectStore.mongodb.db=objectStore
services.objectStore.mongodb.connectionsPerHost=20
services.objectStore.mongodb.upsert=true
services.objectstore.mongodb.integritycheck.cron=0 0 2 1/1 * ? *
services.objectStore.RESTURI=http://${container.hostname}:${container.port}/${container.context}/mvc/objectStore/retrieve.do
services.objectstore.dao=fSObjectstoreDao

53
dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/filesystem/applicationContext-filesystem-objectstore.xml
Unescape Escape View File

@ -0,0 +1,53 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:p="http://www.springframework.org/schema/p" xmlns:t="http://dnetlib.eu/springbeans/t"
xmlns="http://www.springframework.org/schema/beans"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
<bean id="fSObjectstoreDao"
class="eu.dnetlib.data.objectstore.filesystem.FileSystemObjectStoreDao"
p:db-ref="objectstoreMongoDB" p:objectStoreRESTURI-ref="restURIString"
p:upsert="${services.objectStore.mongodb.upsert}"/>
<bean id="restURIString" class="java.lang.String">
<constructor-arg value="${services.objectStore.RESTURI}"/>
</bean>
<bean id="objectStoreFileSystemUtility" class="eu.dnetlib.data.objectstore.filesystem.FileSystemUtility"/>
<bean id="objectStoreMongoClient" class="com.mongodb.MongoClient">
<constructor-arg index="0" type="com.mongodb.ServerAddress">
<bean class="com.mongodb.ServerAddress">
<constructor-arg index="0" value="${services.objectStore.mongodb.host}"/>
<constructor-arg index="1" value="${services.objectStore.mongodb.port}"/>
</bean>
</constructor-arg>
<constructor-arg index="1" type="com.mongodb.MongoClientOptions">
<bean class="eu.dnetlib.data.objectstore.filesystem.MongoFSOptionsFactory"
p:connectionsPerHost="${services.objectStore.mongodb.connectionsPerHost}"/>
</constructor-arg>
</bean>
<bean id="objectstoreMongoDB" factory-bean="objectStoreMongoClient" factory-method="getDatabase">
<constructor-arg index="0" type="java.lang.String" value="${services.objectStore.mongodb.db}"/>
</bean>
<bean id="objectStoreIntegrityChecker" class="eu.dnetlib.data.objectstore.filesystem.IndexIntegrityChecker"/>
<bean t:id="objectStoreJobSchedulerAccessor"
class="org.springframework.scheduling.quartz.SchedulerAccessorBean"
p:scheduler-ref="jobScheduler">
<property name="triggers">
<list>
<bean class="org.springframework.scheduling.quartz.CronTriggerFactoryBean">
<property name="jobDetail">
<bean class="org.springframework.scheduling.quartz.MethodInvokingJobDetailFactoryBean"
p:targetObject-ref="objectStoreIntegrityChecker" p:targetMethod="check"/>
</property>
<property name="cronExpression" value="${services.objectstore.mongodb.integritycheck.cron}"/>
</bean>
</list>
</property>
</bean>
</beans>

20
dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/inspector/webContext-objectstore-inspector.xml
Unescape Escape View File

@ -0,0 +1,20 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:p="http://www.springframework.org/schema/p"
xmlns:util="http://www.springframework.org/schema/util"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-2.5.xsd">
<bean id="objectstoreInspectorGroup"
class="eu.dnetlib.enabling.inspector.StaticEntryPointDescriptorGroup"
p:name="objectstore">
<property name="descriptors">
<list>
<bean class="eu.dnetlib.enabling.inspector.StaticEntryPointDescriptor"
p:name="objectstores" p:relativeUrl="objectstores.do" />
</list>
</property>
</bean>
</beans>

24
dnet-data-services/src/main/resources/eu/dnetlib/data/objectstore/objectstoreds-template.xml
Unescape Escape View File

@ -0,0 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
<RESOURCE_PROFILE>
<HEADER>
<RESOURCE_IDENTIFIER value=""/>
<RESOURCE_TYPE value="ObjectStoreDSResourceType"/>
<RESOURCE_KIND value="ObjectStoreDSResources"/>
<RESOURCE_URI value="$serviceUri$?wsdl"/>
<DATE_OF_CREATION value=""/>
</HEADER>
<BODY>
<CONFIGURATION>
<OBJECTSTORE_INTERPRETATION>$interpretation$</OBJECTSTORE_INTERPRETATION>
</CONFIGURATION>
<STATUS>
<PENULTIMATE_STORAGE_DATE></PENULTIMATE_STORAGE_DATE>
<LAST_STORAGE_DATE></LAST_STORAGE_DATE>
<FETCHING_FREQUENCY/>
<COUNT_STORE>0</COUNT_STORE>
<STORE_SIZE>0</STORE_SIZE>
<STATISTICS_FIELDS/>
</STATUS>
<SECURITY_PARAMETERS></SECURITY_PARAMETERS>
</BODY>
</RESOURCE_PROFILE>

51
dnet-data-services/src/test/java/eu/dnetlib/data/objectstore/filesystem/ConfigurationTestConfig.java
Unescape Escape View File

@ -0,0 +1,51 @@
package eu.dnetlib.data.objectstore.filesystem;
import java.net.UnknownHostException;
import com.mongodb.MongoClient;
import com.mongodb.client.MongoDatabase;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
// TODO: Auto-generated Javadoc
/**
* The Class ConfigurationTestConfig.
*/
@Configuration
public class ConfigurationTestConfig {
/**
* Mongo db.
*
* @return the db
* @throws UnknownHostException the unknown host exception
*/
@Bean
public MongoDatabase objectstoreMongoDB() throws UnknownHostException {
MongoClient mongoClient = new MongoClient("localhost");
return mongoClient.getDatabase("objectStoreTest");
}
@Bean
public FileSystemObjectStoreDao fsObjectStoreDAO() {
FileSystemObjectStoreDao fileSystemObjectStoreDao = new FileSystemObjectStoreDao();
fileSystemObjectStoreDao.setObjectStoreRESTURI("http://www.objectstore.com");
return fileSystemObjectStoreDao;
}
/**
* Fs utility.
*
* @return the file system utility
*/
@Bean
public FileSystemUtility fsUtility () {
return new FileSystemUtility();
}
}

80
dnet-data-services/src/test/java/eu/dnetlib/data/objectstore/filesystem/InputIterator.java
Unescape Escape View File

@ -0,0 +1,80 @@
/**
*
*/
package eu.dnetlib.data.objectstore.filesystem;
import java.io.IOException;
import java.util.Iterator;
import eu.dnetlib.data.objectstore.modular.ObjectStoreRecord;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
import eu.dnetlib.data.objectstore.rmi.Protocols;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.Resource;
/**
* @author sandro
*
*/
public class InputIterator implements Iterable<ObjectStoreRecord>, Iterator<ObjectStoreRecord> {
private int counter = 0;
private String pdfResourcePath = "test.pdf";
private Resource testPDF = new ClassPathResource(pdfResourcePath);
/**
* {@inheritDoc}
* @see java.util.Iterator#hasNext()
*/
@Override
public boolean hasNext() {
return counter<100;
}
/**
* {@inheritDoc}
* @see java.util.Iterator#next()
*/
@Override
public ObjectStoreRecord next() {
try {
counter ++;
ObjectStoreRecord record = new ObjectStoreRecord();
ObjectStoreFile fileMetadata = new ObjectStoreFile();
fileMetadata.setAccessProtocol(Protocols.File_System);
fileMetadata.setDownloadedURL("file://" + pdfResourcePath);
fileMetadata.setURI("file://" + pdfResourcePath);
fileMetadata.setObjectID("Oggetto_"+counter);
fileMetadata.setMimeType("application/pdf");
System.out.println("Aggiungo Elemento "+counter);
record.setInputStream(testPDF.getInputStream());
record.setFileMetadata(fileMetadata );
return record;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* {@inheritDoc}
* @see java.util.Iterator#remove()
*/
@Override
public void remove() {
// TODO Auto-generated method stub
}
/**
* {@inheritDoc}
* @see java.lang.Iterable#iterator()
*/
@Override
public Iterator<ObjectStoreRecord> iterator() {
counter =0;
return this;
}
}

117
dnet-data-services/src/test/java/eu/dnetlib/data/objectstore/filesystem/ObjectStoreServiceDAOTest.java
Unescape Escape View File

@ -0,0 +1,117 @@
package eu.dnetlib.data.objectstore.filesystem;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import com.mongodb.client.MongoDatabase;
import eu.dnetlib.data.objectstore.modular.connector.ObjectStore;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreFile;
import eu.dnetlib.data.objectstore.rmi.ObjectStoreServiceException;
import eu.dnetlib.enabling.resultset.ResultSetListener;
import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
/**
* TODO: Test class set to ignored because it requires a mongo server running on localhost. Tests with mocks should be prepared.
* @author sandro
*
*/
@Ignore
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(classes = ConfigurationTestConfig.class)
public class ObjectStoreServiceDAOTest {
private static final Log log = LogFactory.getLog(ObjectStoreServiceDAOTest.class); // NOPMD by marko on 11/24/08 5:02 PM
@Autowired
private FileSystemObjectStoreDao objectStoreDAO;
@Autowired
private MongoDatabase objectstoreMongoDB;
private String interpretation = "interp";
private String basePath = "/tmp/basePath";
//must be 36 chars
private String obsID = "test56789111315171921232527293133350";
private String pdfPath = "./test.pdf";
@Before
public void setup() throws IOException, ObjectStoreServiceException {
Path baseDirPath = FileSystems.getDefault().getPath(basePath);
if (!Files.exists(baseDirPath)) {
Files.createDirectory(baseDirPath);
}
objectStoreDAO.createObjectStore(obsID + "_T2JqZWN0U3RvcmVEU1Jlc291cmNlcy9PYmplY3RTdG9yZURTUmVzb3VyY2VUeXBl", interpretation, basePath);
}
@After
public void tearDown() throws IOException, ObjectStoreServiceException {
objectstoreMongoDB.drop();
FileUtils.deleteDirectory(new File(basePath));
}
@Test
public void testList() {
final List<String> objectStores = this.objectStoreDAO.listObjectStores();
for (String o : objectStores)
System.out.println(o);
assertTrue(objectStores.contains(obsID + "_T2JqZWN0U3RvcmVEU1Jlc291cmNlcy9PYmplY3RTdG9yZURTUmVzb3VyY2VUeXBl"));
}
@Test
public void testGet() throws ObjectStoreServiceException {
ObjectStore o = objectStoreDAO.getObjectStore(obsID);
System.out.println(o.toString());
assertNotNull(o);
}
@Test
public void testFeed() throws ObjectStoreServiceException, FileNotFoundException {
objectStoreDAO.getObjectStore(obsID).feed(new InputIterator(),true);
}
@Test
public void testDeliverObject() throws ObjectStoreServiceException {
objectStoreDAO.getObjectStore(obsID).feed(new InputIterator(), true);
final ObjectStoreFile obj2 = objectStoreDAO.getObjectStore(obsID).deliverObject("Oggetto_2");
System.out.println(obj2.toJSON());
assertNotNull(obj2);
}
@Test
public void testGetStream() throws Exception {
objectStoreDAO.getObjectStore(obsID).feed(new InputIterator(), true);
ResultSetListener rs = objectStoreDAO.getObjectStore(obsID).deliver(0L, System.currentTimeMillis());
for (int i=1; i< (rs.getSize()/10); ) {
int from = i;
int to = Math.max(rs.getSize(), i*10);
List<String> data = rs.getResult(from, to);
for (String s: data) {
System.out.println(s);
}
i= to ;
}
}
}

BIN
dnet-data-services/src/test/resources/test.pdf
View File

Binary file not shown.
Write Preview
Loading…
Cancel
Save