D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

moved some mdstore componenets under dnet-data-services

This commit is contained in:
Michele Artini 2023-02-09 13:36:26 +01:00
parent 73ade91f43
commit f8094c659c
39 changed files with 936 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
apps/dnet-is-application/pom.xml
View File

@ -19,6 +19,12 @@
<artifactId>dnet-is-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dnet-data-services</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
@ -29,28 +35,6 @@
<groupId>jaxen</groupId>
<artifactId>jaxen</artifactId>
</dependency>
<!-- Hadoop -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
</dependency>
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-hadoop</artifactId>
<version>1.5.0-cdh5.13.3</version>
</dependency>
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-avro</artifactId>
<version>1.5.0-cdh5.13.3</version>
</dependency>
<!-- Hadoop D-Net Schemas -->
<dependency>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dhp-schemas</artifactId>
</dependency>
<!-- hot swapping, disable cache for template, enable live reload -->
<dependency>

2
apps/dnet-is-application/src/main/java/eu/dnetlib/MainDBConfig.java
View File

@ -38,7 +38,7 @@ public class MainDBConfig {
@Qualifier("mainDataSource") final DataSource ds) {
return builder
.dataSource(ds)
.packages("eu.dnetlib.is.model", "eu.dnetlib.msro.model", "eu.dnetlib.dhp.schema.mdstore")
.packages("eu.dnetlib.is.model", "eu.dnetlib.msro.model", "eu.dnetlib.data.mdstore.model")
.persistenceUnit("is")
.build();
}

7
apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/AbstractMDStoreController.java
View File

@ -8,15 +8,16 @@ import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestParam;
import eu.dnetlib.common.controller.AbstractDnetController;
import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
import eu.dnetlib.data.mdstore.hadoop.HadoopMDStoreService;
import eu.dnetlib.data.mdstore.model.MDStoreVersion;
import eu.dnetlib.data.mdstore.model.MDStoreWithInfo;
import eu.dnetlib.errors.MDStoreManagerException;
import io.swagger.v3.oas.annotations.Operation;
public class AbstractMDStoreController extends AbstractDnetController {
@Autowired
protected MDStoreService service;
protected HadoopMDStoreService service;
@Operation(summary = "Return all the mdstores")
@GetMapping("/")

5
apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreApiController.java
View File

@ -10,7 +10,7 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
import eu.dnetlib.data.mdstore.model.MDStoreVersion;
import eu.dnetlib.errors.MDStoreManagerException;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
@ -44,7 +44,8 @@ public class MDStoreApiController extends AbstractMDStoreController {
@Operation(summary = "Delete expired versions")
@DeleteMapping("/versions/expired")
public StatusResponse deleteExpiredVersions() {
return service.deleteExpiredVersions();
new Thread(service::deleteExpiredVersions).start();
return StatusResponse.DELETING;
}
@Operation(summary = "list the file inside the path of a mdstore version")

2
apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/ZeppelinController.java
View File

@ -6,7 +6,7 @@ import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import eu.dnetlib.data.mdstore.hadoop.ZeppelinClient;
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
import eu.dnetlib.data.mdstore.model.MDStoreWithInfo;
import eu.dnetlib.errors.MDStoreManagerException;
@Controller

2
apps/dnet-is-application/src/main/java/eu/dnetlib/is/importer/OldProfilesImporter.java
View File

@ -17,10 +17,10 @@ import eu.dnetlib.is.model.resource.SimpleResource;
import eu.dnetlib.is.model.vocabulary.Synonym;
import eu.dnetlib.is.model.vocabulary.Vocabulary;
import eu.dnetlib.is.model.vocabulary.VocabularyTerm;
import eu.dnetlib.is.resource.ResourceValidator;
import eu.dnetlib.is.resource.repository.SimpleResourceRepository;
import eu.dnetlib.is.vocabulary.repository.VocabularyRepository;
import eu.dnetlib.is.vocabulary.repository.VocabularyTermRepository;
import eu.dnetlib.utils.ResourceValidator;
import eu.dnetlib.utils.XmlIndenter;
@Service

2
apps/dnet-is-application/src/main/java/eu/dnetlib/utils/ResourceValidator.java → apps/dnet-is-application/src/main/java/eu/dnetlib/is/resource/ResourceValidator.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.utils;
package eu.dnetlib.is.resource;
import java.io.IOException;
import java.io.InputStream;

1
apps/dnet-is-application/src/main/java/eu/dnetlib/is/resource/SimpleResourceService.java
View File

@ -17,7 +17,6 @@ import org.springframework.web.bind.annotation.PathVariable;
import eu.dnetlib.errors.InformationServiceException;
import eu.dnetlib.is.model.resource.SimpleResource;
import eu.dnetlib.is.resource.repository.SimpleResourceRepository;
import eu.dnetlib.utils.ResourceValidator;
@Service
public class SimpleResourceService {

68
libs/dnet-data-services/pom.xml Normal file
View File

@ -0,0 +1,68 @@
<?xml version="1.0"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>libs</artifactId>
<version>3.3.3-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dnet-data-services</artifactId>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>eu.dnetlib.dhp</groupId>
<artifactId>dnet-is-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</dependency>
<!-- Hadoop -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
</dependency>
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-hadoop</artifactId>
<version>1.5.0-cdh5.13.3</version>
</dependency>
<dependency>
<groupId>com.twitter</groupId>
<artifactId>parquet-avro</artifactId>
<version>1.5.0-cdh5.13.3</version>
</dependency>
<!-- Tests -->
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

84
apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/MDStoreService.java → libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/MDStoreService.java
View File

@ -2,11 +2,8 @@ package eu.dnetlib.data.mdstore;
import java.util.Comparator;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
@ -16,25 +13,19 @@ import javax.transaction.Transactional;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import com.google.common.collect.Sets;
import eu.dnetlib.data.mdstore.hadoop.HdfsClient;
import eu.dnetlib.data.mdstore.model.MDStore;
import eu.dnetlib.data.mdstore.model.MDStoreCurrentVersion;
import eu.dnetlib.data.mdstore.model.MDStoreVersion;
import eu.dnetlib.data.mdstore.model.MDStoreWithInfo;
import eu.dnetlib.data.mdstore.repository.MDStoreCurrentVersionRepository;
import eu.dnetlib.data.mdstore.repository.MDStoreRepository;
import eu.dnetlib.data.mdstore.repository.MDStoreVersionRepository;
import eu.dnetlib.data.mdstore.repository.MDStoreWithInfoRepository;
import eu.dnetlib.dhp.schema.mdstore.MDStore;
import eu.dnetlib.dhp.schema.mdstore.MDStoreCurrentVersion;
import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
import eu.dnetlib.errors.MDStoreManagerException;
@Service
public class MDStoreService {
public abstract class MDStoreService {
@Autowired
private MDStoreRepository mdstoreRepository;
@ -45,12 +36,7 @@ public class MDStoreService {
@Autowired
private MDStoreWithInfoRepository mdstoreWithInfoRepository;
@Autowired
private JdbcTemplate jdbcTemplate;
@Autowired
protected HdfsClient hdfsClient;
@Value("${dhp.mdstore-manager.hdfs.base-path}")
private String hdfsBasePath;
protected JdbcTemplate jdbcTemplate;
private static final Logger log = LoggerFactory.getLogger(MDStoreService.class);
@ -94,17 +80,27 @@ public class MDStoreService {
final String dsId,
final String apiId) {
final MDStore md = MDStore.newInstance(format, layout, interpretation, dsName, dsId, apiId, hdfsBasePath);
final MDStore md = newMDStore(format, layout, interpretation, dsName, dsId, apiId);
mdstoreRepository.save(md);
final MDStoreVersion v = MDStoreVersion.newInstance(md.getId(), false, hdfsBasePath);
final MDStoreVersion v = newMDStoreVersion(md.getId(), false);
v.setLastUpdate(new Date());
mdstoreVersionRepository.save(v);
mdstoreCurrentVersionRepository.save(MDStoreCurrentVersion.newInstance(v));
return md.getId();
}
protected abstract MDStore newMDStore(final String format,
final String layout,
final String interpretation,
final String dsName,
final String dsId,
final String apiId);
protected abstract MDStoreVersion newMDStoreVersion(final String mdId, final boolean writing);
@Transactional
public void deleteMdStore(final String mdId) throws MDStoreManagerException {
@ -129,7 +125,7 @@ public class MDStoreService {
mdstoreVersionRepository.deleteByMdstore(mdId);
mdstoreRepository.deleteById(mdId);
hdfsClient.deletePath(md.get().getHdfsPath());
deleteFromBackend(md.get().getHdfsPath());
}
@Transactional
@ -159,7 +155,7 @@ public class MDStoreService {
@Transactional
public MDStoreVersion prepareMdStoreVersion(final String mdId) {
final MDStoreVersion v = MDStoreVersion.newInstance(mdId, true, hdfsBasePath);
final MDStoreVersion v = newMDStoreVersion(mdId, true);
mdstoreVersionRepository.save(v);
return v;
}
@ -176,12 +172,7 @@ public class MDStoreService {
return v;
}
public StatusResponse deleteExpiredVersions() {
new Thread(this::performDeleteOfExpiredVersions).start();
return StatusResponse.DELETING;
}
private synchronized void performDeleteOfExpiredVersions() {
public synchronized void deleteExpiredVersions() {
log.info("Deleting expired version...");
for (final String versionId : listExpiredVersions()) {
try {
@ -210,37 +201,8 @@ public class MDStoreService {
mdstoreVersionRepository.delete(v);
hdfsClient.deletePath(v.getHdfsPath());
}
public Set<String> fixHdfsInconsistencies(final boolean delete) throws MDStoreManagerException {
final Set<String> hdfsDirs = hdfsClient.listHadoopDirs();
final Set<String> validDirs = new HashSet<>(jdbcTemplate
.queryForList("select hdfs_path from mdstores union all select hdfs_path from mdstore_versions", String.class));
final Set<String> toDelete = Sets.difference(hdfsDirs, validDirs);
log.info("Found " + toDelete.size() + " hdfs paths to remove");
if (delete) {
for (final String p : toDelete) {
hdfsClient.deletePath(p);
}
}
return toDelete;
}
public List<Map<String, String>> listMdstoreParquet(final String mdId, final long limit) throws MDStoreManagerException {
return listVersionParquet(findMdStore(mdId).getCurrentVersion(), limit);
}
public List<Map<String, String>> listVersionParquet(final String versionId, final long limit) throws MDStoreManagerException {
final String path = findVersion(versionId).getHdfsPath();
return hdfsClient.readParquetFiles(path + "/store", limit);
}
public Set<String> listVersionFiles(final String versionId) throws MDStoreManagerException {
final String path = findVersion(versionId).getHdfsPath();
return hdfsClient.listContent(path + "/store", HdfsClient::isParquetFile);
deleteFromBackend(v.getHdfsPath());
}
protected abstract void deleteFromBackend(String path) throws MDStoreManagerException;
}

82
libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/hadoop/HadoopMDStoreService.java Normal file
View File

@ -0,0 +1,82 @@
package eu.dnetlib.data.mdstore.hadoop;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import com.google.common.collect.Sets;
import eu.dnetlib.data.mdstore.MDStoreService;
import eu.dnetlib.data.mdstore.model.MDStore;
import eu.dnetlib.data.mdstore.model.MDStoreVersion;
import eu.dnetlib.errors.MDStoreManagerException;
@Service
public class HadoopMDStoreService extends MDStoreService {
private static final Logger log = LoggerFactory.getLogger(HadoopMDStoreService.class);
@Autowired
protected HdfsClient hdfsClient;
@Value("${dhp.mdstore-manager.hdfs.base-path}")
private String hdfsBasePath;
@Override
protected void deleteFromBackend(final String path) throws MDStoreManagerException {
hdfsClient.deletePath(path);
}
@Override
protected MDStore newMDStore(final String format,
final String layout,
final String interpretation,
final String dsName,
final String dsId,
final String apiId) {
return MDStore.newInstance(format, layout, interpretation, dsName, dsId, apiId, hdfsBasePath);
}
@Override
protected MDStoreVersion newMDStoreVersion(final String mdId, final boolean writing) {
return MDStoreVersion.newInstance(mdId, writing, hdfsBasePath);
}
public Set<String> fixHdfsInconsistencies(final boolean delete) throws MDStoreManagerException {
final Set<String> hdfsDirs = hdfsClient.listHadoopDirs();
final Set<String> validDirs = new HashSet<>(jdbcTemplate
.queryForList("select hdfs_path from mdstores union all select hdfs_path from mdstore_versions", String.class));
final Set<String> toDelete = Sets.difference(hdfsDirs, validDirs);
log.info("Found " + toDelete.size() + " hdfs paths to remove");
if (delete) {
for (final String p : toDelete) {
deleteFromBackend(p);
}
}
return toDelete;
}
public List<Map<String, String>> listMdstoreParquet(final String mdId, final long limit) throws MDStoreManagerException {
return listVersionParquet(findMdStore(mdId).getCurrentVersion(), limit);
}
public List<Map<String, String>> listVersionParquet(final String versionId, final long limit) throws MDStoreManagerException {
final String path = findVersion(versionId).getHdfsPath();
return hdfsClient.readParquetFiles(path + "/store", limit);
}
public Set<String> listVersionFiles(final String versionId) throws MDStoreManagerException {
final String path = findVersion(versionId).getHdfsPath();
return hdfsClient.listContent(path + "/store", HdfsClient::isParquetFile);
}
}

0
apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/hadoop/HasStatus.java → libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/hadoop/HasStatus.java
View File

0
apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/hadoop/HdfsClient.java → libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/hadoop/HdfsClient.java
View File

0
apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/hadoop/ListResponse.java → libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/hadoop/ListResponse.java
View File

0
apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/hadoop/MapResponse.java → libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/hadoop/MapResponse.java
View File

0
apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/hadoop/Note.java → libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/hadoop/Note.java
View File

0
apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/hadoop/Paragraph.java → libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/hadoop/Paragraph.java
View File

0
apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/hadoop/SimpleResponse.java → libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/hadoop/SimpleResponse.java
View File

0
apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/hadoop/StringResponse.java → libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/hadoop/StringResponse.java
View File

2
apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/hadoop/ZeppelinClient.java → libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/hadoop/ZeppelinClient.java
View File

@ -33,7 +33,7 @@ import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import org.springframework.web.client.RestTemplate;
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
import eu.dnetlib.data.mdstore.model.MDStoreWithInfo;
import eu.dnetlib.errors.MDStoreManagerException;
@Component

2
apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/repository/MDStoreCurrentVersionRepository.java → libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/repository/MDStoreCurrentVersionRepository.java
View File

@ -3,7 +3,7 @@ package eu.dnetlib.data.mdstore.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import eu.dnetlib.dhp.schema.mdstore.MDStoreCurrentVersion;
import eu.dnetlib.data.mdstore.model.MDStoreCurrentVersion;
@Repository
public interface MDStoreCurrentVersionRepository extends JpaRepository<MDStoreCurrentVersion, String> {

2
apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/repository/MDStoreRepository.java → libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/repository/MDStoreRepository.java
View File

@ -3,7 +3,7 @@ package eu.dnetlib.data.mdstore.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import eu.dnetlib.dhp.schema.mdstore.MDStore;
import eu.dnetlib.data.mdstore.model.MDStore;
@Repository
public interface MDStoreRepository extends JpaRepository<MDStore, String> {

2
apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/repository/MDStoreVersionRepository.java → libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/repository/MDStoreVersionRepository.java
View File

@ -3,7 +3,7 @@ package eu.dnetlib.data.mdstore.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import eu.dnetlib.dhp.schema.mdstore.MDStoreVersion;
import eu.dnetlib.data.mdstore.model.MDStoreVersion;
@Repository
public interface MDStoreVersionRepository extends JpaRepository<MDStoreVersion, String> {

2
apps/dnet-is-application/src/main/java/eu/dnetlib/data/mdstore/repository/MDStoreWithInfoRepository.java → libs/dnet-data-services/src/main/java/eu/dnetlib/data/mdstore/repository/MDStoreWithInfoRepository.java
View File

@ -2,7 +2,7 @@ package eu.dnetlib.data.mdstore.repository;
import org.springframework.stereotype.Repository;
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
import eu.dnetlib.data.mdstore.model.MDStoreWithInfo;
import eu.dnetlib.utils.ReadOnlyRepository;
@Repository

175
libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStore.java Normal file
View File

@ -0,0 +1,175 @@
package eu.dnetlib.data.mdstore.model;
import java.io.Serializable;
import java.util.Date;
import java.util.Objects;
import java.util.UUID;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
@Entity
@Table(name = "mdstores")
public class MDStore implements Serializable {
/** */
private static final long serialVersionUID = 3160530489149700055L;
@Id
@Column(name = "id")
private String id;
@Column(name = "format")
private String format;
@Column(name = "layout")
private String layout;
@Column(name = "interpretation")
private String interpretation;
@Column(name = "datasource_name")
private String datasourceName;
@Column(name = "datasource_id")
private String datasourceId;
@Column(name = "api_id")
private String apiId;
@Column(name = "hdfs_path")
private String hdfsPath;
@Column(name = "creation_date")
@Temporal(TemporalType.TIMESTAMP)
private Date creationDate;
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getFormat() {
return format;
}
public void setFormat(final String format) {
this.format = format;
}
public String getLayout() {
return layout;
}
public void setLayout(final String layout) {
this.layout = layout;
}
public String getInterpretation() {
return interpretation;
}
public void setInterpretation(final String interpretation) {
this.interpretation = interpretation;
}
public String getDatasourceName() {
return datasourceName;
}
public void setDatasourceName(final String datasourceName) {
this.datasourceName = datasourceName;
}
public String getDatasourceId() {
return datasourceId;
}
public void setDatasourceId(final String datasourceId) {
this.datasourceId = datasourceId;
}
public String getApiId() {
return apiId;
}
public void setApiId(final String apiId) {
this.apiId = apiId;
}
public String getHdfsPath() {
return hdfsPath;
}
public void setHdfsPath(final String hdfsPath) {
this.hdfsPath = hdfsPath;
}
public Date getCreationDate() {
return creationDate;
}
public void setCreationDate(final Date creationDate) {
this.creationDate = creationDate;
}
public static MDStore newInstance(
final String format,
final String layout,
final String interpretation,
final String hdfsBasePath) {
return newInstance(format, layout, interpretation, null, null, null, hdfsBasePath);
}
public static MDStore newInstance(
final String format,
final String layout,
final String interpretation,
final String dsName,
final String dsId,
final String apiId,
final String hdfsBasePath) {
final String mdId = "md-" + UUID.randomUUID();
final MDStore md = new MDStore();
md.setId(mdId);
md.setFormat(format);
md.setLayout(layout);
md.setInterpretation(interpretation);
md.setCreationDate(new Date());
md.setDatasourceName(dsName);
md.setDatasourceId(dsId);
md.setApiId(apiId);
md.setHdfsPath(String.format("%s/%s", hdfsBasePath, mdId));
return md;
}
@Override
public String toString() {
return String
.format("MDStore [id=%s, format=%s, layout=%s, interpretation=%s, datasourceName=%s, datasourceId=%s, apiId=%s, hdfsPath=%s, creationDate=%s]", id, format, layout, interpretation, datasourceName, datasourceId, apiId, hdfsPath, creationDate);
}
@Override
public int hashCode() {
return Objects.hash(id);
}
@Override
public boolean equals(final Object obj) {
if (this == obj) { return true; }
if (!(obj instanceof MDStore)) { return false; }
final MDStore other = (MDStore) obj;
return Objects.equals(id, other.id);
}
}

69
libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreCurrentVersion.java Normal file
View File

@ -0,0 +1,69 @@
package eu.dnetlib.data.mdstore.model;
import java.io.Serializable;
import java.util.Objects;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Table;
@Entity
@Table(name = "mdstore_current_versions")
public class MDStoreCurrentVersion implements Serializable {
/** */
private static final long serialVersionUID = -4757725888593745773L;
@Id
@Column(name = "mdstore")
private String mdstore;
@Column(name = "current_version")
private String currentVersion;
public String getMdstore() {
return mdstore;
}
public void setMdstore(final String mdstore) {
this.mdstore = mdstore;
}
public String getCurrentVersion() {
return currentVersion;
}
public void setCurrentVersion(final String currentVersion) {
this.currentVersion = currentVersion;
}
public static MDStoreCurrentVersion newInstance(final String mdId, final String versionId) {
final MDStoreCurrentVersion cv = new MDStoreCurrentVersion();
cv.setMdstore(mdId);
cv.setCurrentVersion(versionId);
return cv;
}
public static MDStoreCurrentVersion newInstance(final MDStoreVersion v) {
return newInstance(v.getMdstore(), v.getId());
}
@Override
public String toString() {
return String.format("MDStoreCurrentVersion [mdstore=%s, currentVersion=%s]", mdstore, currentVersion);
}
@Override
public int hashCode() {
return Objects.hash(currentVersion, mdstore);
}
@Override
public boolean equals(final Object obj) {
if (this == obj) { return true; }
if (!(obj instanceof MDStoreCurrentVersion)) { return false; }
final MDStoreCurrentVersion other = (MDStoreCurrentVersion) obj;
return Objects.equals(currentVersion, other.currentVersion) && Objects.equals(mdstore, other.mdstore);
}
}

133
libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreVersion.java Normal file
View File

@ -0,0 +1,133 @@
package eu.dnetlib.data.mdstore.model;
import java.io.Serializable;
import java.util.Date;
import java.util.Objects;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
@Entity
@Table(name = "mdstore_versions")
public class MDStoreVersion implements Serializable {
/** */
private static final long serialVersionUID = -4763494442274298339L;
@Id
@Column(name = "id")
private String id;
@Column(name = "mdstore")
private String mdstore;
@Column(name = "writing")
private boolean writing;
@Column(name = "readcount")
private int readCount = 0;
@Column(name = "lastupdate")
@Temporal(TemporalType.TIMESTAMP)
private Date lastUpdate;
@Column(name = "size")
private long size = 0;
@Column(name = "hdfs_path")
private String hdfsPath;
public static MDStoreVersion newInstance(final String mdId, final boolean writing, final String hdfsBasePath) {
final MDStoreVersion v = new MDStoreVersion();
final String versionId = mdId + "-" + new Date().getTime();
v.setId(versionId);
v.setMdstore(mdId);
v.setLastUpdate(null);
v.setWriting(writing);
v.setReadCount(0);
v.setSize(0);
v.setHdfsPath(String.format("%s/%s/%s", hdfsBasePath, mdId, versionId));
return v;
}
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getMdstore() {
return mdstore;
}
public void setMdstore(final String mdstore) {
this.mdstore = mdstore;
}
public boolean isWriting() {
return writing;
}
public void setWriting(final boolean writing) {
this.writing = writing;
}
public int getReadCount() {
return readCount;
}
public void setReadCount(final int readCount) {
this.readCount = readCount;
}
public Date getLastUpdate() {
return lastUpdate;
}
public void setLastUpdate(final Date lastUpdate) {
this.lastUpdate = lastUpdate;
}
public long getSize() {
return size;
}
public void setSize(final long size) {
this.size = size;
}
public String getHdfsPath() {
return hdfsPath;
}
public void setHdfsPath(final String hdfsPath) {
this.hdfsPath = hdfsPath;
}
@Override
public String toString() {
return String
.format("MDStoreVersion [id=%s, mdstore=%s, writing=%s, readCount=%s, lastUpdate=%s, size=%s, hdfsPath=%s]", id, mdstore, writing, readCount, lastUpdate, size, hdfsPath);
}
@Override
public int hashCode() {
return Objects.hash(id);
}
@Override
public boolean equals(final Object obj) {
if (this == obj) { return true; }
if (!(obj instanceof MDStoreVersion)) { return false; }
final MDStoreVersion other = (MDStoreVersion) obj;
return Objects.equals(id, other.id);
}
}

186
libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MDStoreWithInfo.java Normal file
View File

@ -0,0 +1,186 @@
package eu.dnetlib.data.mdstore.model;
import java.io.Serializable;
import java.util.Date;
import java.util.Objects;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
@Entity
@Table(name = "mdstores_with_info")
public class MDStoreWithInfo implements Serializable {
/** */
private static final long serialVersionUID = -8445784770687571492L;
@Id
@Column(name = "id")
private String id;
@Column(name = "format")
private String format;
@Column(name = "layout")
private String layout;
@Column(name = "interpretation")
private String interpretation;
@Column(name = "datasource_name")
private String datasourceName;
@Column(name = "datasource_id")
private String datasourceId;
@Column(name = "api_id")
private String apiId;
@Column(name = "current_version")
private String currentVersion;
@Column(name = "creation_date")
@Temporal(TemporalType.TIMESTAMP)
private Date creationDate;
@Column(name = "lastupdate")
@Temporal(TemporalType.TIMESTAMP)
private Date lastUpdate;
@Column(name = "size")
private long size = 0;
@Column(name = "n_versions")
private long numberOfVersions = 0;
@Column(name = "hdfs_path")
private String hdfsPath;
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getFormat() {
return format;
}
public void setFormat(final String format) {
this.format = format;
}
public String getLayout() {
return layout;
}
public void setLayout(final String layout) {
this.layout = layout;
}
public String getInterpretation() {
return interpretation;
}
public void setInterpretation(final String interpretation) {
this.interpretation = interpretation;
}
public String getDatasourceName() {
return datasourceName;
}
public void setDatasourceName(final String datasourceName) {
this.datasourceName = datasourceName;
}
public String getDatasourceId() {
return datasourceId;
}
public void setDatasourceId(final String datasourceId) {
this.datasourceId = datasourceId;
}
public String getApiId() {
return apiId;
}
public void setApiId(final String apiId) {
this.apiId = apiId;
}
public String getCurrentVersion() {
return currentVersion;
}
public void setCurrentVersion(final String currentVersion) {
this.currentVersion = currentVersion;
}
public Date getCreationDate() {
return creationDate;
}
public void setCreationDate(final Date creationDate) {
this.creationDate = creationDate;
}
public Date getLastUpdate() {
return lastUpdate;
}
public void setLastUpdate(final Date lastUpdate) {
this.lastUpdate = lastUpdate;
}
public long getSize() {
return size;
}
public void setSize(final long size) {
this.size = size;
}
public long getNumberOfVersions() {
return numberOfVersions;
}
public void setNumberOfVersions(final long numberOfVersions) {
this.numberOfVersions = numberOfVersions;
}
public String getHdfsPath() {
return hdfsPath;
}
public void setHdfsPath(final String hdfsPath) {
this.hdfsPath = hdfsPath;
}
@Override
public String toString() {
return String
.format("MDStoreWithInfo [id=%s, format=%s, layout=%s, interpretation=%s, datasourceName=%s, datasourceId=%s, apiId=%s, currentVersion=%s, creationDate=%s, lastUpdate=%s, size=%s, numberOfVersions=%s, hdfsPath=%s]", id, format, layout, interpretation, datasourceName, datasourceId, apiId, currentVersion, creationDate, lastUpdate, size, numberOfVersions, hdfsPath);
}
@Override
public int hashCode() {
return Objects.hash(id);
}
@Override
public boolean equals(final Object obj) {
if (this == obj) { return true; }
if (!(obj instanceof MDStoreWithInfo)) { return false; }
final MDStoreWithInfo other = (MDStoreWithInfo) obj;
return Objects.equals(id, other.id);
}
}

126
libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/MetadataRecord.java Normal file
View File

@ -0,0 +1,126 @@
package eu.dnetlib.data.mdstore.model;
import java.io.Serializable;
import org.apache.commons.codec.digest.DigestUtils;
/**
* This class models a record in a Metadata store collection
*/
public class MetadataRecord implements Serializable {
private static final long serialVersionUID = -226995288313973437L;
/** The D-Net Identifier associated to the record */
private String id;
/** The original Identifier of the record */
private String originalId;
/** The encoding of the record, should be JSON or XML */
private String encoding;
/**
* The information about the provenance of the record see @{@link Provenance} for the model of this information
*/
private Provenance provenance;
/** The content of the metadata */
private String body;
/** the date when the record has been stored */
private Long dateOfCollection;
/** the date when the record has been stored */
private Long dateOfTransformation;
public MetadataRecord() {
}
public MetadataRecord(
final String originalId,
final String encoding,
final Provenance provenance,
final String body,
final Long dateOfCollection) {
this.originalId = originalId;
this.encoding = encoding;
this.provenance = provenance;
this.body = body;
this.dateOfCollection = dateOfCollection;
this.id = generateIdentifier(originalId, this.provenance.getNsPrefix());
}
public static String generateIdentifier(final String originalId, final String nsPrefix) {
return String.format("%s::%s", nsPrefix, DigestUtils.md5Hex(originalId));
}
public String getId() {
return id;
}
public void setId(final String id) {
this.id = id;
}
public String getOriginalId() {
return originalId;
}
public void setOriginalId(final String originalId) {
this.originalId = originalId;
}
public String getEncoding() {
return encoding;
}
public void setEncoding(final String encoding) {
this.encoding = encoding;
}
public Provenance getProvenance() {
return provenance;
}
public void setProvenance(final Provenance provenance) {
this.provenance = provenance;
}
public String getBody() {
return body;
}
public void setBody(final String body) {
this.body = body;
}
public Long getDateOfCollection() {
return dateOfCollection;
}
public void setDateOfCollection(final Long dateOfCollection) {
this.dateOfCollection = dateOfCollection;
}
public Long getDateOfTransformation() {
return dateOfTransformation;
}
public void setDateOfTransformation(final Long dateOfTransformation) {
this.dateOfTransformation = dateOfTransformation;
}
@Override
public boolean equals(final Object o) {
if (!(o instanceof MetadataRecord)) { return false; }
return ((MetadataRecord) o).getId().equalsIgnoreCase(id);
}
@Override
public int hashCode() {
return id.hashCode();
}
}

50
libs/dnet-is-common/src/main/java/eu/dnetlib/data/mdstore/model/Provenance.java Normal file
View File

@ -0,0 +1,50 @@
package eu.dnetlib.data.mdstore.model;
import java.io.Serializable;
/**
* @author Sandro La Bruzzo
* <p>
* Provenace class models the provenance of the record in the metadataStore It contains the identifier and the name of the
* datasource that gives the record
*/
public class Provenance implements Serializable {
private String datasourceId;
private String datasourceName;
private String nsPrefix;
public Provenance() {}
public Provenance(final String datasourceId, final String datasourceName, final String nsPrefix) {
this.datasourceId = datasourceId;
this.datasourceName = datasourceName;
this.nsPrefix = nsPrefix;
}
public String getDatasourceId() {
return datasourceId;
}
public void setDatasourceId(final String datasourceId) {
this.datasourceId = datasourceId;
}
public String getDatasourceName() {
return datasourceName;
}
public void setDatasourceName(final String datasourceName) {
this.datasourceName = datasourceName;
}
public String getNsPrefix() {
return nsPrefix;
}
public void setNsPrefix(final String nsPrefix) {
this.nsPrefix = nsPrefix;
}
}

0
apps/dnet-is-application/src/main/java/eu/dnetlib/errors/DsmException.java → libs/dnet-is-common/src/main/java/eu/dnetlib/errors/DsmException.java
View File

0
apps/dnet-is-application/src/main/java/eu/dnetlib/errors/DsmForbiddenException.java → libs/dnet-is-common/src/main/java/eu/dnetlib/errors/DsmForbiddenException.java
View File

0
apps/dnet-is-application/src/main/java/eu/dnetlib/errors/DsmNotFoundException.java → libs/dnet-is-common/src/main/java/eu/dnetlib/errors/DsmNotFoundException.java
View File

0
apps/dnet-is-application/src/main/java/eu/dnetlib/errors/InformationServiceException.java → libs/dnet-is-common/src/main/java/eu/dnetlib/errors/InformationServiceException.java
View File

0
apps/dnet-is-application/src/main/java/eu/dnetlib/errors/MDStoreManagerException.java → libs/dnet-is-common/src/main/java/eu/dnetlib/errors/MDStoreManagerException.java
View File

0
apps/dnet-is-application/src/main/java/eu/dnetlib/utils/DateUtils.java → libs/dnet-is-common/src/main/java/eu/dnetlib/utils/DateUtils.java
View File

0
apps/dnet-is-application/src/main/java/eu/dnetlib/utils/ReadOnlyRepository.java → libs/dnet-is-common/src/main/java/eu/dnetlib/utils/ReadOnlyRepository.java
View File

4
apps/dnet-is-application/src/main/java/eu/dnetlib/utils/XmlIndenter.java → libs/dnet-is-common/src/main/java/eu/dnetlib/utils/XmlIndenter.java
View File

@ -10,11 +10,9 @@ import org.dom4j.Node;
import org.dom4j.io.OutputFormat;
import org.dom4j.io.XMLWriter;
import eu.dnetlib.is.resource.SimpleResourceService;
public class XmlIndenter {
private static final Log log = LogFactory.getLog(SimpleResourceService.class);
private static final Log log = LogFactory.getLog(XmlIndenter.class);
public static String indent(final String xml) {
try {

1
libs/pom.xml
View File

@ -17,6 +17,7 @@
<module>dnet-openaire-broker-common</module>
<module>dnet-broker-apps-common</module>
<module>dnet-is-common</module>
<module>dnet-data-services</module>
</modules>
</project>