D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

added versions support

This commit is contained in:
Enrico Ottonello 2019-03-20 15:36:23 +01:00
parent 46bc3e1f53
commit 042d477878
12 changed files with 275 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
dhp-applications/dhp-mdstore-manager-app/src/main/java/eu/dnetlib/data/mdstore/manager/MainApplication.java
View File

@ -1,6 +1,5 @@
package eu.dnetlib.data.mdstore.manager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.SpringApplication;
@ -33,7 +32,7 @@ public class MainApplication {
return new Docket(DocumentationType.SWAGGER_2)
.select()
.apis(RequestHandlerSelectors.any())
.paths(p -> p.startsWith("/api/"))
.paths(p -> p.startsWith("/mdstores"))
.build().apiInfo(new ApiInfoBuilder()
.title("MDStore Manager APIs")
.description("APIs documentation")
@ -45,5 +44,4 @@ public class MainApplication {
}
}

132
dhp-applications/dhp-mdstore-manager-app/src/main/java/eu/dnetlib/data/mdstore/manager/controller/MDStoreController.java
View File

@ -1,79 +1,155 @@
package eu.dnetlib.data.mdstore.manager.controller;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
import javax.transaction.Transactional;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.data.mdstore.manager.MDStoreManagerException;
import eu.dnetlib.data.mdstore.manager.exceptions.MDStoreManagerException;
import eu.dnetlib.data.mdstore.manager.exceptions.NoContentException;
import eu.dnetlib.data.mdstore.manager.model.MDStore;
import eu.dnetlib.data.mdstore.manager.model.MDStoreCurrentVersion;
import eu.dnetlib.data.mdstore.manager.model.MDStoreVersion;
import eu.dnetlib.data.mdstore.manager.model.MDStoreWithInfo;
import eu.dnetlib.data.mdstore.manager.model.Transaction;
import eu.dnetlib.data.mdstore.manager.repository.MDStoreCurrentVersionRepository;
import eu.dnetlib.data.mdstore.manager.repository.MDStoreRepository;
import eu.dnetlib.data.mdstore.manager.repository.MDStoreVersionRepository;
import eu.dnetlib.data.mdstore.manager.repository.MDStoreWithInfoRepository;
import eu.dnetlib.data.mdstore.manager.repository.TransactionRepository;
@RestController
@RequestMapping("/api/mdstores")
@RequestMapping("/mdstores")
public class MDStoreController {
@Autowired
private MDStoreRepository mdstoreRepository;
@Autowired
private TransactionRepository transactionRepository;
private MDStoreVersionRepository mdstoreVersionRepository;
@Autowired
private MDStoreCurrentVersionRepository mdstoreCurrentVersionRepository;
@Autowired
private MDStoreWithInfoRepository mdstoreWithInfoRepository;
@Autowired
private JdbcTemplate jdbcTemplate;
@RequestMapping(value = "/", method = RequestMethod.GET)
public final Iterable<MDStoreWithInfo> find() {
public Iterable<MDStoreWithInfo> find() {
return mdstoreWithInfoRepository.findAll();
}
@RequestMapping(value = "/identifiers", method = RequestMethod.GET)
public final List<String> findIdentifiers() {
@RequestMapping(value = "/ids", method = RequestMethod.GET)
public List<String> findIdentifiers() {
return mdstoreRepository.findAll().stream().map(MDStore::getId).collect(Collectors.toList());
}
@RequestMapping(value = "/byId/{id}", method = RequestMethod.GET)
public final MDStoreWithInfo get(@PathVariable final String id) {
return mdstoreWithInfoRepository.findById(id).orElse(null);
@RequestMapping(value = "/mdstore/{mdId}", method = RequestMethod.GET)
public MDStoreWithInfo get(@PathVariable final String mdId) throws NoContentException {
return mdstoreWithInfoRepository.findById(mdId).orElseThrow(() -> new NoContentException("Missing mdstore: " + mdId));
}
@RequestMapping(value = "/count", method = RequestMethod.GET)
public final long count() {
public long count() {
return mdstoreRepository.count();
}
@RequestMapping(value = "/new/{format}/{layout}/{interpretation}", method = RequestMethod.PUT)
public MDStoreWithInfo createMDStore(
@PathVariable String format,
@PathVariable String layout,
@PathVariable String interpretation,
@RequestParam(required=false) String dsId,
@RequestParam(required=false) String apiId) throws MDStoreManagerException {
@PathVariable final String format,
@PathVariable final String layout,
@PathVariable final String interpretation,
@RequestParam(required = false) final String dsId,
@RequestParam(required = false) final String apiId) throws MDStoreManagerException {
final String id = _createMDStore(format, layout, interpretation, dsId, apiId);
return mdstoreWithInfoRepository.findById(id).orElseThrow(() -> new MDStoreManagerException("MDStore not found"));
}
@Transactional
private String _createMDStore(final String format, final String layout, final String interpretation, final String dsId, final String apiId) {
final MDStore md = MDStore.newInstance(dsId, apiId, format, layout, interpretation);
mdstoreRepository.save(md);
final Transaction t = Transaction.newInstance(md.getId());
t.setCurrent(true);
transactionRepository.save(t);
final MDStoreVersion v = MDStoreVersion.newInstance(md.getId(), false);
v.setLastUpdate(new Date());
mdstoreVersionRepository.save(v);
mdstoreCurrentVersionRepository.save(MDStoreCurrentVersion.newInstance(v));
return mdstoreWithInfoRepository.findById(md.getId()).orElseThrow(() -> new MDStoreManagerException("MDStore not found"));
return md.getId();
}
@RequestMapping(value = "/{id}", method = RequestMethod.DELETE)
public void delete(@PathVariable final String id) throws MDStoreManagerException {
if (transactionRepository.countByMdstoreAndActive(id, true) == 0) {
transactionRepository.deleteByMdstore(id);
mdstoreRepository.deleteById(id);
} else {
throw new MDStoreManagerException("Active transactions found on mdstore : " + id);
@Transactional
@RequestMapping(value = "/mdstore/{mdId}", method = RequestMethod.DELETE)
public void delete(@PathVariable final String mdId) throws MDStoreManagerException {
if (!mdstoreRepository.existsById(mdId)) { throw new NoContentException("On delete MDStore not found " + "[" + mdId + "]"); }
if (mdstoreVersionRepository.countByMdstoreAndReadCountGreaterThan(mdId,
0) > 0) { throw new MDStoreManagerException("Read transactions found on mdstore : " + mdId); }
if (mdstoreVersionRepository.countByMdstoreAndWriting(mdId,
true) > 0) { throw new MDStoreManagerException("Write transactions found on mdstore : " + mdId); }
mdstoreCurrentVersionRepository.deleteById(mdId);
mdstoreVersionRepository.deleteByMdstore(mdId);
mdstoreRepository.deleteById(mdId);
}
@Transactional
@RequestMapping(value = "/mdstore/{mdId}/newVersion", method = RequestMethod.GET)
public MDStoreVersion prepareNewVersion1(@PathVariable final String mdId) {
final MDStoreVersion v = MDStoreVersion.newInstance(mdId, true);
mdstoreVersionRepository.save(v);
return v;
}
@Transactional
@RequestMapping(value = "/version/{versionId}/commit/{size}", method = RequestMethod.GET)
public void commitVersion(@PathVariable final String versionId, @PathVariable final int size) throws NoContentException {
final MDStoreVersion v = mdstoreVersionRepository.findById(versionId).orElseThrow(() -> new NoContentException("Invalid version: " + versionId));
mdstoreCurrentVersionRepository.save(MDStoreCurrentVersion.newInstance(v));
v.setWriting(false);
v.setSize(size);
v.setLastUpdate(new Date());
mdstoreVersionRepository.save(v);
}
@RequestMapping(value = "/versions/expired", method = RequestMethod.GET)
public List<String> listExpiredVersions() {
return jdbcTemplate.queryForList(
"select v.id from mdstore_versions v left outer join mdstore_current_versions cv on (v.id = cv.current_version) where v.writing = false and v.readcount = 0 and cv.mdstore is null;",
String.class);
}
@Transactional
@RequestMapping(value = "/version/{versionId}", method = RequestMethod.DELETE)
public void deleteVersion(@PathVariable final String versionId) throws MDStoreManagerException {
_deleteVersion(versionId);
}
@Transactional
@RequestMapping(value = "/versions/deleteList", method = RequestMethod.POST)
public void deleteVersions(@RequestBody final List<String> versions) throws MDStoreManagerException {
for (final String v : versions) {
_deleteVersion(v);
}
}
private void _deleteVersion(final String versionId) throws MDStoreManagerException {
final MDStoreVersion v = mdstoreVersionRepository.findById(versionId).orElseThrow(() -> new MDStoreManagerException("Version not found"));
if (v.isWriting()) { throw new MDStoreManagerException("I cannot delete this version because it is in write mode"); }
if (v.getReadCount() > 0) { throw new MDStoreManagerException("I cannot delete this version because it is in read mode"); }
if (mdstoreCurrentVersionRepository
.countByCurrentVersion(versionId) > 0) { throw new MDStoreManagerException("I cannot delete this version because it is the current version"); }
mdstoreVersionRepository.delete(v);
}
}

4
dhp-applications/dhp-mdstore-manager-app/src/main/java/eu/dnetlib/data/mdstore/manager/SwaggerController.java → dhp-applications/dhp-mdstore-manager-app/src/main/java/eu/dnetlib/data/mdstore/manager/controller/SwaggerController.java
View File

@ -1,4 +1,4 @@
package eu.dnetlib.data.mdstore.manager;
package eu.dnetlib.data.mdstore.manager.controller;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
@ -7,7 +7,7 @@ import org.springframework.web.bind.annotation.RequestMethod;
@Controller
public class SwaggerController {
@RequestMapping(value = { "/", "/apidoc", "/api-doc", "/doc", "/swagger" }, method = RequestMethod.GET)
@RequestMapping(value = { "/apidoc", "/api-doc", "/doc", "/swagger" }, method = RequestMethod.GET)
public String apiDoc() {
return "redirect:swagger-ui.html";
}

41
dhp-applications/dhp-mdstore-manager-app/src/main/java/eu/dnetlib/data/mdstore/manager/controller/TransactionController.java
View File

@ -1,41 +0,0 @@
package eu.dnetlib.data.mdstore.manager.controller;
import java.util.List;
import java.util.stream.Collectors;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
import eu.dnetlib.data.mdstore.manager.model.Transaction;
import eu.dnetlib.data.mdstore.manager.repository.TransactionRepository;
@RestController
@RequestMapping("/api/transactions")
public class TransactionController {
@Autowired
private TransactionRepository repo;
@RequestMapping(value = "/", method = RequestMethod.GET)
public final List<Transaction> find() {
return repo.findAll();
}
@RequestMapping(value = "/identifiers", method = RequestMethod.GET)
public final List<String> findIdentifiers() {
return repo.findAll().stream().map(Transaction::getId).collect(Collectors.toList());
}
@RequestMapping(value = "/{id}", method = RequestMethod.GET)
public final Transaction get(@PathVariable final String id) {
return repo.findById(id).orElse(null);
}
@RequestMapping(value = "/count", method = RequestMethod.GET)
public final long count() {
return repo.count();
}
}

6
dhp-applications/dhp-mdstore-manager-app/src/main/java/eu/dnetlib/data/mdstore/manager/MDStoreManagerException.java → dhp-applications/dhp-mdstore-manager-app/src/main/java/eu/dnetlib/data/mdstore/manager/exceptions/MDStoreManagerException.java
View File

@ -1,5 +1,9 @@
package eu.dnetlib.data.mdstore.manager;
package eu.dnetlib.data.mdstore.manager.exceptions;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.ResponseStatus;
@ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR)
public class MDStoreManagerException extends Exception{
/**

36
dhp-applications/dhp-mdstore-manager-app/src/main/java/eu/dnetlib/data/mdstore/manager/exceptions/NoContentException.java Normal file
View File

@ -0,0 +1,36 @@
package eu.dnetlib.data.mdstore.manager.exceptions;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.ResponseStatus;
@ResponseStatus(HttpStatus.NO_CONTENT)
public class NoContentException extends MDStoreManagerException{
/**
*
*/
private static final long serialVersionUID = -278683963172303773L;
public NoContentException() {
super();
}
public NoContentException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
public NoContentException(String message, Throwable cause) {
super(message, cause);
}
public NoContentException(String message) {
super(message);
}
public NoContentException(Throwable cause) {
super(cause);
}
}

56
dhp-applications/dhp-mdstore-manager-app/src/main/java/eu/dnetlib/data/mdstore/manager/model/MDStoreCurrentVersion.java Normal file
View File

@ -0,0 +1,56 @@
package eu.dnetlib.data.mdstore.manager.model;
import java.io.Serializable;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Table;
@Entity
@Table(name = "mdstore_current_versions")
public class MDStoreCurrentVersion implements Serializable {
/**
*
*/
private static final long serialVersionUID = -4757725888593745773L;
@Id
@Column(name = "mdstore")
private String mdstore;
@Column(name = "current_version")
private String currentVersion;
public String getMdstore() {
return mdstore;
}
public void setMdstore(String mdstore) {
this.mdstore = mdstore;
}
public String getCurrentVersion() {
return currentVersion;
}
public void setCurrentVersion(String currentVersion) {
this.currentVersion = currentVersion;
}
public static long getSerialversionuid() {
return serialVersionUID;
}
public static MDStoreCurrentVersion newInstance(final String mdId, final String versionId) {
final MDStoreCurrentVersion cv = new MDStoreCurrentVersion();
cv.setMdstore(mdId);
cv.setCurrentVersion(versionId);
return cv;
}
public static MDStoreCurrentVersion newInstance(final MDStoreVersion v) {
return newInstance(v.getMdstore(), v.getId());
}
}

53
dhp-applications/dhp-mdstore-manager-app/src/main/java/eu/dnetlib/data/mdstore/manager/model/Transaction.java → dhp-applications/dhp-mdstore-manager-app/src/main/java/eu/dnetlib/data/mdstore/manager/model/MDStoreVersion.java
View File

@ -11,8 +11,9 @@ import javax.persistence.Temporal;
import javax.persistence.TemporalType;
@Entity
@Table(name = "transactions")
public class Transaction implements Serializable{
@Table(name = "mdstore_versions")
public class MDStoreVersion implements Serializable {
/**
*
*/
@ -25,11 +26,8 @@ public class Transaction implements Serializable{
@Column(name = "mdstore")
private String mdstore;
@Column(name = "current")
private boolean current;
@Column(name = "active")
private boolean active;
@Column(name = "writing")
private boolean writing;
@Column(name = "readcount")
private int readCount;
@ -41,6 +39,17 @@ public class Transaction implements Serializable{
@Column(name = "size")
private int size;
public static MDStoreVersion newInstance(final String mdId, boolean writing) {
final MDStoreVersion t = new MDStoreVersion();
t.setId(mdId + "-" + new Date().getTime());
t.setMdstore(mdId);
t.setLastUpdate(null);
t.setWriting(writing);
t.setReadCount(0);
t.setSize(0);
return t;
}
public String getId() {
return id;
}
@ -57,20 +66,12 @@ public class Transaction implements Serializable{
this.mdstore = mdstore;
}
public boolean isCurrent() {
return current;
public boolean isWriting() {
return writing;
}
public void setCurrent(boolean current) {
this.current = current;
}
public boolean isActive() {
return active;
}
public void setActive(boolean active) {
this.active = active;
public void setWriting(boolean writing) {
this.writing = writing;
}
public int getReadCount() {
@ -97,18 +98,4 @@ public class Transaction implements Serializable{
this.size = size;
}
public static Transaction newInstance(final String mdId) {
final Transaction t = new Transaction();
t.setId(mdId + "-" + new Date().getTime());
t.setMdstore(mdId);
t.setLastUpdate(null);
t.setActive(false);
t.setCurrent(false);
t.setReadCount(0);
t.setSize(0);
return t;
}
}

12
dhp-applications/dhp-mdstore-manager-app/src/main/java/eu/dnetlib/data/mdstore/manager/repository/MDStoreCurrentVersionRepository.java Normal file
View File

@ -0,0 +1,12 @@
package eu.dnetlib.data.mdstore.manager.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import eu.dnetlib.data.mdstore.manager.model.MDStoreCurrentVersion;
@Repository
public interface MDStoreCurrentVersionRepository extends JpaRepository<MDStoreCurrentVersion, String> {
long countByCurrentVersion(String versionId);
}

17
dhp-applications/dhp-mdstore-manager-app/src/main/java/eu/dnetlib/data/mdstore/manager/repository/MDStoreVersionRepository.java Normal file
View File

@ -0,0 +1,17 @@
package eu.dnetlib.data.mdstore.manager.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import eu.dnetlib.data.mdstore.manager.model.MDStoreVersion;
@Repository
public interface MDStoreVersionRepository extends JpaRepository<MDStoreVersion, String> {
void deleteByMdstore(String id);
int countByMdstoreAndWriting(String id, boolean b);
int countByMdstoreAndReadCountGreaterThan(String id, int count);
}

14
dhp-applications/dhp-mdstore-manager-app/src/main/java/eu/dnetlib/data/mdstore/manager/repository/TransactionRepository.java
View File

@ -1,14 +0,0 @@
package eu.dnetlib.data.mdstore.manager.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import eu.dnetlib.data.mdstore.manager.model.Transaction;
@Repository
public interface TransactionRepository extends JpaRepository<Transaction, String> {
void deleteByMdstore(String id);
int countByMdstoreAndActive(String id, boolean active);
}

24
dhp-applications/dhp-mdstore-manager-app/src/main/resources/schema.sql
View File

@ -1,5 +1,6 @@
DROP VIEW IF EXISTS mdstores_with_info;
DROP TABLE IF EXISTS transactions;
DROP TABLE IF EXISTS mdstore_current_versions;
DROP TABLE IF EXISTS mdstore_versions;
DROP TABLE IF EXISTS mdstores;
CREATE TABLE mdstores (
@ -11,16 +12,20 @@ CREATE TABLE mdstores (
api_id text
);
CREATE TABLE transactions (
CREATE TABLE mdstore_versions (
id text PRIMARY KEY,
mdstore text REFERENCES mdstores(id),
current boolean,
active boolean,
writing boolean,
readcount int,
lastupdate timestamp,
size int
);
CREATE TABLE mdstore_current_versions (
mdstore text PRIMARY KEY REFERENCES mdstores(id),
current_version text REFERENCES mdstore_versions(id)
);
CREATE VIEW mdstores_with_info AS SELECT
md.id AS id,
md.format AS format,
@ -28,11 +33,10 @@ CREATE VIEW mdstores_with_info AS SELECT
md.interpretation AS interpretation,
md.datasource_id AS datasource_id,
md.api_id AS api_id,
t.id AS current_version,
t.lastupdate AS lastupdate,
t.size AS size
cv.current_version AS current_version,
v.lastupdate AS lastupdate,
v.size AS size
FROM
mdstores md
LEFT OUTER JOIN transactions t ON (md.id = t.mdstore)
WHERE
t.current = TRUE;
LEFT OUTER JOIN mdstore_current_versions cv ON (md.id = cv.mdstore)
LEFT OUTER JOIN mdstore_versions v ON (cv.current_version = v.id);