no message

dmp-editor
Ioannis Kalyvas 5 years ago
parent efb7af10b5
commit 78b91ee3e0

@ -1,6 +1,6 @@
TAG=6.2.1
ENV=dev
TAG=6.3.0
ENV=prod
PROFILE=production
AOT=no-aot
ELASTIC_VERSION=6.2.1
AOT=aot
ELASTIC_VERSION=6.3.0
ELASTIC_PASSWORD=changeme

2
.gitignore vendored

@ -25,3 +25,5 @@ dmp-backend/web/src/main/ui-resources/static/
dmp-backend/data/target/data-1.0-SNAPSHOT.jar
dmp-backend/data/target/
dmp-backend/queryable/target/
dmp-backend/elastic/target/
dmp-backend/queryengine/target/

@ -1,3 +1,3 @@
TAG=6.2.1
ELASTIC_VERSION=6.2.1
TAG=6.3.1
ELASTIC_VERSION=6.3.1
ELASTIC_PASSWORD=changeme

@ -100,7 +100,72 @@ services:
networks: ['stack']
depends_on: ['kibana']
#volumes:
##########################DOCSBOX######################################################################
web:
restart: always
build: ./docsbox-master/docsbox
expose:
- "8000"
links:
- redis:redis
volumes:
- docsbox:/home/docsbox
- media:/home/docsbox/media
command: gunicorn -b :8000 docsbox:app
networks: ['stack']
rqworker:
restart: always
build: ./docsbox-master/docsbox
links:
- redis:redis
volumes:
- web
command: rq worker -c docsbox.settings
networks: ['stack']
rqscheduler:
restart: always
build: ./docsbox-master/docsbox
links:
- redis:redis
volumes:
- web
command: rqscheduler -H redis -p 6379 -d 0
networks: ['stack']
nginx:
restart: always
build: ./docsbox-master/nginx/
ports:
- "81:80"
volumes:
- web
links:
- web:web
networks: ['stack']
redis:
restart: always
image: redis:latest
expose:
- "6379"
volumes:
- redisdata:/data
networks: ['stack']
##########################SETTIGNS######################################################################
volumes:
#esdata:
# driver: local
#driver: local
redisdata:
driver: local
docsbox:
driver: local
media:
driver: local
networks: {stack: {}}

@ -1,5 +1,11 @@
FROM openjdk:8-jdk-alpine
RUN apk add --update \
curl \
&& rm -rf /var/cache/apk/*
VOLUME /tmp
ARG PROFILE=dev
ARG PROFILE=production
ENV PROF $PROFILE
ADD web/src/main/resources/ProjectConfiguration.xml /tmp/ProjectConfiguration.xml
ADD web/src/main/resources/ExternalUrls.xml /tmp/ExternalUrls.xml
ADD web/target/web-1.0-SNAPSHOT.jar app.jar
ENTRYPOINT ["java","-Djava.security.egd=file:/dev/./urandom -Dspring.profiles.active=${PROFILE}","-jar","/app.jar"]
ENTRYPOINT ["java","-Djava.security.egd=file:/dev/./urandom" ,"-Dspring.profiles.active=${PROF}","-jar","/app.jar"]

@ -21,5 +21,10 @@
<artifactId>queryable</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>eu.eudat</groupId>
<artifactId>elastic</artifactId>
<version>1.0.0-SNAPSHOT</version>
</dependency>
</dependencies>
</project>

@ -1,6 +1,7 @@
package eu.eudat.data.dao.criteria;
import eu.eudat.data.entities.Dataset;
import eu.eudat.elastic.entities.Tag;
import java.util.Date;
import java.util.List;
@ -12,8 +13,10 @@ public class DatasetCriteria extends Criteria<Dataset> {
private Date periodStart;
private Date periodEnd;
private List<UUID> dmpIds;
private List<Tag> tags;
private boolean allVersions;
public boolean getAllVersions() {
return allVersions;
}
@ -53,4 +56,12 @@ public class DatasetCriteria extends Criteria<Dataset> {
public void setDmpIds(List<UUID> dmpIds) {
this.dmpIds = dmpIds;
}
public List<Tag> getTags() {
return tags;
}
public void setTags(List<Tag> tags) {
this.tags = tags;
}
}

@ -14,15 +14,23 @@ import java.util.stream.Collectors;
@NamedEntityGraphs({
@NamedEntityGraph(
name = "datasetListingModel",
attributeNodes = {@NamedAttributeNode("services"), @NamedAttributeNode("datasetDataRepositories"), @NamedAttributeNode("datasetExternalDatasets"), @NamedAttributeNode("registries"),
attributeNodes = {@NamedAttributeNode("services"), @NamedAttributeNode(value = "datasetDataRepositories", subgraph = "datasetDataRepositories"), @NamedAttributeNode("datasetExternalDatasets"), @NamedAttributeNode("registries"),
@NamedAttributeNode(value = "dmp", subgraph = "dmp"), @NamedAttributeNode("profile"), @NamedAttributeNode("creator")},
subgraphs = @NamedSubgraph(name = "dmp", attributeNodes = {@NamedAttributeNode("creator"), @NamedAttributeNode("users")})),
subgraphs = {
@NamedSubgraph(name = "dmp", attributeNodes = {@NamedAttributeNode("creator"), @NamedAttributeNode("users")}),
@NamedSubgraph(name = "datasetDataRepositories", attributeNodes = {@NamedAttributeNode("dataRepository")})
}),
@NamedEntityGraph(
name = "datasetWizardModel",
attributeNodes = {@NamedAttributeNode("services"), @NamedAttributeNode("datasetDataRepositories"), @NamedAttributeNode("datasetExternalDatasets"), @NamedAttributeNode("registries"),
@NamedAttributeNode("dmp"), @NamedAttributeNode("profile"), @NamedAttributeNode("creator")}),
@NamedEntityGraph(
name = "datasetRecentActivity",
attributeNodes = {@NamedAttributeNode(value = "dmp", subgraph = "dmp")},
subgraphs = @NamedSubgraph(name = "dmp", attributeNodes = {@NamedAttributeNode("users")})),
@NamedEntityGraph(
name = "datasetDataRepositories",
attributeNodes = {@NamedAttributeNode(value = "dmp", subgraph = "dmp"), @NamedAttributeNode("creator")},
subgraphs = @NamedSubgraph(name = "dmp", attributeNodes = {@NamedAttributeNode("creator"), @NamedAttributeNode("users")}))
})
@ -291,10 +299,10 @@ public class Dataset implements DataEntity<Dataset, UUID> {
@Override
public void update(Dataset entity) {
this.setRegistries(entity.getRegistries());
if(this.getDatasetDataRepositories()==null) this.setDatasetDataRepositories(new HashSet<>());
if(!this.getDatasetDataRepositories().containsAll(entity.getDatasetDataRepositories())){
if (this.getDatasetDataRepositories() == null) this.setDatasetDataRepositories(new HashSet<>());
if (!this.getDatasetDataRepositories().containsAll(entity.getDatasetDataRepositories())) {
this.getDatasetDataRepositories().removeAll(this.getDatasetDataRepositories());
this.getDatasetDataRepositories().addAll(entity.getDatasetDataRepositories().stream().map(item->{
this.getDatasetDataRepositories().addAll(entity.getDatasetDataRepositories().stream().map(item -> {
item.setDataset(this);
return item;
}).collect(Collectors.toList()));
@ -302,8 +310,8 @@ public class Dataset implements DataEntity<Dataset, UUID> {
this.setDescription(entity.getDescription());
this.setLabel(entity.getLabel());
this.setProperties(entity.getProperties());
if(this.getDatasetExternalDatasets()==null) this.setDatasetExternalDatasets(new HashSet<>());
if(!this.getDatasetExternalDatasets().containsAll(entity.getDatasetExternalDatasets())) {
if (this.getDatasetExternalDatasets() == null) this.setDatasetExternalDatasets(new HashSet<>());
if (!this.getDatasetExternalDatasets().containsAll(entity.getDatasetExternalDatasets())) {
this.getDatasetExternalDatasets().removeAll(this.getDatasetExternalDatasets());
this.getDatasetExternalDatasets().addAll(entity.getDatasetExternalDatasets().stream().map(item -> {
item.setDataset(this);

@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>dmp-backend</artifactId>
<groupId>eu.eudat</groupId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>elastic</artifactId>
</project>

@ -0,0 +1,7 @@
package eu.eudat.elastic.criteria;
/**
* Created by ikalyvas on 7/5/2018.
*/
public abstract class Criteria {
}

@ -0,0 +1,20 @@
package eu.eudat.elastic.criteria;
import eu.eudat.elastic.entities.Tag;
import java.util.List;
/**
* Created by ikalyvas on 7/5/2018.
*/
public class DatasetCriteria extends Criteria {
public List<Tag> tags;
public List<Tag> getTags() {
return tags;
}
public void setTags(List<Tag> tags) {
this.tags = tags;
}
}

@ -0,0 +1,7 @@
package eu.eudat.elastic.criteria;
/**
* Created by ikalyvas on 7/5/2018.
*/
public class TagCriteria extends Criteria {
}

@ -0,0 +1,60 @@
package eu.eudat.elastic.entities;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
/**
* Created by ikalyvas on 7/5/2018.
*/
public class Dataset implements ElasticEntity<Dataset> {
private String id;
private List<Tag> tags = new LinkedList<>();
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public List<Tag> getTags() {
return tags;
}
public void setTags(List<Tag> tags) {
this.tags = tags;
}
@Override
public XContentBuilder toElasticEntity(XContentBuilder builder) throws IOException {
builder.startObject();
builder.field("id", this.id);
builder.startArray("tags");
this.tags.forEach(x -> {
try {
x.toElasticEntity(builder);
} catch (IOException e) {
e.printStackTrace();
}
});
builder.endArray();
builder.endObject();
return builder;
}
@Override
public Dataset fromElasticEntity(Map<String, Object> fields) {
if (fields != null) {
this.id = (String) fields.get("id");
this.tags = ((List<Tag>) fields.get("tags"));
}
return this;
}
}

@ -0,0 +1,15 @@
package eu.eudat.elastic.entities;
import org.elasticsearch.common.document.DocumentField;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.IOException;
import java.util.Map;
/**
* Created by ikalyvas on 7/5/2018.
*/
public interface ElasticEntity<T> {
XContentBuilder toElasticEntity(XContentBuilder builder) throws IOException;
T fromElasticEntity(Map<String, Object> fields);
}

@ -0,0 +1,45 @@
package eu.eudat.elastic.entities;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.IOException;
import java.util.Map;
/**
* Created by ikalyvas on 7/5/2018.
*/
public class Tag implements ElasticEntity {
private String id;
private String name;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
@Override
public XContentBuilder toElasticEntity(XContentBuilder builder) throws IOException {
builder.startObject();
builder.field("id", this.id);
builder.field("name", this.name);
builder.endObject();
return builder;
}
@Override
public Object fromElasticEntity(Map fields) {
return null;
}
}

@ -0,0 +1,72 @@
package eu.eudat.elastic.repository;
import eu.eudat.elastic.criteria.DatasetCriteria;
import eu.eudat.elastic.entities.Dataset;
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
/**
* Created by ikalyvas on 7/5/2018.
*/
@Service("datasetRepository")
public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteria> {
@Autowired
public DatasetRepository(RestHighLevelClient client) {
super(client);
}
@Override
public Dataset createOrUpdate(Dataset entity) throws IOException {
XContentBuilder builder = XContentFactory.jsonBuilder();
IndexRequest request = new IndexRequest("datasets", "doc", entity.getId()).source(entity.toElasticEntity(builder));
this.getClient().index(request);
return entity;
}
@Override
public Dataset findDocument(String id) throws IOException {
GetRequest request = new GetRequest("datasets","doc",id);
GetResponse response = this.getClient().get(request);
return new Dataset().fromElasticEntity(response.getSourceAsMap());
}
@Override
public List<Dataset> query(DatasetCriteria criteria) throws ExecutionException, InterruptedException, IOException {
SearchRequest searchRequest = new SearchRequest();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery()
.should(QueryBuilders.termsQuery("tags.name.keyword", criteria.getTags().stream().map(x -> x.getName()).collect(Collectors.toList())));
searchSourceBuilder.query(boolQuery);
searchRequest.source(searchSourceBuilder);
SearchResponse response = this.getClient().search(searchRequest);
return Arrays.stream(response.getHits().getHits()).map(x -> this.transformFromString(x.getSourceAsString(), Dataset.class)).collect(Collectors.toList());
}
@Override
public boolean exists() throws IOException {
GetIndexRequest request = new GetIndexRequest();
request.indices("datasets");
return this.getClient().indices().exists(request);
}
}

@ -0,0 +1,36 @@
package eu.eudat.elastic.repository;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.eudat.elastic.criteria.Criteria;
import eu.eudat.elastic.entities.ElasticEntity;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import java.io.IOException;
/**
* Created by ikalyvas on 7/5/2018.
*/
public abstract class ElasticRepository<T extends ElasticEntity,C extends Criteria> implements Repository<T,C> {
private RestHighLevelClient client;
public RestHighLevelClient getClient() {
return client;
}
public ElasticRepository(RestHighLevelClient client) {
this.client = client;
}
public <T> T transformFromString(String value, Class<T> tClass) {
ObjectMapper mapper = new ObjectMapper();
T item = null;
try {
item = mapper.readValue(value, tClass);
} catch (IOException e) {
e.printStackTrace();
}
return item;
}
}

@ -0,0 +1,22 @@
package eu.eudat.elastic.repository;
import eu.eudat.elastic.criteria.Criteria;
import eu.eudat.elastic.entities.ElasticEntity;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.ExecutionException;
/**
* Created by ikalyvas on 7/5/2018.
*/
public interface Repository<ET extends ElasticEntity, C extends Criteria> {
ET createOrUpdate(ET entity) throws IOException;
ET findDocument(String id) throws IOException;
List<ET> query(C criteria) throws ExecutionException, InterruptedException, IOException;
boolean exists() throws IOException;
}

@ -18,7 +18,8 @@
<module>web</module>
<module>data</module>
<module>logging</module>
<module>queryengine</module>
<module>elastic</module>
</modules>
<properties>
@ -65,6 +66,19 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
</dependency>
<!-- https://mvnrepository.com/artifact/org.elasticsearch/elasticsearch -->
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>6.3.1</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>6.3.1</version>
</dependency>
<dependency>
<groupId>org.hibernate</groupId>
<artifactId>hibernate-core</artifactId>
@ -172,6 +186,11 @@
<artifactId>hibernate-jpamodelgen</artifactId>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>transport</artifactId>
<version>6.3.0</version>
</dependency>
</dependencies>
@ -188,7 +207,7 @@
<profile>
<id>production</id>
<properties>
<packaging.type>war</packaging.type>
<packaging.type>jar</packaging.type>
</properties>
<dependencies>
<dependency>

@ -0,0 +1,22 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>dmp-backend</artifactId>
<groupId>eu.eudat</groupId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>query-engine</artifactId>
<dependencies>
<dependency>
<groupId>eu.eudat</groupId>
<artifactId>data</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
</dependencies>
</project>

@ -0,0 +1,14 @@
import eu.eudat.data.entities.DataRepository;
import eu.eudat.query.engine.builder.QueryBuilder;
import eu.eudat.query.engine.builder.QueryBuilderImpl;
/**
* Created by ikalyvas on 7/5/2018.
*/
public class Main {
public void Test(){
QueryBuilder<DataRepository> queryBuilder = new QueryBuilderImpl<>();
queryBuilder.where((entity, comparisonExpression) -> comparisonExpression.field("id").greaterThan(5));
//queryBuilder.and(((entity, expression) -> expression.field("id").greaterThan(4)),((entity, expression) -> expression.field("id").greaterThan(5)));
}
}

@ -0,0 +1,16 @@
package eu.eudat.query.engine.builder;
import eu.eudat.query.engine.expressions.Expression;
import eu.eudat.query.engine.predicates.AndPredicate;
import eu.eudat.query.engine.predicates.ComparisonPredicate;
/**
* Created by ikalyvas on 7/5/2018.
*/
public interface QueryBuilder<T> {
Expression where(ComparisonPredicate<T> wherePredicate);
Expression and(AndPredicate<T> andPredicate);
}

@ -0,0 +1,22 @@
package eu.eudat.query.engine.builder;
import eu.eudat.query.engine.expressions.Expression;
import eu.eudat.query.engine.predicates.AndPredicate;
import eu.eudat.query.engine.predicates.ComparisonPredicate;
/**
* Created by ikalyvas on 7/5/2018.
*/
public class QueryBuilderImpl<T> implements QueryBuilder<T> {
@Override
public Expression where(ComparisonPredicate wherePredicate) {
return null;
}
@Override
public Expression and(AndPredicate<T> andPredicate) {
return null;
}
}

@ -0,0 +1,17 @@
package eu.eudat.query.engine.expressions;
/**
* Created by ikalyvas on 7/5/2018.
*/
public abstract class AbstractFieldExpression<T extends AbstractFieldExpression<T>> {
private String field;
protected String getField() {
return field;
}
public T field(String field) {
this.field = field;
return (T)this;
}
}

@ -0,0 +1,43 @@
package eu.eudat.query.engine.expressions;
import eu.eudat.query.engine.types.expression.comparison.ComparisonExpressionType;
/**
* Created by ikalyvas on 7/5/2018.
*/
public class ComparisonExpression extends AbstractFieldExpression<ComparisonExpression> implements Expression {
private ComparisonExpressionType type;
private Object value;
public <T> Expression greaterThan(T value){
this.value = value;
this.type = ComparisonExpressionType.GREATER_THAN;
return this;
}
public <T> Expression greaterThanOrEqual(T value){
this.value = value;
this.type = ComparisonExpressionType.GREATER_OR_EQUAL_THAN;
return this;
}
public <T> Expression equal(T value){
this.value = value;
this.type = ComparisonExpressionType.EQUAL;
return this;
}
public <T> Expression lessThan(T value){
this.value = value;
this.type = ComparisonExpressionType.LESS_THAN;
return this;
}
public <T> Expression lessOrEqualThan(T value){
this.value = value;
this.type = ComparisonExpressionType.LESS_OR_EQUAL_THAN;
return this;
}
}

@ -0,0 +1,7 @@
package eu.eudat.query.engine.expressions;
/**
* Created by ikalyvas on 7/5/2018.
*/
public interface Expression {
}

@ -0,0 +1,11 @@
package eu.eudat.query.engine.predicates;
import eu.eudat.query.engine.expressions.ComparisonExpression;
import eu.eudat.query.engine.expressions.Expression;
/**
* Created by ikalyvas on 7/5/2018.
*/
public interface AndPredicate<T> {
<E extends Expression> Expression and(ComparisonPredicate<T>... predicates);
}

@ -0,0 +1,9 @@
package eu.eudat.query.engine.predicates;
import eu.eudat.query.engine.expressions.ComparisonExpression;
/**
* Created by ikalyvas on 7/5/2018.
*/
public interface ComparisonPredicate<T> extends Predicate<T, ComparisonExpression> {
}

@ -0,0 +1,11 @@
package eu.eudat.query.engine.predicates;
import eu.eudat.query.engine.expressions.Expression;
/**
* Created by ikalyvas on 7/5/2018.
*/
public interface Predicate<T,E extends Expression> {
Expression where(T entity, E expression);
}

@ -0,0 +1,8 @@
package eu.eudat.query.engine.types.expression.comparison;
/**
* Created by ikalyvas on 7/5/2018.
*/
public enum ComparisonExpressionType {
EQUAL, GREATER_THAN, GREATER_OR_EQUAL_THAN, LESS_THAN, LESS_OR_EQUAL_THAN
}

@ -31,6 +31,15 @@
<artifactId>queryable</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>eu.eudat</groupId>
<artifactId>elastic</artifactId>
<version>1.0.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>

@ -32,6 +32,7 @@ public class ResponsesCache {
caches.add(new GuavaCache("organisations", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("registries", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("services", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("tags", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("researchers", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
simpleCacheManager.setCaches(caches);
System.out.println("OK");

@ -0,0 +1,39 @@
package eu.eudat.configurations;
import org.apache.http.HttpHost;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.env.Environment;
import java.net.InetAddress;
/**
* Created by ikalyvas on 7/5/2018.
*/
@Configuration
public class ElasticSearchConfiguration {
private Environment environment;
@Autowired
public ElasticSearchConfiguration(Environment environment) {
this.environment = environment;
}
@Bean
public RestHighLevelClient client() throws Exception {
RestHighLevelClient client = new RestHighLevelClient(
RestClient.builder(
new HttpHost(this.environment.getProperty("elasticsearch.host"),
Integer.parseInt(this.environment.getProperty("elasticsearch.port")), "http")));
return client;
}
}

@ -14,6 +14,7 @@ import javax.xml.bind.Unmarshaller;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.nio.file.Paths;
import java.util.LinkedList;
import java.util.List;
@ -47,7 +48,7 @@ public class DynamicProjectConfigurationProdImpl implements DynamicProjectConfig
JAXBContext jaxbContext = JAXBContext.newInstance(Configuration.class);
Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
is = new URL("file:///" + System.getenv("CATALINA_HOME") + fileUrl).openStream();
is = new URL(Paths.get(fileUrl).toUri().toURL().toString()).openStream();
this.configuration = (Configuration) jaxbUnmarshaller.unmarshal(is);
} catch (Exception ex) {
ex.printStackTrace();

@ -74,7 +74,8 @@ public class DatasetWizardController extends BaseController {
public @ResponseBody
ResponseEntity<ResponseItem<DatasetWizardModel>> getSingle(@PathVariable String id, Principal principal) {
try {
DatasetWizardModel dataset = new DatasetManager().getSingle(this.getApiContext().getOperationsContext().getDatabaseRepository().getDatasetDao(), id);
DatasetWizardModel dataset = new DatasetManager().getSingle(this.getApiContext().getOperationsContext().getDatabaseRepository().getDatasetDao(),this.getApiContext().getOperationsContext().getDatasetRepository()
,id);
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<DatasetWizardModel>().status(ApiMessageCode.NO_MESSAGE).payload(dataset));
} catch (Exception ex) {
ex.printStackTrace();

@ -0,0 +1,67 @@
package eu.eudat.controllers;
import eu.eudat.elastic.criteria.TagCriteria;
import eu.eudat.elastic.entities.Dataset;
import eu.eudat.elastic.entities.Tag;
import eu.eudat.elastic.repository.Repository;
import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
import eu.eudat.logic.services.ApiContext;
import eu.eudat.models.data.external.TagExternalSourcesModel;
import eu.eudat.models.data.helpers.responses.ResponseItem;
import eu.eudat.types.ApiMessageCode;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.*;
import java.util.List;
import java.util.Map;
/**
* Created by ikalyvas on 7/5/2018.
*/
@RestController
@CrossOrigin
@RequestMapping(value = {"/api"})
public class TagController extends BaseController{
private Repository<Dataset,TagCriteria> datasetRepository;
@Autowired
public TagController(ApiContext apiContext, Repository tagRepository) {
super(apiContext);
this.datasetRepository = tagRepository;
}
@Transactional
@RequestMapping(method = RequestMethod.POST, value = {"/tag/create"}, consumes = "application/json", produces = "application/json")
public @ResponseBody
ResponseEntity<ResponseItem<Dataset>> create(@RequestBody Dataset dataset) {
try {
Dataset tagEntity = this.datasetRepository.createOrUpdate(dataset);
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<Dataset>().payload(dataset).status(ApiMessageCode.SUCCESS_MESSAGE));
} catch (Exception e) {
e.printStackTrace();
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(new ResponseItem<Dataset>().status(ApiMessageCode.SUCCESS_MESSAGE).message(e.getMessage()));
}
}
@RequestMapping(method = RequestMethod.GET, value = {"/external/tags"}, produces = "application/json")
public @ResponseBody
ResponseEntity<ResponseItem<TagExternalSourcesModel>> listExternalTagModel(
@RequestParam(value = "query", required = false) String query,@RequestParam(value = "type", required = false) String type) {
try {
List<Map<String, String>> remoteRepos = this.getApiContext().getOperationsContext().getRemoteFetcher().getTags(query,type);
TagExternalSourcesModel researchersExternalSourcesModel = new TagExternalSourcesModel().fromExternalItem(remoteRepos);
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<TagExternalSourcesModel>().payload(researchersExternalSourcesModel).status(ApiMessageCode.NO_MESSAGE));
} catch (NoURLFound ex) {
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(new ResponseItem<TagExternalSourcesModel>().status(ApiMessageCode.ERROR_MESSAGE).message("External Url Not Found"));
} catch (HugeResultSet ex) {
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(new ResponseItem<TagExternalSourcesModel>().status(ApiMessageCode.ERROR_MESSAGE).message("Huge Result Set"));
} catch (Exception ex) {
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(new ResponseItem<TagExternalSourcesModel>().status(ApiMessageCode.ERROR_MESSAGE).message(ex.getMessage()));
}
}
}

@ -20,6 +20,8 @@ public class CommonsManager {
.map(item-> new ExternalSourcesConfiguration.ExternalSourcesUrlModel(item.getKey(),item.getLabel())).collect(Collectors.toList()));
externalSourcesConfiguration.setServices(configLoader.getExternalUrls().getServices().getUrls().stream()
.map(item-> new ExternalSourcesConfiguration.ExternalSourcesUrlModel(item.getKey(),item.getLabel())).collect(Collectors.toList()));
externalSourcesConfiguration.setTags(configLoader.getExternalUrls().getTags().getUrls().stream()
.map(item-> new ExternalSourcesConfiguration.ExternalSourcesUrlModel(item.getKey(),item.getLabel())).collect(Collectors.toList()));
return externalSourcesConfiguration;
}
}

@ -1,25 +1,27 @@
package eu.eudat.logic.managers;
import eu.eudat.logic.builders.entity.UserInfoBuilder;
import eu.eudat.data.dao.criteria.DataRepositoryCriteria;
import eu.eudat.data.dao.criteria.ExternalDatasetCriteria;
import eu.eudat.data.dao.criteria.RegistryCriteria;
import eu.eudat.data.dao.criteria.ServiceCriteria;
import eu.eudat.data.dao.entities.*;
import eu.eudat.data.entities.*;
import eu.eudat.data.query.items.table.dataset.DatasetTableRequest;
import eu.eudat.elastic.criteria.DatasetCriteria;
import eu.eudat.elastic.repository.DatasetRepository;
import eu.eudat.logic.builders.entity.UserInfoBuilder;
import eu.eudat.logic.services.ApiContext;
import eu.eudat.logic.services.forms.VisibilityRuleService;
import eu.eudat.logic.utilities.documents.helpers.FileEnvelope;
import eu.eudat.logic.utilities.documents.word.WordBuilder;
import eu.eudat.logic.utilities.documents.xml.ExportXmlBuilder;
import eu.eudat.models.HintedModelFactory;
import eu.eudat.data.dao.criteria.DataRepositoryCriteria;
import eu.eudat.data.dao.criteria.ExternalDatasetCriteria;
import eu.eudat.data.dao.criteria.RegistryCriteria;
import eu.eudat.data.dao.criteria.ServiceCriteria;
import eu.eudat.data.query.items.table.dataset.DatasetTableRequest;
import eu.eudat.models.data.datasetwizard.DatasetWizardModel;
import eu.eudat.models.data.helpers.common.DataTableData;
import eu.eudat.models.data.listingmodels.DatasetListingModel;
import eu.eudat.models.data.security.Principal;
import eu.eudat.models.data.user.composite.PagedDatasetProfile;
import eu.eudat.queryable.QueryableList;
import eu.eudat.logic.services.ApiContext;
import eu.eudat.logic.services.forms.VisibilityRuleService;
import org.apache.commons.io.IOUtils;
import org.json.JSONObject;
import org.springframework.core.env.Environment;
@ -35,6 +37,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
@ -42,8 +45,19 @@ import java.util.zip.ZipInputStream;
public class DatasetManager {
public DataTableData<DatasetListingModel> getPaged(ApiContext apiContext, DatasetTableRequest datasetTableRequest, Principal principal) throws Exception {
DatasetCriteria datasetCriteria = new DatasetCriteria();
datasetCriteria.setTags(datasetTableRequest.getCriteria().getTags());
List<eu.eudat.elastic.entities.Dataset> datasets = apiContext.getOperationsContext().getDatasetRepository().exists() ?
apiContext.getOperationsContext().getDatasetRepository().query(datasetCriteria) : new LinkedList<>();
UserInfo userInfo = apiContext.getOperationsContext().getBuilderFactory().getBuilder(UserInfoBuilder.class).id(principal.getId()).build();
QueryableList<eu.eudat.data.entities.Dataset> items = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));
if (datasetTableRequest.getCriteria().getTags() != null && !datasetTableRequest.getCriteria().getTags().isEmpty()) {
if (!datasets.isEmpty())
items.where((builder, root) -> root.get("id").in(datasets.stream().map(x -> UUID.fromString(x.getId())).collect(Collectors.toList())));
else
items.where((builder, root) -> root.get("id").in(new UUID[]{UUID.randomUUID()}));
}
QueryableList<eu.eudat.data.entities.Dataset> authItems = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getAuthenticated(items, userInfo);
QueryableList<eu.eudat.data.entities.Dataset> pagedItems = PaginationManager.applyPaging(authItems, datasetTableRequest);
DataTableData<DatasetListingModel> dataTable = new DataTableData<DatasetListingModel>();
@ -62,11 +76,14 @@ public class DatasetManager {
return dataTable;
}
public DatasetWizardModel getSingle(DatasetDao datatasetRepository, String id) throws InstantiationException, IllegalAccessException {
public DatasetWizardModel getSingle(DatasetDao datatasetRepository, DatasetRepository elasticDatasetRepository, String id) throws InstantiationException, IllegalAccessException, IOException {
DatasetWizardModel dataset = new DatasetWizardModel();
eu.eudat.data.entities.Dataset datasetEntity = datatasetRepository.find(UUID.fromString(id), HintedModelFactory.getHint(DatasetWizardModel.class));
eu.eudat.elastic.entities.Dataset datasetElastic = elasticDatasetRepository.exists() ?
elasticDatasetRepository.findDocument(id) : new eu.eudat.elastic.entities.Dataset();
dataset.setDatasetProfileDefinition(getPagedProfile(dataset, datasetEntity));
dataset.fromDataModel(datasetEntity);
dataset.setTags(datasetElastic.getTags());
return dataset;
}
@ -75,7 +92,7 @@ public class DatasetManager {
datasetprofile.setStatus(dataset.getStatus());
if (datasetEntity.getProperties() != null) {
JSONObject jobject = new JSONObject(datasetEntity.getProperties());
Map<String, Object> properties = (Map<String, Object>) jobject.toMap();
Map<String, Object> properties = jobject.toMap();
datasetprofile.fromJsonObject(properties);
}
PagedDatasetProfile pagedDatasetProfile = new PagedDatasetProfile();
@ -180,15 +197,16 @@ public class DatasetManager {
return newFile;
}
public static eu.eudat.data.entities.Dataset createOrUpdate(ApiContext apiContext, DatasetWizardModel profile, Principal principal) throws Exception {
eu.eudat.data.entities.Dataset dataset = profile.toDataModel();
propertiesModelToString(profile, dataset);
public static eu.eudat.data.entities.Dataset createOrUpdate(ApiContext apiContext, DatasetWizardModel datasetWizardModel, Principal principal) throws Exception {
eu.eudat.data.entities.Dataset dataset = datasetWizardModel.toDataModel();
propertiesModelToString(datasetWizardModel, dataset);
UserInfo userInfo = apiContext.getOperationsContext().getBuilderFactory().getBuilder(UserInfoBuilder.class).id(principal.getId()).build();
dataset.setCreator(userInfo);
updateTags(apiContext.getOperationsContext().getDatasetRepository(), datasetWizardModel);
createRegistriesIfTheyDontExist(apiContext.getOperationsContext().getDatabaseRepository().getRegistryDao(), dataset);
createDataRepositoriesIfTheyDontExist(apiContext.getOperationsContext().getDatabaseRepository().getDataRepositoryDao(), dataset);
createServicesIfTheyDontExist(apiContext.getOperationsContext().getDatabaseRepository().getDatasetServiceDao(),apiContext.getOperationsContext().getDatabaseRepository().getServiceDao(), dataset);
createExternalDatasetsIfTheyDontExist(apiContext.getOperationsContext().getDatabaseRepository().getDatasetExternalDatasetDao(),apiContext.getOperationsContext().getDatabaseRepository().getExternalDatasetDao(), dataset);
createServicesIfTheyDontExist(apiContext.getOperationsContext().getDatabaseRepository().getDatasetServiceDao(), apiContext.getOperationsContext().getDatabaseRepository().getServiceDao(), dataset);
createExternalDatasetsIfTheyDontExist(apiContext.getOperationsContext().getDatabaseRepository().getDatasetExternalDatasetDao(), apiContext.getOperationsContext().getDatabaseRepository().getExternalDatasetDao(), dataset);
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().createOrUpdate(dataset);
}
@ -200,6 +218,15 @@ public class DatasetManager {
dataset.setProperties(jobject.toString());
}
private static void updateTags(DatasetRepository datasetRepository, DatasetWizardModel datasetWizardModel) throws IOException {
if (datasetWizardModel.getTags() != null && !datasetWizardModel.getTags().isEmpty()) {
eu.eudat.elastic.entities.Dataset dataset = new eu.eudat.elastic.entities.Dataset();
dataset.setId(datasetWizardModel.getId().toString());
dataset.setTags(datasetWizardModel.getTags());
datasetRepository.createOrUpdate(dataset);
}
}
private static void createRegistriesIfTheyDontExist(RegistryDao registryDao, eu.eudat.data.entities.Dataset dataset) {
if (dataset.getRegistries() != null && !dataset.getRegistries().isEmpty()) {
@ -226,8 +253,7 @@ public class DatasetManager {
datasetDataRepository.getDataRepository().setId(entries.get(0).getId());
datasetDataRepository.setDataset(dataset);
dataset.getDatasetDataRepositories().add(datasetDataRepository);
}
else {
} else {
DataRepository dataRepository = dataRepositoryDao.createOrUpdate(datasetDataRepository.getDataRepository());
datasetDataRepository.setDataRepository(dataRepository);
dataset.getDatasetDataRepositories().add(datasetDataRepository);
@ -236,7 +262,7 @@ public class DatasetManager {
}
}
private static void createServicesIfTheyDontExist(DatasetServiceDao datasetServiceDao,ServiceDao serviceDao, eu.eudat.data.entities.Dataset dataset) {
private static void createServicesIfTheyDontExist(DatasetServiceDao datasetServiceDao, ServiceDao serviceDao, eu.eudat.data.entities.Dataset dataset) {
Set<DatasetService> services = dataset.getServices();
dataset.setServices(new HashSet<>());
if (services != null && !services.isEmpty()) {
@ -244,21 +270,20 @@ public class DatasetManager {
ServiceCriteria criteria = new ServiceCriteria();
criteria.setLike(datasetService.getService().getLabel());
List<eu.eudat.data.entities.Service> entries = serviceDao.getWithCriteria(criteria).toList();
if (entries != null && !entries.isEmpty()){
if (entries != null && !entries.isEmpty()) {
datasetService.getService().setId(entries.get(0).getId());
datasetService.setDataset(dataset);
dataset.getServices().add(datasetService);
}
else {
} else {
Service service = serviceDao.createOrUpdate(datasetService.getService());
datasetService.setService(service );
datasetService.setService(service);
dataset.getServices().add(datasetService);
}
}
}
}
private static void createExternalDatasetsIfTheyDontExist(DatasetExternalDatasetDao datasetExternalDatasetDao,ExternalDatasetDao externalDatasetDao, eu.eudat.data.entities.Dataset dataset) {
private static void createExternalDatasetsIfTheyDontExist(DatasetExternalDatasetDao datasetExternalDatasetDao, ExternalDatasetDao externalDatasetDao, eu.eudat.data.entities.Dataset dataset) {
Set<DatasetExternalDataset> externalDatasets = dataset.getDatasetExternalDatasets();
dataset.setDatasetExternalDatasets(new HashSet<>());
if (externalDatasets != null && !externalDatasets.isEmpty()) {
@ -270,8 +295,7 @@ public class DatasetManager {
datasetExternalDataset.getExternalDataset().setId(entries.get(0).getId());
datasetExternalDataset.setDataset(dataset);
dataset.getDatasetExternalDatasets().add(datasetExternalDataset);
}
else {
} else {
ExternalDataset externalDataset = externalDatasetDao.createOrUpdate(datasetExternalDataset.getExternalDataset());
datasetExternalDataset.setExternalDataset(externalDataset);
dataset.getDatasetExternalDatasets().add(datasetExternalDataset);

@ -23,6 +23,7 @@ public class ExternalUrls implements Serializable {
ResearcherUrls researchers;
OrganisationUrls organisations;
DatasetUrls datasets;
TagUrls tags;
public RegistryUrls getRegistries() {
@ -59,6 +60,15 @@ public class ExternalUrls implements Serializable {
return researchers;
}
public TagUrls getTags() {
return tags;
}
@XmlElement(name = "tags")
public void setTags(TagUrls tags) {
this.tags = tags;
}
@XmlElement(name = "researchers")
public void setResearchers(ResearcherUrls researchers) {
this.researchers = researchers;

@ -11,6 +11,7 @@ import javax.xml.bind.Unmarshaller;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.nio.file.Paths;
/**
* Created by ikalyvas on 2/9/2018.
@ -34,7 +35,7 @@ public class ProductionConfigLoader implements ConfigLoader {
JAXBContext jaxbContext = JAXBContext.newInstance(ExternalUrls.class);
Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
is = new URL("file:///" + System.getenv("CATALINA_HOME") + fileUrl).openStream();
is = new URL(Paths.get(fileUrl).toUri().toURL().toString()).openStream();
externalUrls = (ExternalUrls) jaxbUnmarshaller.unmarshal(is);
} catch (Exception ex) {

@ -0,0 +1,35 @@
package eu.eudat.logic.proxy.config.entities;
import eu.eudat.logic.proxy.config.FetchStrategy;
import eu.eudat.logic.proxy.config.UrlConfiguration;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
/**
* Created by ikalyvas on 7/9/2018.
*/
public class TagUrls {
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

@ -85,6 +85,15 @@ public class RemoteFetcher {
return getAll(urlConfigs, fetchStrategy, query);
}
@Cacheable("tags")
public List<Map<String, String>> getTags(String query, String key) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs =
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getTags().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
: configLoader.getExternalUrls().getTags().getUrls();
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getTags().getFetchMode();
return getAll(urlConfigs, fetchStrategy, query);
}
@Cacheable("datasets")
public List<Map<String, String>> getDatasets(String query, String key) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs =

@ -3,6 +3,7 @@ package eu.eudat.logic.services;
import eu.eudat.logic.services.helpers.HelpersService;
import eu.eudat.logic.services.operations.OperationsContext;
import eu.eudat.logic.services.utilities.UtilitiesService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@ -13,6 +14,7 @@ public class ApiContextImpl implements ApiContext {
private HelpersService helpersService;
private UtilitiesService utilitiesService;
@Autowired
public ApiContextImpl(OperationsContext operationsContext, HelpersService helpersService, UtilitiesService utilitiesService) {
this.operationsContext = operationsContext;
this.helpersService = helpersService;

@ -1,5 +1,6 @@
package eu.eudat.logic.services.operations;
import eu.eudat.elastic.repository.DatasetRepository;
import eu.eudat.logic.builders.BuilderFactory;
import eu.eudat.logic.proxy.fetching.RemoteFetcher;
import eu.eudat.logic.services.helpers.FileStorageService;
@ -20,4 +21,5 @@ public interface OperationsContext {
FileStorageService getFileStorageService();
DatasetRepository getDatasetRepository();
}