refactoring
This commit is contained in:
parent
7ee950801f
commit
0c79c5e974
|
|
@ -0,0 +1,69 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
|
||||
<parent>
|
||||
<groupId>eu.dnetlib.docker</groupId>
|
||||
<artifactId>apps</artifactId>
|
||||
<version>7.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<artifactId>dnet-contexts</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<dependencies>
|
||||
|
||||
<dependency>
|
||||
<groupId>${project.groupId}</groupId>
|
||||
<artifactId>dnet-db-common</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- hot swapping, disable cache for template, enable live reload -->
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-devtools</artifactId>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
|
||||
<!-- Tests -->
|
||||
<dependency>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-core</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-junit-jupiter</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-help-plugin</artifactId>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
package eu.dnetlib.services.contexts;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
import eu.dnetlib.base.AbstractDnetApp;
|
||||
|
||||
@SpringBootApplication
|
||||
public class ContextsApplication extends AbstractDnetApp {
|
||||
|
||||
public static void main(final String[] args) {
|
||||
SpringApplication.run(ContextsApplication.class, args);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,12 +1,13 @@
|
|||
package eu.dnetlib.is.context;
|
||||
package eu.dnetlib.services.contexts.controller;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
|
||||
import eu.dnetlib.common.controller.AbstractDnetController;
|
||||
import eu.dnetlib.is.model.context.Context;
|
||||
import eu.dnetlib.base.AbstractDnetController;
|
||||
import eu.dnetlib.data.model.context.Context;
|
||||
import eu.dnetlib.services.contexts.service.ContextService;
|
||||
|
||||
public class AbstractContextController extends AbstractDnetController {
|
||||
|
||||
|
|
@ -1,14 +1,14 @@
|
|||
package eu.dnetlib.is.context;
|
||||
package eu.dnetlib.services.contexts.controller;
|
||||
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import eu.dnetlib.data.model.context.Category;
|
||||
import eu.dnetlib.data.model.context.Context;
|
||||
import eu.dnetlib.data.model.context.CtxChildInfo;
|
||||
import eu.dnetlib.errors.InformationServiceException;
|
||||
import eu.dnetlib.is.model.context.Category;
|
||||
import eu.dnetlib.is.model.context.Context;
|
||||
import eu.dnetlib.is.model.context.CtxChildInfo;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/ajax/contexts")
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package eu.dnetlib.is.context;
|
||||
package eu.dnetlib.services.contexts.controller;
|
||||
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
package eu.dnetlib.services.contexts.controller;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.dom4j.DocumentException;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import eu.dnetlib.base.AbstractDnetController;
|
||||
import eu.dnetlib.services.contexts.importer.ContextImporter;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/import")
|
||||
public class ImporterController extends AbstractDnetController {
|
||||
|
||||
// EXAMPLE:
|
||||
// find ./VocabularyDSResourceType/ -name "*.xml" -exec curl -X POST "http://localhost:8280/api/import/vocabulary" -H "accept: */*" -H
|
||||
// "Content-Type: text/plain" --data-binary @{} \;
|
||||
// find ./DedupConfigurationDSResources/ -name "*.xml" -exec curl -X POST "http://localhost:8280/api/import/resource" -H "accept: */*"
|
||||
// -H "Content-Type: text/plain" --data-binary @{} \;
|
||||
|
||||
@Autowired
|
||||
private ContextImporter contextImporter;
|
||||
|
||||
@PostMapping(value = "/context", consumes = {
|
||||
MediaType.TEXT_PLAIN_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.APPLICATION_JSON_VALUE
|
||||
})
|
||||
public List<String> importContext(final HttpServletRequest request) throws DocumentException, IOException {
|
||||
|
||||
final String xml = IOUtils.toString(request.getInputStream(), StandardCharsets.UTF_8);
|
||||
|
||||
contextImporter.loadFromOldProfile(xml);
|
||||
|
||||
return Arrays.asList("Done.");
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package eu.dnetlib.is.importer;
|
||||
package eu.dnetlib.services.contexts.importer;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
|
|
@ -11,19 +11,19 @@ import org.dom4j.Node;
|
|||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import eu.dnetlib.is.context.repository.CategoryRepository;
|
||||
import eu.dnetlib.is.context.repository.ConceptLevel0Repository;
|
||||
import eu.dnetlib.is.context.repository.ConceptLevel1Repository;
|
||||
import eu.dnetlib.is.context.repository.ConceptLevel2Repository;
|
||||
import eu.dnetlib.is.context.repository.ContextRepository;
|
||||
import eu.dnetlib.is.model.context.Category;
|
||||
import eu.dnetlib.is.model.context.ConceptLevel0;
|
||||
import eu.dnetlib.is.model.context.ConceptLevel1;
|
||||
import eu.dnetlib.is.model.context.ConceptLevel2;
|
||||
import eu.dnetlib.is.model.context.Context;
|
||||
import eu.dnetlib.is.model.context.CtxChildInfo;
|
||||
import eu.dnetlib.is.model.context.CtxInfo;
|
||||
import eu.dnetlib.is.model.context.Parameter;
|
||||
import eu.dnetlib.data.model.context.Category;
|
||||
import eu.dnetlib.data.model.context.ConceptLevel0;
|
||||
import eu.dnetlib.data.model.context.ConceptLevel1;
|
||||
import eu.dnetlib.data.model.context.ConceptLevel2;
|
||||
import eu.dnetlib.data.model.context.Context;
|
||||
import eu.dnetlib.data.model.context.CtxChildInfo;
|
||||
import eu.dnetlib.data.model.context.CtxInfo;
|
||||
import eu.dnetlib.data.model.context.Parameter;
|
||||
import eu.dnetlib.services.contexts.service.repository.CategoryRepository;
|
||||
import eu.dnetlib.services.contexts.service.repository.ConceptLevel0Repository;
|
||||
import eu.dnetlib.services.contexts.service.repository.ConceptLevel1Repository;
|
||||
import eu.dnetlib.services.contexts.service.repository.ConceptLevel2Repository;
|
||||
import eu.dnetlib.services.contexts.service.repository.ContextRepository;
|
||||
|
||||
@Service
|
||||
public class ContextImporter {
|
||||
|
|
@ -0,0 +1,134 @@
|
|||
package eu.dnetlib.services.contexts.service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ArrayNode;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
|
||||
import eu.dnetlib.data.model.context.Category;
|
||||
import eu.dnetlib.data.model.context.Context;
|
||||
import eu.dnetlib.data.model.context.CtxChildInfo;
|
||||
import eu.dnetlib.errors.InformationServiceException;
|
||||
import eu.dnetlib.services.contexts.service.repository.CategoryRepository;
|
||||
import eu.dnetlib.services.contexts.service.repository.ConceptLevel0Repository;
|
||||
import eu.dnetlib.services.contexts.service.repository.ConceptLevel1Repository;
|
||||
import eu.dnetlib.services.contexts.service.repository.ConceptLevel2Repository;
|
||||
import eu.dnetlib.services.contexts.service.repository.ContextRepository;
|
||||
|
||||
@Service
|
||||
public class ContextService {
|
||||
|
||||
@Autowired
|
||||
private ContextRepository contextRepository;
|
||||
@Autowired
|
||||
private CategoryRepository categoryRepository;
|
||||
@Autowired
|
||||
private ConceptLevel0Repository conceptLevel0Repository;
|
||||
@Autowired
|
||||
private ConceptLevel1Repository conceptLevel1Repository;
|
||||
@Autowired
|
||||
private ConceptLevel2Repository conceptLevel2Repository;
|
||||
|
||||
public List<Context> listContexts() {
|
||||
return contextRepository.findAll()
|
||||
.stream()
|
||||
.sorted((c1, c2) -> StringUtils.compareIgnoreCase(c1.getId(), c2.getId()))
|
||||
.map(c -> {
|
||||
c.setnChilds(categoryRepository.countByParent(c.getId()));
|
||||
return c;
|
||||
})
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public Context getContext(final String ctxId) throws InformationServiceException {
|
||||
final Context c = contextRepository.findById(ctxId).orElseThrow(() -> new InformationServiceException("Context Not found"));
|
||||
c.setnChilds(categoryRepository.countByParent(c.getId()));
|
||||
return c;
|
||||
}
|
||||
|
||||
public ObjectNode getContextFull(final String ctxId) throws InformationServiceException {
|
||||
final ObjectMapper mapper = new ObjectMapper();
|
||||
final Context ctx = getContext(ctxId);
|
||||
|
||||
final ObjectNode ctxNode = mapper.convertValue(ctx, ObjectNode.class);
|
||||
|
||||
final ArrayNode catArray = mapper.createArrayNode();
|
||||
|
||||
for (final Category cat : listCategories(ctxId)) {
|
||||
final ObjectNode catNode = mapper.convertValue(cat, ObjectNode.class);
|
||||
catArray.add(catNode);
|
||||
|
||||
final ArrayNode c0Array = mapper.createArrayNode();
|
||||
for (final CtxChildInfo c0 : listConcepts(0, cat.getId())) {
|
||||
final ObjectNode c0Node = mapper.convertValue(c0, ObjectNode.class);
|
||||
c0Array.add(c0Node);
|
||||
|
||||
final ArrayNode c1Array = mapper.createArrayNode();
|
||||
for (final CtxChildInfo c1 : listConcepts(1, c0.getId())) {
|
||||
final ObjectNode c1Node = mapper.convertValue(c1, ObjectNode.class);
|
||||
c1Array.add(c1Node);
|
||||
|
||||
final ArrayNode c2Array = mapper.createArrayNode();
|
||||
for (final CtxChildInfo c2 : listConcepts(2, c1.getId())) {
|
||||
final ObjectNode c2Node = mapper.convertValue(c2, ObjectNode.class);
|
||||
c2Array.add(c2Node);
|
||||
}
|
||||
c1Node.set("concepts", c2Array);
|
||||
}
|
||||
c0Node.set("concepts", c1Array);
|
||||
}
|
||||
catNode.set("concepts", c0Array);
|
||||
}
|
||||
ctxNode.set("categories", catArray);
|
||||
|
||||
return ctxNode;
|
||||
}
|
||||
|
||||
public List<Category> listCategories(final String parent) {
|
||||
final List<Category> list = new ArrayList<>();
|
||||
|
||||
categoryRepository.findByParentOrderById(parent).forEach(c -> {
|
||||
c.setnChilds(conceptLevel0Repository.countByParent(c.getId()));
|
||||
list.add(c);
|
||||
});
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
public List<? extends CtxChildInfo> listConcepts(final int level, final String parent) throws InformationServiceException {
|
||||
final List<CtxChildInfo> list = new ArrayList<>();
|
||||
|
||||
switch (level) {
|
||||
case 0:
|
||||
conceptLevel0Repository.findByParentOrderById(parent).forEach(c -> {
|
||||
c.setnChilds(conceptLevel1Repository.countByParent(c.getId()));
|
||||
list.add(c);
|
||||
});
|
||||
break;
|
||||
case 1:
|
||||
conceptLevel1Repository.findByParentOrderById(parent).forEach(c -> {
|
||||
c.setnChilds(conceptLevel2Repository.countByParent(c.getId()));
|
||||
list.add(c);
|
||||
});
|
||||
break;
|
||||
case 2:
|
||||
conceptLevel2Repository.findByParentOrderById(parent).forEach(c -> {
|
||||
c.setnChilds(0);
|
||||
list.add(c);
|
||||
});
|
||||
break;
|
||||
default:
|
||||
throw new InformationServiceException("Invalid concept level - valid levels are 0, 1, 2");
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
package eu.dnetlib.services.contexts.service.repository;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
|
||||
import eu.dnetlib.data.model.context.Category;
|
||||
|
||||
public interface CategoryRepository extends JpaRepository<Category, String> {
|
||||
|
||||
long countByParent(String parent);
|
||||
|
||||
Iterable<Category> findByParentOrderById(String parent);
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
package eu.dnetlib.services.contexts.service.repository;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
|
||||
import eu.dnetlib.data.model.context.ConceptLevel0;
|
||||
|
||||
public interface ConceptLevel0Repository extends JpaRepository<ConceptLevel0, String> {
|
||||
|
||||
long countByParent(String parent);
|
||||
|
||||
Iterable<ConceptLevel0> findByParentOrderById(String parent);
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
package eu.dnetlib.services.contexts.service.repository;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
|
||||
import eu.dnetlib.data.model.context.ConceptLevel1;
|
||||
|
||||
public interface ConceptLevel1Repository extends JpaRepository<ConceptLevel1, String> {
|
||||
|
||||
long countByParent(String parent);
|
||||
|
||||
Iterable<ConceptLevel1> findByParentOrderById(String parent);
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
package eu.dnetlib.services.contexts.service.repository;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
|
||||
import eu.dnetlib.data.model.context.ConceptLevel2;
|
||||
|
||||
public interface ConceptLevel2Repository extends JpaRepository<ConceptLevel2, String> {
|
||||
|
||||
long countByParent(String parent);
|
||||
|
||||
Iterable<ConceptLevel2> findByParentOrderById(String parent);
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
package eu.dnetlib.services.contexts.service.repository;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
|
||||
import eu.dnetlib.data.model.context.Context;
|
||||
|
||||
public interface ContextRepository extends JpaRepository<Context, String> {
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
|
||||
<parent>
|
||||
<groupId>eu.dnetlib.docker</groupId>
|
||||
<artifactId>apps</artifactId>
|
||||
<version>7.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<artifactId>dnet-info</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<dependencies>
|
||||
|
||||
<!-- hot swapping, disable cache for template, enable live reload -->
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-devtools</artifactId>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
|
||||
<!-- Tests -->
|
||||
<dependency>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-core</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-junit-jupiter</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-help-plugin</artifactId>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
package eu.dnetlib.services.info;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
import eu.dnetlib.base.AbstractDnetApp;
|
||||
|
||||
@SpringBootApplication
|
||||
public class InfoApplication extends AbstractDnetApp {
|
||||
|
||||
public static void main(final String[] args) {
|
||||
SpringApplication.run(InfoApplication.class, args);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package eu.dnetlib.is.info;
|
||||
package eu.dnetlib.services.info.controller;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.lang.management.ManagementFactory;
|
||||
|
|
@ -30,10 +30,11 @@ import org.springframework.web.bind.annotation.GetMapping;
|
|||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import eu.dnetlib.common.controller.AbstractDnetController;
|
||||
import eu.dnetlib.base.AbstractDnetController;
|
||||
import eu.dnetlib.params.KeyValue;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/ajax/info")
|
||||
@RequestMapping("/ajax")
|
||||
public class InfoAjaxController extends AbstractDnetController {
|
||||
|
||||
@Autowired
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package eu.dnetlib.is.info;
|
||||
package eu.dnetlib.services.info.controller;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package eu.dnetlib.is.info;
|
||||
package eu.dnetlib.services.info.controller;
|
||||
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.Set;
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
package eu.dnetlib.services.info.controller;
|
||||
|
||||
import org.springframework.stereotype.Controller;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
|
||||
@Controller
|
||||
public class SwaggerController {
|
||||
|
||||
@RequestMapping(value = {
|
||||
"/", "/docs", "swagger-ui.html", "swagger-ui/"
|
||||
})
|
||||
public String apiDoc() {
|
||||
return "redirect:swagger-ui/index.html";
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
package eu.dnetlib;
|
||||
|
||||
import org.springdoc.core.GroupedOpenApi;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.cache.annotation.EnableCaching;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.ComponentScan;
|
||||
import org.springframework.scheduling.annotation.EnableScheduling;
|
||||
|
||||
import eu.dnetlib.common.app.AbstractDnetApp;
|
||||
|
||||
@SpringBootApplication
|
||||
@EnableCaching
|
||||
@EnableScheduling
|
||||
@ComponentScan(basePackages = "eu.dnetlib")
|
||||
public class MainApplication extends AbstractDnetApp {
|
||||
|
||||
public static void main(final String[] args) {
|
||||
SpringApplication.run(MainApplication.class, args);
|
||||
}
|
||||
|
||||
@Bean
|
||||
public GroupedOpenApi publicApi() {
|
||||
return GroupedOpenApi.builder()
|
||||
.group("D-Net Information Service API")
|
||||
.pathsToMatch("/api/**")
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String swaggerTitle() {
|
||||
return "D-Net Information Service APIs";
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
package eu.dnetlib;
|
||||
|
||||
import javax.persistence.EntityManagerFactory;
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.boot.jdbc.DataSourceBuilder;
|
||||
import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.orm.jpa.JpaTransactionManager;
|
||||
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
import org.springframework.transaction.annotation.EnableTransactionManagement;
|
||||
|
||||
@Configuration
|
||||
@EnableTransactionManagement
|
||||
@EnableJpaRepositories(entityManagerFactoryRef = "mainEntityManagerFactory", transactionManagerRef = "mainTransactionManager", basePackages = {
|
||||
"eu.dnetlib.is",
|
||||
"eu.dnetlib.common",
|
||||
"eu.dnetlib.manager.history",
|
||||
"eu.dnetlib.manager.wf",
|
||||
"eu.dnetlib.data.mdstore"
|
||||
})
|
||||
public class MainDBConfig {
|
||||
|
||||
@Primary
|
||||
@Bean(name = "mainDataSource")
|
||||
@ConfigurationProperties(prefix = "is.datasource")
|
||||
public DataSource dataSource() {
|
||||
return DataSourceBuilder.create().build();
|
||||
}
|
||||
|
||||
@Primary
|
||||
@Bean(name = "mainEntityManagerFactory")
|
||||
public LocalContainerEntityManagerFactoryBean entityManagerFactory(
|
||||
final EntityManagerFactoryBuilder builder,
|
||||
@Qualifier("mainDataSource") final DataSource ds) {
|
||||
return builder
|
||||
.dataSource(ds)
|
||||
.packages("eu.dnetlib.is.model", "eu.dnetlib.manager.history.model", "eu.dnetlib.common.model", "eu.dnetlib.manager.wf.model", "eu.dnetlib.data.mdstore.model")
|
||||
.persistenceUnit("is")
|
||||
.build();
|
||||
}
|
||||
|
||||
@Primary
|
||||
@Bean(name = "mainTransactionManager")
|
||||
public PlatformTransactionManager transactionManager(
|
||||
@Qualifier("mainEntityManagerFactory") final EntityManagerFactory emf) {
|
||||
return new JpaTransactionManager(emf);
|
||||
}
|
||||
|
||||
@Primary
|
||||
@Bean(name = "mainJdbcTemplate")
|
||||
public JdbcTemplate jdbcTemplate1(@Qualifier("mainDataSource") final DataSource ds) {
|
||||
return new JdbcTemplate(ds);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
package eu.dnetlib;
|
||||
|
||||
import javax.persistence.EntityManagerFactory;
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.boot.jdbc.DataSourceBuilder;
|
||||
import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.orm.jpa.JpaTransactionManager;
|
||||
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
|
||||
import org.springframework.transaction.PlatformTransactionManager;
|
||||
import org.springframework.transaction.annotation.EnableTransactionManagement;
|
||||
|
||||
@Configuration
|
||||
@EnableTransactionManagement
|
||||
@EnableJpaRepositories(entityManagerFactoryRef = "openaireEntityManagerFactory", transactionManagerRef = "openaireTransactionManager", basePackages = {
|
||||
"eu.dnetlib.dsm"
|
||||
})
|
||||
public class OpenaireDBConfig {
|
||||
|
||||
@Bean(name = "openaireDataSource")
|
||||
@ConfigurationProperties(prefix = "openaire.datasource")
|
||||
public DataSource dataSource() {
|
||||
return DataSourceBuilder.create().build();
|
||||
}
|
||||
|
||||
@Bean(name = "openaireEntityManagerFactory")
|
||||
public LocalContainerEntityManagerFactoryBean openaireEntityManagerFactory(
|
||||
final EntityManagerFactoryBuilder builder,
|
||||
@Qualifier("openaireDataSource") final DataSource ds) {
|
||||
return builder
|
||||
.dataSource(ds)
|
||||
.packages("eu.dnetlib.dsm.model")
|
||||
.persistenceUnit("openaire")
|
||||
.build();
|
||||
}
|
||||
|
||||
@Bean(name = "openaireTransactionManager")
|
||||
public PlatformTransactionManager opeanaireTransactionManager(
|
||||
@Qualifier("openaireEntityManagerFactory") final EntityManagerFactory emf) {
|
||||
return new JpaTransactionManager(emf);
|
||||
}
|
||||
|
||||
@Bean(name = "openaireJdbcTemplate")
|
||||
public JdbcTemplate jdbcTemplate1(@Qualifier("openaireDataSource") final DataSource ds) {
|
||||
return new JdbcTemplate(ds);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
package eu.dnetlib.data.mdstore;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import eu.dnetlib.common.controller.AbstractDnetController;
|
||||
import eu.dnetlib.data.mdstore.hadoop.ZeppelinClient;
|
||||
import eu.dnetlib.errors.MDStoreManagerException;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/zeppelin")
|
||||
public class ZeppelinAjaxController extends AbstractDnetController {
|
||||
|
||||
@Autowired
|
||||
private ZeppelinClient zeppelinClient;
|
||||
|
||||
@GetMapping("/templates")
|
||||
public List<String> getTemplates() throws MDStoreManagerException {
|
||||
try {
|
||||
return zeppelinClient.listTemplates();
|
||||
} catch (final Throwable e) {
|
||||
throw new MDStoreManagerException("Zeppelin is unreachable", e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
package eu.dnetlib.data.mdstore;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Controller;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
|
||||
import eu.dnetlib.data.mdstore.hadoop.ZeppelinClient;
|
||||
import eu.dnetlib.data.mdstore.model.MDStoreType;
|
||||
import eu.dnetlib.data.mdstore.model.MDStoreVersion;
|
||||
import eu.dnetlib.data.mdstore.model.MDStoreWithInfo;
|
||||
import eu.dnetlib.errors.MDStoreManagerException;
|
||||
|
||||
@Controller
|
||||
public class ZeppelinController {
|
||||
|
||||
@Autowired
|
||||
private ZeppelinClient zeppelinClient;
|
||||
|
||||
@Autowired
|
||||
private MDStoreService service;
|
||||
|
||||
@RequestMapping("/zeppelin/{mdId}/{note}")
|
||||
public String goToZeppelin(@PathVariable final String mdId, final @PathVariable String note) throws MDStoreManagerException {
|
||||
final MDStoreWithInfo mdstore = service.findMdStore(mdId);
|
||||
if (mdstore.getType() == MDStoreType.HDFS) {
|
||||
final String currentVersion = mdstore.getCurrentVersion();
|
||||
final MDStoreVersion version = service.findVersion(currentVersion);
|
||||
if (version.getParams().containsKey("hdfs_path")) {
|
||||
final String path = version.getParams().get("hdfs_path") + "/store";
|
||||
return "redirect:" + zeppelinClient.zeppelinNote(note, mdstore, path);
|
||||
}
|
||||
}
|
||||
throw new MDStoreManagerException();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,80 +0,0 @@
|
|||
package eu.dnetlib.is.importer;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.dom4j.DocumentException;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import eu.dnetlib.common.controller.AbstractDnetController;
|
||||
import eu.dnetlib.is.model.vocabulary.Vocabulary;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/api/import")
|
||||
public class ImporterController extends AbstractDnetController {
|
||||
|
||||
// EXAMPLE:
|
||||
// find ./VocabularyDSResourceType/ -name "*.xml" -exec curl -X POST "http://localhost:8280/api/import/vocabulary" -H "accept: */*" -H
|
||||
// "Content-Type: text/plain" --data-binary @{} \;
|
||||
// find ./DedupConfigurationDSResources/ -name "*.xml" -exec curl -X POST "http://localhost:8280/api/import/resource" -H "accept: */*"
|
||||
// -H "Content-Type: text/plain" --data-binary @{} \;
|
||||
|
||||
@Autowired
|
||||
private ContextImporter contextImporter;
|
||||
|
||||
@Autowired
|
||||
private OldProfilesImporter oldProfilesImporter;
|
||||
|
||||
@Autowired
|
||||
private WfHistoryImporter wfHistoryImporter;
|
||||
|
||||
@PostMapping(value = "/context", consumes = {
|
||||
MediaType.TEXT_PLAIN_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.APPLICATION_JSON_VALUE
|
||||
})
|
||||
public List<String> importContext(final HttpServletRequest request) throws DocumentException, IOException {
|
||||
|
||||
final String xml = IOUtils.toString(request.getInputStream(), StandardCharsets.UTF_8);
|
||||
|
||||
contextImporter.loadFromOldProfile(xml);
|
||||
|
||||
return Arrays.asList("Done.");
|
||||
|
||||
}
|
||||
|
||||
@PostMapping(value = "/resource", consumes = {
|
||||
MediaType.TEXT_PLAIN_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.APPLICATION_JSON_VALUE
|
||||
})
|
||||
public String importResource(final HttpServletRequest request) throws Exception {
|
||||
final String xml = IOUtils.toString(request.getInputStream(), StandardCharsets.UTF_8);
|
||||
|
||||
return oldProfilesImporter.importSimpleResource(xml);
|
||||
}
|
||||
|
||||
@PostMapping(value = "/vocabulary", consumes = {
|
||||
MediaType.TEXT_PLAIN_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.APPLICATION_JSON_VALUE
|
||||
})
|
||||
public Vocabulary importVocabulary(final HttpServletRequest request) throws Exception, IOException {
|
||||
final String xml = IOUtils.toString(request.getInputStream(), StandardCharsets.UTF_8);
|
||||
return oldProfilesImporter.importVocabulary(xml);
|
||||
}
|
||||
|
||||
@GetMapping(value = "/wf_logs")
|
||||
public List<String> importWfLogs(@RequestParam final String path) throws Exception {
|
||||
// mongoexport -d dnet_logs -c wf_logs --jsonArray -o /tmp/mongodump.json
|
||||
|
||||
wfHistoryImporter.load(path);
|
||||
|
||||
return Arrays.asList("Done.");
|
||||
}
|
||||
}
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
package eu.dnetlib.is.info;
|
||||
|
||||
public class KeyValue<T> {
|
||||
|
||||
private final String k;
|
||||
private final T v;
|
||||
|
||||
public KeyValue(final String k, final T v) {
|
||||
this.k = k;
|
||||
this.v = v;
|
||||
}
|
||||
|
||||
public String getK() {
|
||||
return k;
|
||||
}
|
||||
|
||||
public T getV() {
|
||||
return v;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
package eu.dnetlib.is.resource;
|
||||
|
||||
|
||||
public class ResourceTypeRepository {
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
package eu.dnetlib.services.is;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
import eu.dnetlib.base.AbstractDnetApp;
|
||||
|
||||
@SpringBootApplication
|
||||
public class MainApplication extends AbstractDnetApp {
|
||||
|
||||
public static void main(final String[] args) {
|
||||
SpringApplication.run(MainApplication.class, args);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,21 +1,21 @@
|
|||
package eu.dnetlib.is.resource;
|
||||
package eu.dnetlib.services.is.controller;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
|
||||
import eu.dnetlib.common.controller.AbstractDnetController;
|
||||
import eu.dnetlib.base.AbstractDnetController;
|
||||
import eu.dnetlib.data.model.resource.SimpleResource;
|
||||
import eu.dnetlib.errors.InformationServiceException;
|
||||
import eu.dnetlib.is.model.resource.SimpleResource;
|
||||
import eu.dnetlib.services.is.service.SimpleResourceService;
|
||||
import eu.dnetlib.utils.XmlIndenter;
|
||||
import jakarta.servlet.http.HttpServletResponse;
|
||||
|
||||
public class AbstractResourceController extends AbstractDnetController {
|
||||
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
package eu.dnetlib.services.is.controller;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import eu.dnetlib.base.AbstractDnetController;
|
||||
import eu.dnetlib.services.is.importer.OldProfilesImporter;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/api/import")
|
||||
public class ImporterController extends AbstractDnetController {
|
||||
|
||||
// EXAMPLE:
|
||||
// find ./VocabularyDSResourceType/ -name "*.xml" -exec curl -X POST "http://localhost:8280/api/import/vocabulary" -H "accept: */*" -H
|
||||
// "Content-Type: text/plain" --data-binary @{} \;
|
||||
// find ./DedupConfigurationDSResources/ -name "*.xml" -exec curl -X POST "http://localhost:8280/api/import/resource" -H "accept: */*"
|
||||
// -H "Content-Type: text/plain" --data-binary @{} \;
|
||||
|
||||
@Autowired
|
||||
private OldProfilesImporter oldProfilesImporter;
|
||||
|
||||
@PostMapping(value = "/resource", consumes = {
|
||||
MediaType.TEXT_PLAIN_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.APPLICATION_JSON_VALUE
|
||||
})
|
||||
public String importResource(final HttpServletRequest request) throws Exception {
|
||||
final String xml = IOUtils.toString(request.getInputStream(), StandardCharsets.UTF_8);
|
||||
|
||||
return oldProfilesImporter.importSimpleResource(xml);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package eu.dnetlib;
|
||||
package eu.dnetlib.services.is.controller;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.MediaType;
|
||||
|
|
@ -7,6 +7,7 @@ import org.springframework.web.bind.annotation.PathVariable;
|
|||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import eu.dnetlib.data.model.resource.ResourceType;
|
||||
import eu.dnetlib.services.is.repository.ResourceTypeRepository;
|
||||
|
||||
@RestController
|
||||
public class MainAjaxController {
|
||||
|
|
@ -1,10 +1,8 @@
|
|||
package eu.dnetlib.is.resource;
|
||||
package eu.dnetlib.services.is.controller;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
|
|
@ -15,8 +13,9 @@ import org.springframework.web.bind.annotation.RequestMapping;
|
|||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import eu.dnetlib.data.model.resource.SimpleResource;
|
||||
import eu.dnetlib.errors.InformationServiceException;
|
||||
import eu.dnetlib.is.model.resource.SimpleResource;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/ajax/resources")
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package eu.dnetlib.is.resource;
|
||||
package eu.dnetlib.services.is.controller;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
|
@ -9,8 +9,8 @@ import org.springframework.web.bind.annotation.GetMapping;
|
|||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import eu.dnetlib.is.model.resource.ResourceType;
|
||||
import eu.dnetlib.is.resource.repository.ResourceTypeRepository;
|
||||
import eu.dnetlib.data.model.resource.ResourceType;
|
||||
import eu.dnetlib.services.is.repository.ResourceTypeRepository;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/api/resources")
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package eu.dnetlib;
|
||||
package eu.dnetlib.services.is.controller;
|
||||
|
||||
import org.springframework.stereotype.Controller;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
|
|
@ -1,9 +1,7 @@
|
|||
package eu.dnetlib.is.importer;
|
||||
package eu.dnetlib.services.is.importer;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
import javax.transaction.Transactional;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.dom4j.Attribute;
|
||||
import org.dom4j.Document;
|
||||
|
|
@ -14,16 +12,12 @@ import org.dom4j.Node;
|
|||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import eu.dnetlib.data.model.resource.SimpleResource;
|
||||
import eu.dnetlib.errors.InformationServiceException;
|
||||
import eu.dnetlib.is.model.resource.SimpleResource;
|
||||
import eu.dnetlib.is.model.vocabulary.Synonym;
|
||||
import eu.dnetlib.is.model.vocabulary.Vocabulary;
|
||||
import eu.dnetlib.is.model.vocabulary.VocabularyTerm;
|
||||
import eu.dnetlib.is.resource.ResourceValidator;
|
||||
import eu.dnetlib.is.resource.repository.SimpleResourceRepository;
|
||||
import eu.dnetlib.is.vocabulary.repository.VocabularyRepository;
|
||||
import eu.dnetlib.is.vocabulary.repository.VocabularyTermRepository;
|
||||
import eu.dnetlib.services.is.repository.SimpleResourceRepository;
|
||||
import eu.dnetlib.services.is.service.ResourceValidator;
|
||||
import eu.dnetlib.utils.XmlIndenter;
|
||||
import jakarta.transaction.Transactional;
|
||||
|
||||
@Service
|
||||
public class OldProfilesImporter {
|
||||
|
|
@ -31,12 +25,6 @@ public class OldProfilesImporter {
|
|||
@Autowired
|
||||
private SimpleResourceRepository simpleResourceRepository;
|
||||
|
||||
@Autowired
|
||||
private VocabularyRepository vocabularyRepository;
|
||||
|
||||
@Autowired
|
||||
private VocabularyTermRepository vocabularyTermRepository;
|
||||
|
||||
@Autowired
|
||||
private ResourceValidator resourceValidator;
|
||||
|
||||
|
|
@ -119,38 +107,4 @@ public class OldProfilesImporter {
|
|||
}
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public Vocabulary importVocabulary(final String xml) throws Exception {
|
||||
final Document doc = DocumentHelper.parseText(xml);
|
||||
|
||||
final Vocabulary voc = new Vocabulary();
|
||||
final String vocId = doc.valueOf("//VOCABULARY_NAME/@code");
|
||||
final String vocName = doc.valueOf("//VOCABULARY_NAME");
|
||||
final String vocDesc = doc.valueOf("//VOCABULARY_DESCRIPTION");
|
||||
|
||||
voc.setId(vocId);
|
||||
voc.setName(vocName);
|
||||
voc.setDescription(vocDesc);
|
||||
|
||||
vocabularyRepository.save(voc);
|
||||
|
||||
for (final Node n : doc.selectNodes("//TERM")) {
|
||||
final VocabularyTerm term = new VocabularyTerm();
|
||||
term.setVocabulary(vocId);
|
||||
term.setCode(n.valueOf("@code"));
|
||||
term.setName(n.valueOf("@english_name"));
|
||||
term.setEncoding(n.valueOf("@encoding"));
|
||||
term.setSynonyms(n.selectNodes(".//SYNONYM")
|
||||
.stream()
|
||||
.map(ns -> new Synonym(ns.valueOf("@term"), ns.valueOf("@encoding")))
|
||||
.sorted()
|
||||
.distinct()
|
||||
.toArray(Synonym[]::new));
|
||||
|
||||
vocabularyTermRepository.save(term);
|
||||
}
|
||||
|
||||
return voc;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
package eu.dnetlib.services.is.repository;
|
||||
|
||||
import eu.dnetlib.data.model.resource.ResourceType;
|
||||
import eu.dnetlib.utils.ReadOnlyRepository;
|
||||
|
||||
public interface ResourceTypeRepository extends ReadOnlyRepository<ResourceType, String> {
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
package eu.dnetlib.services.is.repository;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.Modifying;
|
||||
import org.springframework.data.jpa.repository.Query;
|
||||
|
||||
import eu.dnetlib.data.model.resource.SimpleResource;
|
||||
|
||||
public interface SimpleResourceRepository extends JpaRepository<SimpleResource, String> {
|
||||
|
||||
@Query(value = "select content from resources where id = ?1", nativeQuery = true)
|
||||
Optional<String> findContentById(String id);
|
||||
|
||||
@Query(value = "select content from resources where type = ?1 and name = ?2", nativeQuery = true)
|
||||
Optional<String> findContentByTypeAndName(String type, String name);
|
||||
|
||||
@Modifying
|
||||
@Query(value = "update resources set content = ?2 where id = ?1", nativeQuery = true)
|
||||
void setContentById(String id, String content);
|
||||
|
||||
List<SimpleResource> findByType(String type);
|
||||
|
||||
@Query(value = "select t.content_type from resources r join resource_types t on (r.type = t.id) where r.id = ?1", nativeQuery = true)
|
||||
Optional<String> findContentTypeById(String id);
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
package eu.dnetlib.services.is.service;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.StringReader;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import javax.xml.XMLConstants;
|
||||
import javax.xml.transform.stream.StreamSource;
|
||||
import javax.xml.validation.Schema;
|
||||
import javax.xml.validation.SchemaFactory;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.dom4j.DocumentException;
|
||||
import org.dom4j.DocumentHelper;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import eu.dnetlib.data.model.resource.ResourceType;
|
||||
import eu.dnetlib.errors.InformationServiceException;
|
||||
import eu.dnetlib.services.is.repository.ResourceTypeRepository;
|
||||
|
||||
@Component
|
||||
public class ResourceValidator {
|
||||
|
||||
private final Log log = LogFactory.getLog(ResourceValidator.class);
|
||||
|
||||
@Autowired
|
||||
private ResourceTypeRepository resourceTypeRepository;
|
||||
|
||||
public void validate(final String type, final String content) throws InformationServiceException {
|
||||
|
||||
final ResourceType rtype = resourceTypeRepository.findById(type).orElseThrow(() -> new InformationServiceException("Invalid type"));
|
||||
|
||||
if (rtype.getContentType().equals(MediaType.APPLICATION_XML_VALUE)) {
|
||||
validateXml(type, content);
|
||||
} else if (rtype.getContentType().equals(MediaType.APPLICATION_JSON_VALUE)) {
|
||||
validateJSON(content);
|
||||
}
|
||||
}
|
||||
|
||||
private void validateXml(final String type, final String content) throws InformationServiceException {
|
||||
final Schema schema = getXmlSchema(type);
|
||||
if (schema != null) {
|
||||
try {
|
||||
schema.newValidator().validate(new StreamSource(new StringReader(content)));
|
||||
} catch (final Exception e) {
|
||||
throw new InformationServiceException(e.getMessage(), e);
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
DocumentHelper.parseText(content);
|
||||
} catch (final DocumentException e) {
|
||||
throw new InformationServiceException(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Schema getXmlSchema(final String type) {
|
||||
try {
|
||||
final InputStream is = getClass().getResourceAsStream("/schemas/" + type + ".xsd");
|
||||
if (is == null) { return null; }
|
||||
final String schemaSource = IOUtils.toString(is, StandardCharsets.UTF_8.name());
|
||||
if (StringUtils.isBlank(schemaSource)) { return null; }
|
||||
return SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI).newSchema(new StreamSource(new StringReader(schemaSource)));
|
||||
} catch (final SAXException | IOException e) {
|
||||
log.fatal("cannot parse resource type schema", e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private void validateJSON(final String content) throws InformationServiceException {
|
||||
final ObjectMapper mapper = new ObjectMapper();
|
||||
try {
|
||||
mapper.readValue(content, Object.class);
|
||||
} catch (final Exception e) {
|
||||
throw new InformationServiceException(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,104 @@
|
|||
package eu.dnetlib.services.is.service;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
|
||||
import eu.dnetlib.data.model.resource.SimpleResource;
|
||||
import eu.dnetlib.errors.InformationServiceException;
|
||||
import eu.dnetlib.services.is.repository.SimpleResourceRepository;
|
||||
import jakarta.transaction.Transactional;
|
||||
|
||||
@Service
|
||||
public class SimpleResourceService {
|
||||
|
||||
@Autowired
|
||||
private SimpleResourceRepository simpleResourceRepository;
|
||||
|
||||
@Autowired
|
||||
private ResourceValidator resourceValidator;
|
||||
|
||||
private static final Log log = LogFactory.getLog(SimpleResourceService.class);
|
||||
|
||||
public SimpleResource getMetadata(final String id) throws InformationServiceException {
|
||||
return simpleResourceRepository.findById(id).orElseThrow(() -> new InformationServiceException("Id not found"));
|
||||
}
|
||||
|
||||
public String getContent(final String id) throws InformationServiceException {
|
||||
return simpleResourceRepository.findContentById(id).orElseThrow(() -> new InformationServiceException("Id not found"));
|
||||
}
|
||||
|
||||
public String getContent(final String type, final String name) throws InformationServiceException {
|
||||
return simpleResourceRepository.findContentByTypeAndName(type, name).orElseThrow(() -> new InformationServiceException("Id not found"));
|
||||
}
|
||||
|
||||
public String getContentType(final String id) throws InformationServiceException {
|
||||
return simpleResourceRepository.findContentTypeById(id).orElseThrow(() -> new InformationServiceException("Id not found"));
|
||||
}
|
||||
|
||||
public List<SimpleResource> listResources(final String type) {
|
||||
return simpleResourceRepository.findByType(type)
|
||||
.stream()
|
||||
.sorted((r1, r2) -> StringUtils.compareIgnoreCase(r1.getName(), r2.getName()))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public void deleteResource(@PathVariable final String id) {
|
||||
log.info("Deleting resource: " + id);
|
||||
simpleResourceRepository.deleteById(id);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public SimpleResource saveNewResource(final String name,
|
||||
final String type,
|
||||
final String subtype,
|
||||
final String description,
|
||||
final String content) throws InformationServiceException {
|
||||
|
||||
resourceValidator.validate(type, content);
|
||||
|
||||
final LocalDateTime now = LocalDateTime.now();
|
||||
|
||||
final SimpleResource res = new SimpleResource();
|
||||
res.setId(UUID.randomUUID().toString());
|
||||
res.setName(name);
|
||||
res.setType(type);
|
||||
res.setSubtype(subtype);
|
||||
res.setDescription(description);
|
||||
res.setCreationDate(now);
|
||||
res.setModificationDate(now);
|
||||
|
||||
simpleResourceRepository.save(res);
|
||||
simpleResourceRepository.setContentById(res.getId(), content);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void saveMetadata(final String id, final SimpleResource r) throws InformationServiceException {
|
||||
if (simpleResourceRepository.existsById(id)) {
|
||||
r.setId(id);
|
||||
simpleResourceRepository.save(r);
|
||||
} else {
|
||||
throw new InformationServiceException("Resource not found");
|
||||
}
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void saveContent(final String id, final String content) throws InformationServiceException {
|
||||
final SimpleResource res = simpleResourceRepository.findById(id).orElseThrow(() -> new InformationServiceException("Resource not found"));
|
||||
|
||||
resourceValidator.validate(res.getType(), content);
|
||||
|
||||
simpleResourceRepository.setContentById(id, content);
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
|
||||
<parent>
|
||||
<groupId>eu.dnetlib.docker</groupId>
|
||||
<artifactId>apps</artifactId>
|
||||
<version>7.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<artifactId>dnet-mdstores</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<dependencies>
|
||||
|
||||
<dependency>
|
||||
<groupId>${project.groupId}</groupId>
|
||||
<artifactId>dnet-db-common</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- hot swapping, disable cache for template, enable live reload -->
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-devtools</artifactId>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
|
||||
<!-- Tests -->
|
||||
<dependency>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-core</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-junit-jupiter</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-help-plugin</artifactId>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-maven-plugin</artifactId>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
package eu.dnetlib.services.mdstores;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
import eu.dnetlib.base.AbstractDnetApp;
|
||||
|
||||
@SpringBootApplication
|
||||
public class MdstoresApplication extends AbstractDnetApp {
|
||||
|
||||
public static void main(final String[] args) {
|
||||
SpringApplication.run(MdstoresApplication.class, args);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
package eu.dnetlib.services.mdstores.backends;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import eu.dnetlib.data.mdstore.model.MDStore;
|
||||
import eu.dnetlib.data.mdstore.model.MDStoreVersion;
|
||||
import eu.dnetlib.data.mdstore.model.records.MetadataRecord;
|
||||
import eu.dnetlib.errors.MDStoreManagerException;
|
||||
|
||||
@Service
|
||||
public class DefaultBackend implements MDStoreBackend {
|
||||
|
||||
@Override
|
||||
public void completeNewMDStore(final MDStore mdstore) {}
|
||||
|
||||
@Override
|
||||
public void completeNewMDStoreVersion(final MDStoreVersion version) {}
|
||||
|
||||
@Override
|
||||
public void delete(final MDStore mdstore) throws MDStoreManagerException {}
|
||||
|
||||
@Override
|
||||
public void delete(final MDStoreVersion version) throws MDStoreManagerException {}
|
||||
|
||||
@Override
|
||||
public List<MetadataRecord> listEntries(final MDStoreVersion version, final long limit) throws MDStoreManagerException {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Stream<MetadataRecord> streamEntries(final MDStoreVersion version) throws MDStoreManagerException {
|
||||
return Stream.empty();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> listInternalFiles(final MDStoreVersion version) throws MDStoreManagerException {
|
||||
return new LinkedHashSet<>();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> fixInconsistencies(final boolean delete) throws MDStoreManagerException {
|
||||
return new HashSet<>();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addRecord(final MetadataRecord record) {}
|
||||
|
||||
@Override
|
||||
public long countRecords(final String versionId) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
package eu.dnetlib.services.mdstores.backends;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import eu.dnetlib.data.mdstore.model.MDStore;
|
||||
import eu.dnetlib.data.mdstore.model.MDStoreVersion;
|
||||
import eu.dnetlib.data.mdstore.model.records.MetadataRecord;
|
||||
import eu.dnetlib.errors.MDStoreManagerException;
|
||||
|
||||
public interface MDStoreBackend {
|
||||
|
||||
void completeNewMDStore(MDStore mdstore);
|
||||
|
||||
void completeNewMDStoreVersion(MDStoreVersion version);
|
||||
|
||||
void delete(MDStore mdstore) throws MDStoreManagerException;
|
||||
|
||||
void delete(MDStoreVersion version) throws MDStoreManagerException;
|
||||
|
||||
List<MetadataRecord> listEntries(MDStoreVersion version, long limit) throws MDStoreManagerException;
|
||||
|
||||
Stream<MetadataRecord> streamEntries(MDStoreVersion version) throws MDStoreManagerException;
|
||||
|
||||
Set<String> listInternalFiles(MDStoreVersion version) throws MDStoreManagerException;
|
||||
|
||||
Set<String> fixInconsistencies(boolean delete) throws MDStoreManagerException;
|
||||
|
||||
void addRecord(MetadataRecord record);
|
||||
|
||||
long countRecords(String versionId);
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
package eu.dnetlib.services.mdstores.backends;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import eu.dnetlib.data.mdstore.model.MDStore;
|
||||
import eu.dnetlib.data.mdstore.model.MDStoreVersion;
|
||||
import eu.dnetlib.data.mdstore.model.records.MetadataRecord;
|
||||
import eu.dnetlib.data.mdstore.model.records.MetadataRecordImpl;
|
||||
import eu.dnetlib.data.mdstore.model.records.Provenance;
|
||||
import eu.dnetlib.errors.MDStoreManagerException;
|
||||
|
||||
@Service
|
||||
public class MockBackend implements MDStoreBackend {
|
||||
|
||||
private static final Provenance MOCK_PROVENANCE = new Provenance("mock________::MOCK_DS", "Mock Datasource", "mock________");
|
||||
|
||||
@Override
|
||||
public void completeNewMDStore(final MDStore mdstore) {
|
||||
mdstore.getParams().put("mockParam1", "test");
|
||||
mdstore.getParams().put("mockParam2", "abc");
|
||||
mdstore.getParams().put("mockParam3", 1234);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void completeNewMDStoreVersion(final MDStoreVersion version) {
|
||||
version.getParams().put("mockParam1", "v_test");
|
||||
version.getParams().put("mockParam2", "v_abc");
|
||||
version.getParams().put("mockParam3", 1234);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void delete(final MDStore mdstore) throws MDStoreManagerException {}
|
||||
|
||||
@Override
|
||||
public void delete(final MDStoreVersion version) throws MDStoreManagerException {}
|
||||
|
||||
@Override
|
||||
public List<MetadataRecord> listEntries(final MDStoreVersion version, final long limit) throws MDStoreManagerException {
|
||||
final List<MetadataRecord> list = new ArrayList<>();
|
||||
|
||||
for (int i = 0; i < limit; i++) {
|
||||
final MetadataRecord rec = new MetadataRecordImpl();
|
||||
rec.setOriginalId("mck-" + i);
|
||||
rec.setId("mock________::mck-" + i);
|
||||
rec.setBody("<RECORD>" + i + "</RECORD>");
|
||||
rec.setDateOfCollection(Instant.now().toEpochMilli());
|
||||
rec.setDateOfTransformation(Instant.now().toEpochMilli());
|
||||
rec.setEncoding("XML");
|
||||
rec.setProvenance(MOCK_PROVENANCE);
|
||||
list.add(rec);
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Stream<MetadataRecord> streamEntries(final MDStoreVersion version) throws MDStoreManagerException {
|
||||
return listEntries(version, 1000).stream();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> listInternalFiles(final MDStoreVersion version) throws MDStoreManagerException {
|
||||
return new LinkedHashSet<>(Arrays.asList("file1", "file2", "file3", "file4"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> fixInconsistencies(final boolean delete) throws MDStoreManagerException {
|
||||
return new LinkedHashSet<>(Arrays.asList("1", "2", "3", "4"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addRecord(final MetadataRecord record) {
|
||||
throw new RuntimeException("NOT_IMPLEMENTED");
|
||||
}
|
||||
|
||||
@Override
|
||||
public long countRecords(final String versionId) {
|
||||
throw new RuntimeException("NOT_IMPLEMENTED");
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package eu.dnetlib.data.mdstore;
|
||||
package eu.dnetlib.services.mdstores.controller;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.web.bind.annotation.DeleteMapping;
|
||||
|
|
@ -7,11 +7,12 @@ import org.springframework.web.bind.annotation.PathVariable;
|
|||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
|
||||
import eu.dnetlib.common.controller.AbstractDnetController;
|
||||
import eu.dnetlib.base.AbstractDnetController;
|
||||
import eu.dnetlib.data.mdstore.model.MDStoreType;
|
||||
import eu.dnetlib.data.mdstore.model.MDStoreVersion;
|
||||
import eu.dnetlib.data.mdstore.model.MDStoreWithInfo;
|
||||
import eu.dnetlib.errors.MDStoreManagerException;
|
||||
import eu.dnetlib.services.mdstores.service.MDStoreService;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
|
||||
public class AbstractMDStoreController extends AbstractDnetController {
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package eu.dnetlib.data.mdstore;
|
||||
package eu.dnetlib.services.mdstores.controller;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
|
|
@ -16,7 +16,7 @@ import io.swagger.v3.oas.annotations.Operation;
|
|||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/ajax/mdstores")
|
||||
@RequestMapping("/ajax")
|
||||
@Tag(name = "Metadata Stores")
|
||||
public class MDStoreAjaxController extends AbstractMDStoreController {
|
||||
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package eu.dnetlib.data.mdstore;
|
||||
package eu.dnetlib.services.mdstores.controller;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
|
@ -18,7 +18,7 @@ import io.swagger.v3.oas.annotations.Operation;
|
|||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/api/mdstores")
|
||||
@RequestMapping("/api")
|
||||
@Tag(name = "Metadata Stores")
|
||||
public class MDStoreApiController extends AbstractMDStoreController {
|
||||
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package eu.dnetlib.data.mdstore;
|
||||
package eu.dnetlib.services.mdstores.controller;
|
||||
|
||||
public class StatusResponse {
|
||||
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
package eu.dnetlib.services.mdstores.repository;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
import eu.dnetlib.data.mdstore.model.MDStoreCurrentVersion;
|
||||
|
||||
@Repository
|
||||
public interface MDStoreCurrentVersionRepository extends JpaRepository<MDStoreCurrentVersion, String> {
|
||||
|
||||
long countByCurrentVersion(String versionId);
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
package eu.dnetlib.services.mdstores.repository;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
import eu.dnetlib.data.mdstore.model.MDStore;
|
||||
|
||||
@Repository
|
||||
public interface MDStoreRepository extends JpaRepository<MDStore, String> {
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
package eu.dnetlib.services.mdstores.repository;
|
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
import eu.dnetlib.data.mdstore.model.MDStoreVersion;
|
||||
|
||||
@Repository
|
||||
public interface MDStoreVersionRepository extends JpaRepository<MDStoreVersion, String> {
|
||||
|
||||
void deleteByMdstore(String id);
|
||||
|
||||
long countByMdstoreAndWriting(String id, boolean b);
|
||||
|
||||
long countByMdstoreAndReadCountGreaterThan(String id, int count);
|
||||
|
||||
Iterable<MDStoreVersion> findByMdstoreOrderById(String mdId);
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
package eu.dnetlib.services.mdstores.repository;
|
||||
|
||||
import org.springframework.stereotype.Repository;
|
||||
|
||||
import eu.dnetlib.data.mdstore.model.MDStoreWithInfo;
|
||||
import eu.dnetlib.utils.ReadOnlyRepository;
|
||||
|
||||
@Repository
|
||||
public interface MDStoreWithInfoRepository extends ReadOnlyRepository<MDStoreWithInfo, String> {
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,307 @@
|
|||
package eu.dnetlib.services.mdstores.service;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.Comparator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
import java.util.stream.StreamSupport;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import eu.dnetlib.data.mdstore.model.MDStore;
|
||||
import eu.dnetlib.data.mdstore.model.MDStoreCurrentVersion;
|
||||
import eu.dnetlib.data.mdstore.model.MDStoreType;
|
||||
import eu.dnetlib.data.mdstore.model.MDStoreVersion;
|
||||
import eu.dnetlib.data.mdstore.model.MDStoreWithInfo;
|
||||
import eu.dnetlib.data.mdstore.model.records.MetadataRecord;
|
||||
import eu.dnetlib.errors.MDStoreManagerException;
|
||||
import eu.dnetlib.services.mdstores.backends.DefaultBackend;
|
||||
import eu.dnetlib.services.mdstores.backends.MDStoreBackend;
|
||||
import eu.dnetlib.services.mdstores.backends.MockBackend;
|
||||
import eu.dnetlib.services.mdstores.repository.MDStoreCurrentVersionRepository;
|
||||
import eu.dnetlib.services.mdstores.repository.MDStoreRepository;
|
||||
import eu.dnetlib.services.mdstores.repository.MDStoreVersionRepository;
|
||||
import eu.dnetlib.services.mdstores.repository.MDStoreWithInfoRepository;
|
||||
import jakarta.transaction.Transactional;
|
||||
|
||||
@Service
|
||||
public class MDStoreService {
|
||||
|
||||
@Autowired
|
||||
private MDStoreRepository mdstoreRepository;
|
||||
@Autowired
|
||||
private MDStoreVersionRepository mdstoreVersionRepository;
|
||||
@Autowired
|
||||
private MDStoreCurrentVersionRepository mdstoreCurrentVersionRepository;
|
||||
@Autowired
|
||||
private MDStoreWithInfoRepository mdstoreWithInfoRepository;
|
||||
@Autowired
|
||||
protected JdbcTemplate jdbcTemplate;
|
||||
|
||||
@Autowired
|
||||
private MockBackend mockBackend;
|
||||
@Autowired
|
||||
private DefaultBackend defaultBackend;
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(MDStoreService.class);
|
||||
|
||||
public static final String REFRESH_MODE = "REFRESH";
|
||||
public static final String INCREMENTAL_MODE = "INCREMENTAL";
|
||||
|
||||
public List<MDStoreWithInfo> listMdStores() {
|
||||
|
||||
return StreamSupport.stream(mdstoreWithInfoRepository.findAll().spliterator(), false)
|
||||
.sorted(Comparator.comparing((Function<MDStoreWithInfo, String>) md -> md.getDatasourceName()).thenComparing(md -> md.getId()))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public List<String> listMdStoreIDs() {
|
||||
return mdstoreRepository.findAll().stream().map(MDStore::getId).sorted().collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public long countMdStores() {
|
||||
return mdstoreRepository.count();
|
||||
}
|
||||
|
||||
public Iterable<MDStoreVersion> listVersions(final String mdId) {
|
||||
return mdstoreVersionRepository.findByMdstoreOrderById(mdId);
|
||||
}
|
||||
|
||||
public List<String> listExpiredVersions() {
|
||||
return jdbcTemplate
|
||||
.queryForList("select v.id from mdstore_versions v left outer join mdstore_current_versions cv on (v.id = cv.current_version) where v.writing = false and v.readcount = 0 and cv.mdstore is null;", String.class);
|
||||
}
|
||||
|
||||
public MDStoreWithInfo findMdStore(final String mdId) throws MDStoreManagerException {
|
||||
return mdstoreWithInfoRepository.findById(mdId).orElseThrow(() -> new MDStoreManagerException("Missing mdstore: " + mdId));
|
||||
}
|
||||
|
||||
public MDStoreVersion findVersion(final String versionId) throws MDStoreManagerException {
|
||||
return mdstoreVersionRepository.findById(versionId).orElseThrow(() -> new MDStoreManagerException("Missing mdstore version: " + versionId));
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public String createMDStore(final String format,
|
||||
final String layout,
|
||||
final String interpretation,
|
||||
final MDStoreType type,
|
||||
final String dsName,
|
||||
final String dsId,
|
||||
final String apiId) {
|
||||
|
||||
final MDStore md = newMDStore(format, layout, interpretation, type, dsName, dsId, apiId, apiId);
|
||||
mdstoreRepository.save(md);
|
||||
|
||||
final MDStoreVersion v = newMDStoreVersion(md, false);
|
||||
mdstoreVersionRepository.save(v);
|
||||
mdstoreCurrentVersionRepository.save(MDStoreCurrentVersion.newInstance(v));
|
||||
|
||||
return md.getId();
|
||||
}
|
||||
|
||||
private MDStoreVersion newMDStoreVersion(final MDStore md, final boolean writing) {
|
||||
final MDStoreVersion v = new MDStoreVersion();
|
||||
|
||||
final LocalDateTime now = LocalDateTime.now();
|
||||
|
||||
final String versionId = md.getId() + "-" + now.toEpochSecond(ZoneOffset.UTC);
|
||||
v.setId(versionId);
|
||||
v.setMdstore(md.getId());
|
||||
v.setLastUpdate(null);
|
||||
v.setWriting(writing);
|
||||
v.setReadCount(0);
|
||||
v.setSize(0);
|
||||
v.setLastUpdate(now);
|
||||
|
||||
selectBackend(md.getType()).completeNewMDStoreVersion(v);
|
||||
|
||||
return v;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void deleteMdStore(final String mdId) throws MDStoreManagerException {
|
||||
|
||||
final MDStore md = mdstoreRepository.findById(mdId).orElseThrow(() -> new MDStoreManagerException("MDStore not found: " + mdId));
|
||||
|
||||
if (mdstoreVersionRepository.countByMdstoreAndReadCountGreaterThan(mdId, 0) > 0) {
|
||||
log.error("Read transactions found on mdstore: " + mdId);
|
||||
throw new MDStoreManagerException("Read transactions found on mdstore: " + mdId);
|
||||
}
|
||||
|
||||
if (mdstoreVersionRepository.countByMdstoreAndWriting(mdId, true) > 0) {
|
||||
log.error("Write transactions found on mdstore: " + mdId);
|
||||
throw new MDStoreManagerException("Write transactions found on mdstore: " + mdId);
|
||||
}
|
||||
|
||||
mdstoreCurrentVersionRepository.deleteById(mdId);
|
||||
mdstoreVersionRepository.deleteByMdstore(mdId);
|
||||
mdstoreRepository.deleteById(mdId);
|
||||
|
||||
selectBackend(md.getType()).delete(md);
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public MDStoreVersion startReading(final String mdId) throws MDStoreManagerException {
|
||||
final MDStoreCurrentVersion cv =
|
||||
mdstoreCurrentVersionRepository.findById(mdId).orElseThrow(() -> new MDStoreManagerException("Missing mdstore: " + mdId));
|
||||
final MDStoreVersion v = mdstoreVersionRepository.findById(cv.getCurrentVersion())
|
||||
.orElseThrow(() -> new MDStoreManagerException("Missing version: " + cv.getCurrentVersion()));
|
||||
v.setReadCount(v.getReadCount() + 1);
|
||||
mdstoreVersionRepository.save(v);
|
||||
return v;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public MDStoreVersion endReading(final String versionId) throws MDStoreManagerException {
|
||||
final MDStoreVersion v = mdstoreVersionRepository.findById(versionId).orElseThrow(() -> new MDStoreManagerException("Version not found"));
|
||||
v.setReadCount(Math.max(0, v.getReadCount() - 1));
|
||||
return v;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public MDStoreVersion resetReading(final String versionId) throws MDStoreManagerException {
|
||||
final MDStoreVersion v = mdstoreVersionRepository.findById(versionId).orElseThrow(() -> new MDStoreManagerException("Version not found"));
|
||||
v.setReadCount(0);
|
||||
return v;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public MDStoreVersion prepareMdStoreVersion(final String mdId) throws MDStoreManagerException {
|
||||
final MDStore md = mdstoreRepository.findById(mdId).orElseThrow(() -> new MDStoreManagerException("MDStore not found"));
|
||||
final MDStoreVersion v = newMDStoreVersion(md, true);
|
||||
mdstoreVersionRepository.save(v);
|
||||
return v;
|
||||
}
|
||||
|
||||
public synchronized void deleteExpiredVersions() {
|
||||
log.info("Deleting expired version...");
|
||||
for (final String versionId : listExpiredVersions()) {
|
||||
try {
|
||||
deleteMdStoreVersion(versionId, true);
|
||||
} catch (final MDStoreManagerException e) {
|
||||
log.warn("Error deleteting version " + versionId, e);
|
||||
}
|
||||
}
|
||||
log.info("Done.");
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public void deleteMdStoreVersion(final String versionId, final boolean force) throws MDStoreManagerException {
|
||||
|
||||
final MDStoreVersion v = mdstoreVersionRepository.findById(versionId).orElseThrow(() -> new MDStoreManagerException("Version not found"));
|
||||
final MDStore md = mdstoreRepository.findById(v.getMdstore()).orElseThrow(() -> new MDStoreManagerException("Version not found"));
|
||||
|
||||
if (mdstoreCurrentVersionRepository
|
||||
.countByCurrentVersion(versionId) > 0) {
|
||||
throw new MDStoreManagerException("I cannot delete this version because it is the current version");
|
||||
}
|
||||
|
||||
if (!force) {
|
||||
if (v.isWriting()) { throw new MDStoreManagerException("I cannot delete this version because it is in write mode"); }
|
||||
if (v.getReadCount() > 0) { throw new MDStoreManagerException("I cannot delete this version because it is in read mode"); }
|
||||
}
|
||||
|
||||
mdstoreVersionRepository.delete(v);
|
||||
|
||||
selectBackend(md.getType()).delete(v);
|
||||
}
|
||||
|
||||
public Set<String> listVersionInternalFiles(final String versionId) throws MDStoreManagerException {
|
||||
final MDStoreVersion v = mdstoreVersionRepository.findById(versionId).orElseThrow(() -> new MDStoreManagerException("Version not found"));
|
||||
final MDStore md = mdstoreRepository.findById(v.getMdstore()).orElseThrow(() -> new MDStoreManagerException("MDStore not found"));
|
||||
return selectBackend(md.getType()).listInternalFiles(v);
|
||||
}
|
||||
|
||||
public List<MetadataRecord> listVersionRecords(final String versionId, final long limit) throws MDStoreManagerException {
|
||||
final MDStoreVersion v = mdstoreVersionRepository.findById(versionId).orElseThrow(() -> new MDStoreManagerException("Version not found"));
|
||||
final MDStore md = mdstoreRepository.findById(v.getMdstore()).orElseThrow(() -> new MDStoreManagerException("MDStore not found"));
|
||||
return selectBackend(md.getType()).listEntries(v, limit);
|
||||
}
|
||||
|
||||
public Stream<MetadataRecord> streamVersionRecords(final String versionId) throws MDStoreManagerException {
|
||||
final MDStoreVersion v = mdstoreVersionRepository.findById(versionId).orElseThrow(() -> new MDStoreManagerException("Version not found"));
|
||||
final MDStore md = mdstoreRepository.findById(v.getMdstore()).orElseThrow(() -> new MDStoreManagerException("MDStore not found"));
|
||||
return selectBackend(md.getType()).streamEntries(v);
|
||||
}
|
||||
|
||||
public MDStore newMDStore(
|
||||
final String format,
|
||||
final String layout,
|
||||
final String interpretation,
|
||||
final MDStoreType type,
|
||||
final String dsName,
|
||||
final String dsId,
|
||||
final String apiId,
|
||||
final String hdfsBasePath) {
|
||||
|
||||
final String mdId = "md-" + UUID.randomUUID();
|
||||
|
||||
final MDStore md = new MDStore();
|
||||
md.setId(mdId);
|
||||
md.setFormat(format);
|
||||
md.setLayout(layout);
|
||||
md.setType(type);
|
||||
md.setInterpretation(interpretation);
|
||||
md.setCreationDate(LocalDateTime.now());
|
||||
md.setDatasourceName(dsName);
|
||||
md.setDatasourceId(dsId);
|
||||
md.setApiId(apiId);
|
||||
|
||||
selectBackend(type).completeNewMDStore(md);
|
||||
|
||||
return md;
|
||||
}
|
||||
|
||||
public Map<MDStoreType, Set<String>> fixInconsistencies(final boolean delete) throws MDStoreManagerException {
|
||||
final Map<MDStoreType, Set<String>> res = new LinkedHashMap<>();
|
||||
// res.put(MDStoreType.HDFS, hdfsBackend.fixInconsistencies(delete));
|
||||
res.put(MDStoreType.MOCK, mockBackend.fixInconsistencies(delete));
|
||||
|
||||
// TODO: ADD HERE THE INVOCATION FOR OTHER MDSTORE TYPE
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
private MDStoreBackend selectBackend(final MDStoreType type) {
|
||||
switch (type) {
|
||||
case MOCK:
|
||||
return mockBackend;
|
||||
default:
|
||||
return defaultBackend;
|
||||
}
|
||||
}
|
||||
|
||||
public void addRecord(final String newVersion, final MDStoreType type, final MetadataRecord record) {
|
||||
selectBackend(type).addRecord(record);
|
||||
}
|
||||
|
||||
public long commitMdStoreVersion(final String versionId, final MDStoreType mdStoreType) throws MDStoreManagerException {
|
||||
final long count = selectBackend(mdStoreType).countRecords(versionId);
|
||||
commitMdStoreVersion(versionId, count);
|
||||
return count;
|
||||
}
|
||||
|
||||
@Transactional
|
||||
public MDStoreVersion commitMdStoreVersion(final String versionId, final long size) throws MDStoreManagerException {
|
||||
final MDStoreVersion v = mdstoreVersionRepository.findById(versionId).orElseThrow(() -> new MDStoreManagerException("Invalid version: " + versionId));
|
||||
mdstoreCurrentVersionRepository.save(MDStoreCurrentVersion.newInstance(v));
|
||||
v.setWriting(false);
|
||||
v.setSize(size);
|
||||
v.setLastUpdate(LocalDateTime.now());
|
||||
mdstoreVersionRepository.save(v);
|
||||
|
||||
return v;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,90 @@
|
|||
package eu.dnetlib.services.mdstores.service;
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.util.Spliterators;
|
||||
import java.util.stream.Stream;
|
||||
import java.util.stream.StreamSupport;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import eu.dnetlib.data.mdstore.model.MDStoreWithInfo;
|
||||
import eu.dnetlib.data.mdstore.model.records.MetadataRecord;
|
||||
import eu.dnetlib.errors.MDStoreManagerException;
|
||||
|
||||
@Service
|
||||
public class MDStoreStreamReader {
|
||||
|
||||
@Autowired
|
||||
private MDStoreService mdStoreService;
|
||||
|
||||
private enum Status {
|
||||
PREPARED,
|
||||
READING,
|
||||
COMPLETED,
|
||||
FAILED
|
||||
}
|
||||
|
||||
// TODO the failure could be throw consuming the stream, so it is necessary to perform a refactoring of this method
|
||||
public Stream<MetadataRecord> prepareMDStoreStream(final String mdstoreId) throws MDStoreManagerException {
|
||||
|
||||
final MDStoreWithInfo mdstore = mdStoreService.findMdStore(mdstoreId);
|
||||
final Iterator<? extends MetadataRecord> innerIterator = mdStoreService.streamVersionRecords(mdstore.getCurrentVersion()).iterator();
|
||||
|
||||
return StreamSupport.stream(Spliterators.spliteratorUnknownSize(new Iterator<MetadataRecord>() {
|
||||
|
||||
private Status status = Status.PREPARED;
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
if (innerIterator.hasNext()) {
|
||||
return true;
|
||||
} else {
|
||||
try {
|
||||
complete();
|
||||
return false;
|
||||
} catch (final MDStoreManagerException e) {
|
||||
throw new RuntimeException("Error reading mdstore", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public MetadataRecord next() {
|
||||
try {
|
||||
verifyStart();
|
||||
return innerIterator.next();
|
||||
} catch (final Throwable e) {
|
||||
try {
|
||||
fail();
|
||||
throw new RuntimeException("Error reading mdstore", e);
|
||||
} catch (final MDStoreManagerException e1) {
|
||||
throw new RuntimeException("Error reading mdstore", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private synchronized void verifyStart() throws MDStoreManagerException {
|
||||
if (status == Status.PREPARED) {
|
||||
status = Status.READING;
|
||||
mdStoreService.startReading(mdstoreId);
|
||||
}
|
||||
}
|
||||
|
||||
private synchronized void complete() throws MDStoreManagerException {
|
||||
if (status == Status.PREPARED || status == Status.READING) {
|
||||
status = Status.COMPLETED;
|
||||
mdStoreService.endReading(mdstoreId);
|
||||
}
|
||||
}
|
||||
|
||||
private synchronized void fail() throws MDStoreManagerException {
|
||||
if (status == Status.PREPARED || status == Status.READING) {
|
||||
status = Status.FAILED;
|
||||
mdStoreService.endReading(mdstoreId);
|
||||
}
|
||||
}
|
||||
}, 0), false);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -12,6 +12,9 @@
|
|||
<packaging>pom</packaging>
|
||||
|
||||
<modules>
|
||||
<module>info</module>
|
||||
<module>mdstores</module>
|
||||
<module>contexts</module>
|
||||
<module>collector</module>
|
||||
<module>dsm</module>
|
||||
<module>oaiExporter</module>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,39 @@
|
|||
package eu.dnetlib.service.transform.controller;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import eu.dnetlib.base.AbstractDnetController;
|
||||
import eu.dnetlib.data.model.vocabulary.Vocabulary;
|
||||
import eu.dnetlib.service.transform.importer.VocabularyImporter;
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/api/import")
|
||||
public class ImporterController extends AbstractDnetController {
|
||||
|
||||
// EXAMPLE:
|
||||
// find ./VocabularyDSResourceType/ -name "*.xml" -exec curl -X POST "http://localhost:8280/api/import/vocabulary" -H "accept: */*" -H
|
||||
// "Content-Type: text/plain" --data-binary @{} \;
|
||||
// find ./DedupConfigurationDSResources/ -name "*.xml" -exec curl -X POST "http://localhost:8280/api/import/resource" -H "accept: */*"
|
||||
// -H "Content-Type: text/plain" --data-binary @{} \;
|
||||
|
||||
@Autowired
|
||||
private VocabularyImporter vocabularyImporter;
|
||||
|
||||
@PostMapping(value = "/vocabulary", consumes = {
|
||||
MediaType.TEXT_PLAIN_VALUE, MediaType.APPLICATION_XML_VALUE, MediaType.APPLICATION_JSON_VALUE
|
||||
})
|
||||
public Vocabulary importVocabulary(final HttpServletRequest request) throws Exception, IOException {
|
||||
final String xml = IOUtils.toString(request.getInputStream(), StandardCharsets.UTF_8);
|
||||
return vocabularyImporter.importVocabulary(xml);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
package eu.dnetlib.service.transform.importer;
|
||||
|
||||
import org.dom4j.Document;
|
||||
import org.dom4j.DocumentHelper;
|
||||
import org.dom4j.Node;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import eu.dnetlib.data.model.vocabulary.Synonym;
|
||||
import eu.dnetlib.data.model.vocabulary.Vocabulary;
|
||||
import eu.dnetlib.data.model.vocabulary.VocabularyTerm;
|
||||
import eu.dnetlib.service.transform.vocabulary.VocabularyRepository;
|
||||
import eu.dnetlib.service.transform.vocabulary.VocabularyTermRepository;
|
||||
import jakarta.transaction.Transactional;
|
||||
|
||||
@Service
|
||||
public class VocabularyImporter {
|
||||
|
||||
@Autowired
|
||||
private VocabularyRepository vocabularyRepository;
|
||||
|
||||
@Autowired
|
||||
private VocabularyTermRepository vocabularyTermRepository;
|
||||
|
||||
@Transactional
|
||||
public Vocabulary importVocabulary(final String xml) throws Exception {
|
||||
final Document doc = DocumentHelper.parseText(xml);
|
||||
|
||||
final Vocabulary voc = new Vocabulary();
|
||||
final String vocId = doc.valueOf("//VOCABULARY_NAME/@code");
|
||||
final String vocName = doc.valueOf("//VOCABULARY_NAME");
|
||||
final String vocDesc = doc.valueOf("//VOCABULARY_DESCRIPTION");
|
||||
|
||||
voc.setId(vocId);
|
||||
voc.setName(vocName);
|
||||
voc.setDescription(vocDesc);
|
||||
|
||||
vocabularyRepository.save(voc);
|
||||
|
||||
for (final Node n : doc.selectNodes("//TERM")) {
|
||||
final VocabularyTerm term = new VocabularyTerm();
|
||||
term.setVocabulary(vocId);
|
||||
term.setCode(n.valueOf("@code"));
|
||||
term.setName(n.valueOf("@english_name"));
|
||||
term.setEncoding(n.valueOf("@encoding"));
|
||||
term.setSynonyms(n.selectNodes(".//SYNONYM")
|
||||
.stream()
|
||||
.map(ns -> new Synonym(ns.valueOf("@term"), ns.valueOf("@encoding")))
|
||||
.sorted()
|
||||
.distinct()
|
||||
.toArray(Synonym[]::new));
|
||||
|
||||
vocabularyTermRepository.save(term);
|
||||
}
|
||||
|
||||
return voc;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
package eu.dnetlib.manager.wf.controller;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import eu.dnetlib.base.AbstractDnetController;
|
||||
import eu.dnetlib.manager.wf.importer.WfHistoryImporter;
|
||||
|
||||
@RestController
|
||||
@RequestMapping("/import")
|
||||
public class ImporterController extends AbstractDnetController {
|
||||
|
||||
// EXAMPLE:
|
||||
// find ./VocabularyDSResourceType/ -name "*.xml" -exec curl -X POST "http://localhost:8280/api/import/vocabulary" -H "accept: */*" -H
|
||||
// "Content-Type: text/plain" --data-binary @{} \;
|
||||
// find ./DedupConfigurationDSResources/ -name "*.xml" -exec curl -X POST "http://localhost:8280/api/import/resource" -H "accept: */*"
|
||||
// -H "Content-Type: text/plain" --data-binary @{} \;
|
||||
|
||||
@Autowired
|
||||
private WfHistoryImporter wfHistoryImporter;
|
||||
|
||||
@GetMapping(value = "/wf_logs")
|
||||
public List<String> importWfLogs(@RequestParam final String path) throws Exception {
|
||||
// mongoexport -d dnet_logs -c wf_logs --jsonArray -o /tmp/mongodump.json
|
||||
|
||||
wfHistoryImporter.load(path);
|
||||
|
||||
return Arrays.asList("Done.");
|
||||
}
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package eu.dnetlib.is.importer;
|
||||
package eu.dnetlib.manager.wf.importer;
|
||||
|
||||
import java.io.File;
|
||||
import java.time.Instant;
|
||||
|
|
@ -19,8 +19,8 @@ import org.springframework.stereotype.Service;
|
|||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import eu.dnetlib.manager.history.model.WfHistoryEntry;
|
||||
import eu.dnetlib.manager.history.repository.WfHistoryEntryRepository;
|
||||
import eu.dnetlib.data.wfs.model.WfHistoryEntry;
|
||||
import eu.dnetlib.manager.wf.repository.WfHistoryEntryRepository;
|
||||
|
||||
@Service
|
||||
public class WfHistoryImporter {
|
||||
Loading…
Reference in New Issue