refactoring

data/nginx-templates/default.conf.template

@@ -10,10 +10,6 @@ server {
 
 	location / {}
 		
-	location /collector-postgres/ {
-		 proxy_pass http://collector-postgres:8080/collector/;
-	}
-
 	location /context-manager/ {
 		 proxy_pass http://context-manager:8080/context-manager/;
 	}
@@ -42,8 +38,8 @@ server {
 		 proxy_pass http://resource-manager:8080/resource-manager/;
 	}
 		
-	location /transformer-postgres/ {
-		 proxy_pass http://transformer-postgres:8080/transformer/;
+	location /wf-exec-postgres/ {
+		 proxy_pass http://wf-exec-postgres:8080/wf-exec/;
 	}
 	
 	location /wf-manager/ {

dnet-app/apps/dnet-collector-postgres/src/main/java/eu/dnetlib/services/collector/CollectorApplication.java

@@ -1,19 +0,0 @@
-package eu.dnetlib.services.collector;
-
-import org.springframework.boot.SpringApplication;
-import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.boot.autoconfigure.domain.EntityScan;
-
-import eu.dnetlib.common.app.AbstractDnetApp;
-import eu.dnetlib.domain.protocol.Protocol;
-import eu.dnetlib.domain.protocol.ProtocolParam;
-
-@SpringBootApplication
-@EntityScan(basePackageClasses = { Protocol.class, ProtocolParam.class })
-public class CollectorApplication extends AbstractDnetApp {
-
-	public static void main(final String[] args) {
-		SpringApplication.run(CollectorApplication.class, args);
-	}
-
-}

dnet-app/apps/dnet-collector-postgres/src/main/resources/application.properties

@@ -1,6 +0,0 @@
-server.title                = D-NET Collector API (postgres)
-server.description          = Simple service to collect metadata to a postgres database 
-server.servlet.context-path = /collector
-server.public_url           = 
-
-maven.pom.path = /META-INF/maven/eu.dnetlib.docker/dnet-collector-postgres/effective-pom.xml

dnet-app/apps/dnet-context-manager/src/main/java/eu/dnetlib/services/contexts/ContextsApplication.java

@@ -2,10 +2,17 @@ package eu.dnetlib.services.contexts;
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.domain.EntityScan;
 
 import eu.dnetlib.common.app.AbstractDnetApp;
+import eu.dnetlib.domain.context.Category;
+import eu.dnetlib.domain.context.ConceptLevel0;
+import eu.dnetlib.domain.context.ConceptLevel1;
+import eu.dnetlib.domain.context.ConceptLevel2;
+import eu.dnetlib.domain.context.Context;
 
 @SpringBootApplication
+@EntityScan(basePackageClasses = { Context.class, Category.class, ConceptLevel0.class, ConceptLevel1.class, ConceptLevel2.class })
 public class ContextsApplication extends AbstractDnetApp {
 
 	public static void main(final String[] args) {

dnet-app/apps/dnet-context-manager/src/main/java/eu/dnetlib/services/contexts/importer/ContextImporter.java

@@ -19,11 +19,11 @@ import eu.dnetlib.domain.context.Context;
 import eu.dnetlib.domain.context.CtxChildInfo;
 import eu.dnetlib.domain.context.CtxInfo;
 import eu.dnetlib.domain.context.Parameter;
-import eu.dnetlib.services.contexts.service.repository.CategoryRepository;
-import eu.dnetlib.services.contexts.service.repository.ConceptLevel0Repository;
-import eu.dnetlib.services.contexts.service.repository.ConceptLevel1Repository;
-import eu.dnetlib.services.contexts.service.repository.ConceptLevel2Repository;
-import eu.dnetlib.services.contexts.service.repository.ContextRepository;
+import eu.dnetlib.services.contexts.repository.CategoryRepository;
+import eu.dnetlib.services.contexts.repository.ConceptLevel0Repository;
+import eu.dnetlib.services.contexts.repository.ConceptLevel1Repository;
+import eu.dnetlib.services.contexts.repository.ConceptLevel2Repository;
+import eu.dnetlib.services.contexts.repository.ContextRepository;
 
 @Service
 public class ContextImporter {

dnet-app/apps/dnet-context-manager/src/main/java/eu/dnetlib/services/contexts/repository/CategoryRepository.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.services.contexts.service.repository;
+package eu.dnetlib.services.contexts.repository;
 
 import org.springframework.data.jpa.repository.JpaRepository;
 

dnet-app/apps/dnet-context-manager/src/main/java/eu/dnetlib/services/contexts/repository/ConceptLevel0Repository.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.services.contexts.service.repository;
+package eu.dnetlib.services.contexts.repository;
 
 import org.springframework.data.jpa.repository.JpaRepository;
 

dnet-app/apps/dnet-context-manager/src/main/java/eu/dnetlib/services/contexts/repository/ConceptLevel1Repository.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.services.contexts.service.repository;
+package eu.dnetlib.services.contexts.repository;
 
 import org.springframework.data.jpa.repository.JpaRepository;
 

dnet-app/apps/dnet-context-manager/src/main/java/eu/dnetlib/services/contexts/repository/ConceptLevel2Repository.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.services.contexts.service.repository;
+package eu.dnetlib.services.contexts.repository;
 
 import org.springframework.data.jpa.repository.JpaRepository;
 

dnet-app/apps/dnet-context-manager/src/main/java/eu/dnetlib/services/contexts/repository/ContextRepository.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.services.contexts.service.repository;
+package eu.dnetlib.services.contexts.repository;
 
 import org.springframework.data.jpa.repository.JpaRepository;
 

dnet-app/apps/dnet-context-manager/src/main/java/eu/dnetlib/services/contexts/service/ContextService.java

@@ -16,11 +16,11 @@ import eu.dnetlib.domain.context.Category;
 import eu.dnetlib.domain.context.Context;
 import eu.dnetlib.domain.context.CtxChildInfo;
 import eu.dnetlib.errors.InformationServiceException;
-import eu.dnetlib.services.contexts.service.repository.CategoryRepository;
-import eu.dnetlib.services.contexts.service.repository.ConceptLevel0Repository;
-import eu.dnetlib.services.contexts.service.repository.ConceptLevel1Repository;
-import eu.dnetlib.services.contexts.service.repository.ConceptLevel2Repository;
-import eu.dnetlib.services.contexts.service.repository.ContextRepository;
+import eu.dnetlib.services.contexts.repository.CategoryRepository;
+import eu.dnetlib.services.contexts.repository.ConceptLevel0Repository;
+import eu.dnetlib.services.contexts.repository.ConceptLevel1Repository;
+import eu.dnetlib.services.contexts.repository.ConceptLevel2Repository;
+import eu.dnetlib.services.contexts.repository.ContextRepository;
 
 @Service
 public class ContextService {

dnet-app/apps/dnet-datasource-manager/src/main/java/eu/dnetlib/services/dsm/DsmApplication.java

@@ -2,10 +2,27 @@ package eu.dnetlib.services.dsm;
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.domain.EntityScan;
 
 import eu.dnetlib.common.app.AbstractDnetApp;
+import eu.dnetlib.domain.dsm.Api;
+import eu.dnetlib.domain.dsm.BrowseTerm;
+import eu.dnetlib.domain.dsm.Datasource;
+import eu.dnetlib.domain.dsm.readonly.ApiWithAdditionalInfo;
+import eu.dnetlib.domain.dsm.readonly.SimpleDsWithApis;
+import eu.dnetlib.domain.protocol.Protocol;
+import eu.dnetlib.domain.protocol.ProtocolParam;
 
 @SpringBootApplication
+@EntityScan(basePackageClasses = {
+		Datasource.class,
+		Api.class,
+		BrowseTerm.class,
+		SimpleDsWithApis.class,
+		ApiWithAdditionalInfo.class,
+		Protocol.class,
+		ProtocolParam.class
+})
 public class DsmApplication extends AbstractDnetApp {
 
 	public static void main(final String[] args) {

dnet-app/apps/dnet-datasource-manager/src/main/java/eu/dnetlib/services/dsm/controller/AbstractProtocolController.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.services.collector.controller;
+package eu.dnetlib.services.dsm.controller;
 
 import java.util.List;
 
@@ -7,7 +7,7 @@ import org.springframework.web.bind.annotation.GetMapping;
 
 import eu.dnetlib.common.controller.DnetRestController;
 import eu.dnetlib.domain.protocol.ProtocolDesc;
-import eu.dnetlib.services.collector.service.ProtocolService;
+import eu.dnetlib.services.dsm.service.ProtocolService;
 
 public class AbstractProtocolController extends DnetRestController {
 

dnet-app/apps/dnet-datasource-manager/src/main/java/eu/dnetlib/services/dsm/controller/ProtocolAjaxController.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.services.collector.controller;
+package eu.dnetlib.services.dsm.controller;
 
 import java.util.List;
 

dnet-app/apps/dnet-datasource-manager/src/main/java/eu/dnetlib/services/dsm/controller/ProtocolApiController.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.services.collector.controller;
+package eu.dnetlib.services.dsm.controller;
 
 
 import org.springframework.web.bind.annotation.RequestMapping;

dnet-app/apps/dnet-datasource-manager/src/main/java/eu/dnetlib/services/dsm/repository/ProtocolParamRepository.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.services.collector.repository;
+package eu.dnetlib.services.dsm.repository;
 
 import org.springframework.data.jpa.repository.JpaRepository;
 

dnet-app/apps/dnet-datasource-manager/src/main/java/eu/dnetlib/services/dsm/repository/ProtocolRepository.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.services.collector.repository;
+package eu.dnetlib.services.dsm.repository;
 
 import org.springframework.data.jpa.repository.JpaRepository;
 

dnet-app/apps/dnet-datasource-manager/src/main/java/eu/dnetlib/services/dsm/service/ProtocolService.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.services.collector.service;
+package eu.dnetlib.services.dsm.service;
 
 import java.util.ArrayList;
 import java.util.Comparator;
@@ -14,8 +14,8 @@ import org.springframework.stereotype.Service;
 import eu.dnetlib.domain.protocol.Protocol;
 import eu.dnetlib.domain.protocol.ProtocolDesc;
 import eu.dnetlib.domain.protocol.ProtocolParam;
-import eu.dnetlib.services.collector.repository.ProtocolParamRepository;
-import eu.dnetlib.services.collector.repository.ProtocolRepository;
+import eu.dnetlib.services.dsm.repository.ProtocolParamRepository;
+import eu.dnetlib.services.dsm.repository.ProtocolRepository;
 import jakarta.transaction.Transactional;
 
 @Service
@@ -33,15 +33,15 @@ public class ProtocolService {
 	public List<ProtocolDesc> listProtocols() {
 
 		final Map<String, List<ProtocolParam>> params = protocolParamRepository.findAll()
-			.stream()
-			.collect(Collectors.groupingBy(ProtocolParam::getProtocol));
+				.stream()
+				.collect(Collectors.groupingBy(ProtocolParam::getProtocol));
 
 		return protocolRepository.findAll()
-			.stream()
-			.map(Protocol::getId)
-			.map(id -> new ProtocolDesc(id, params.containsKey(id) ? params.get(id) : new ArrayList<>()))
-			.sorted(Comparator.comparing(ProtocolDesc::getId))
-			.collect(Collectors.toList());
+				.stream()
+				.map(Protocol::getId)
+				.map(id -> new ProtocolDesc(id, params.containsKey(id) ? params.get(id) : new ArrayList<>()))
+				.sorted(Comparator.comparing(ProtocolDesc::getId))
+				.collect(Collectors.toList());
 
 	}
 

dnet-app/apps/dnet-mdstore-manager/src/main/java/eu/dnetlib/services/mdstores/MdstoresApplication.java

@@ -2,10 +2,16 @@ package eu.dnetlib.services.mdstores;
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.domain.EntityScan;
 
 import eu.dnetlib.common.app.AbstractDnetApp;
+import eu.dnetlib.domain.mdstore.MDStore;
+import eu.dnetlib.domain.mdstore.MDStoreCurrentVersion;
+import eu.dnetlib.domain.mdstore.MDStoreVersion;
+import eu.dnetlib.domain.mdstore.MDStoreWithInfo;
 
 @SpringBootApplication
+@EntityScan(basePackageClasses = { MDStore.class, MDStoreCurrentVersion.class, MDStoreVersion.class, MDStoreWithInfo.class })
 public class MdstoresApplication extends AbstractDnetApp {
 
 	public static void main(final String[] args) {

dnet-app/apps/dnet-mdstore-manager/src/main/resources/application.properties

@@ -5,3 +5,7 @@ server.public_url =
 
 maven.pom.path = /META-INF/maven/eu.dnetlib.docker/dnet-mdstore-manager/effective-pom.xml
 
+dhp.mdstore-manager.hadoop.cluster = DEV
+dhp.mdstore-manager.hadoop.user = dnet
+dhp.mdstore-manager.hdfs.base-path = /tmp
+

dnet-app/apps/dnet-resource-manager/src/main/java/eu/dnetlib/services/is/MainApplication.java

@@ -2,10 +2,15 @@ package eu.dnetlib.services.is;
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.domain.EntityScan;
 
 import eu.dnetlib.common.app.AbstractDnetApp;
+import eu.dnetlib.domain.resource.ResourceType;
+import eu.dnetlib.domain.resource.SimpleResource;
 
 @SpringBootApplication
+
+@EntityScan(basePackageClasses = { ResourceType.class, SimpleResource.class })
 public class MainApplication extends AbstractDnetApp {
 
 	public static void main(final String[] args) {

dnet-app/apps/dnet-transformer-postgres/dockerfile

@@ -1,14 +0,0 @@
-# Fetching latest version of Java
-FROM openjdk:18
- 
-# Setting up work directory
-WORKDIR /app
-
-# Copy the jar file into our app
-COPY ./target/*.jar /app/app.jar
-
-# Exposing port 8080
-EXPOSE 8080
-
-# Starting the application
-CMD ["java", "-jar", "app.jar"]

dnet-app/apps/dnet-transformer-postgres/src/main/java/eu/dnetlib/service/transform/TransformerApplication.java

@@ -1,15 +0,0 @@
-package eu.dnetlib.service.transform;
-
-import org.springframework.boot.SpringApplication;
-import org.springframework.boot.autoconfigure.SpringBootApplication;
-
-import eu.dnetlib.common.app.AbstractDnetApp;
-
-@SpringBootApplication
-public class TransformerApplication extends AbstractDnetApp {
-
-	public static void main(final String[] args) {
-		SpringApplication.run(TransformerApplication.class, args);
-	}
-
-}

dnet-app/apps/dnet-transformer-postgres/src/main/resources/application.properties

@@ -1,6 +0,0 @@
-server.title                = D-NET Transformer API (postgres)
-server.description          = Simple service to transform metadata in a postgres database 
-server.servlet.context-path = /transformer
-server.public_url           = 
-
-maven.pom.path = /META-INF/maven/eu.dnetlib.docker/dnet-transformer-postgres/effective-pom.xml

dnet-app/apps/dnet-wf-executor-postgres/pom.xml

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 
 	<parent>
@@ -8,7 +8,7 @@
 	</parent>
 
 	<modelVersion>4.0.0</modelVersion>
-	<artifactId>dnet-collector-postgres</artifactId>
+	<artifactId>dnet-wf-executor-postgres</artifactId>
 	<packaging>jar</packaging>
 
 	<dependencies>
@@ -18,7 +18,7 @@
       		<artifactId>dnet-db-common</artifactId>
       		<version>${project.version}</version>
     	</dependency>
-      
+		
 		<!-- hot swapping, disable cache for template, enable live reload -->
 		<dependency>
 			<groupId>org.springframework.boot</groupId>

dnet-app/apps/dnet-wf-executor-postgres/src/main/java/eu/dnetlib/manager/WfExecutorApplication.java

@@ -0,0 +1,19 @@
+package eu.dnetlib.manager;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.domain.EntityScan;
+
+import eu.dnetlib.common.app.AbstractDnetApp;
+import eu.dnetlib.domain.vocabulary.Vocabulary;
+import eu.dnetlib.domain.vocabulary.VocabularyTerm;
+
+@SpringBootApplication
+@EntityScan(basePackageClasses = { Vocabulary.class, VocabularyTerm.class })
+public class WfExecutorApplication extends AbstractDnetApp {
+
+	public static void main(final String[] args) {
+		SpringApplication.run(WfExecutorApplication.class, args);
+	}
+
+}

dnet-app/apps/dnet-wf-executor-postgres/src/main/java/eu/dnetlib/service/collector/controller/CollectorMainController.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.services.collector.controller;
+package eu.dnetlib.service.collector.controller;
 
 import org.springframework.stereotype.Controller;
 

dnet-app/apps/dnet-wf-executor-postgres/src/main/java/eu/dnetlib/service/collector/plugins/filesystem/FileSystemIterable.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.services.collector.plugins.filesystem;
+package eu.dnetlib.service.collector.plugins.filesystem;
 
 import java.io.File;
 import java.io.FileInputStream;

dnet-app/apps/dnet-wf-executor-postgres/src/main/java/eu/dnetlib/service/collector/plugins/filesystem/FileSystemIterator.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.services.collector.plugins.filesystem;
+package eu.dnetlib.service.collector.plugins.filesystem;
 
 import java.io.IOException;
 import java.nio.file.Files;

dnet-app/apps/dnet-wf-executor-postgres/src/main/java/eu/dnetlib/service/collector/utils/MetadataBuilder.java

@@ -1,4 +1,4 @@
-package eu.dnetlib.services.collector.utils;
+package eu.dnetlib.service.collector.utils;
 
 import org.dom4j.Document;
 import org.dom4j.DocumentHelper;

dnet-app/apps/dnet-wf-executor-postgres/src/main/resources/application.properties

@@ -0,0 +1,6 @@
+server.title                = D-NET Wf executor that uses postgres as backend
+server.description          = D-NET Wf executor that uses postgres as backend
+server.servlet.context-path = /wf-exec
+server.public_url           = 
+
+maven.pom.path = /META-INF/maven/eu.dnetlib.docker/dnet-wf-executor-postgres/effective-pom.xml

dnet-app/apps/dnet-wf-manager/src/main/java/eu/dnetlib/manager/wf/WfManagerApplication.java

@@ -2,10 +2,21 @@ package eu.dnetlib.manager.wf;
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.domain.EntityScan;
 
 import eu.dnetlib.common.app.AbstractDnetApp;
+import eu.dnetlib.domain.wfs.WfHistoryEntry;
+import eu.dnetlib.domain.wfs.WorkflowConfiguration;
+import eu.dnetlib.domain.wfs.WorkflowSection;
+import eu.dnetlib.domain.wfs.WorkflowSubscription;
 
 @SpringBootApplication
+@EntityScan(basePackageClasses = {
+		WfHistoryEntry.class,
+		WorkflowConfiguration.class,
+		WorkflowSection.class,
+		WorkflowSubscription.class
+})
 public class WfManagerApplication extends AbstractDnetApp {
 
 	public static void main(final String[] args) {

dnet-app/apps/pom.xml

@@ -17,8 +17,7 @@
     <module>dnet-context-manager</module>
     <module>dnet-datasource-manager</module>
     <module>dnet-wf-manager</module>
-    <module>dnet-collector-postgres</module>
-    <module>dnet-transformer-postgres</module>
+    <module>dnet-wf-executor-postgres</module>
     <module>dnet-oai-exporter</module>
     <module>dnet-indexer-solr</module>
     <module>dnet-email-sender</module>

docker-compose.yml

@@ -78,8 +78,8 @@ services:
       - SPRING_DATASOURCE_USERNAME=${PG_USER}
       - SPRING_DATASOURCE_PASSWORD=${PG_PASSWORD}
 
-  transformer-postgres:
-    build: ./dnet-app/apps/dnet-transformer-postgres
+  wf-exec-postgres:
+    build: ./dnet-app/apps/dnet-wf-executor-postgres
     expose: 
       - ${SPRING_BOOT_PORT}
     networks:
@@ -104,19 +104,6 @@ services:
       - SPRING_DATASOURCE_USERNAME=${PG_USER}
       - SPRING_DATASOURCE_PASSWORD=${PG_PASSWORD}
 
-  collector-postgres:
-    build: ./dnet-app/apps/dnet-collector-postgres
-    expose: 
-      - ${SPRING_BOOT_PORT}
-    networks:
-      - backend
-    depends_on:
-      - maindb
-    environment:
-      - SPRING_DATASOURCE_URL=jdbc:postgresql://maindb:${PG_PORT}/${PG_MAIN_DB}
-      - SPRING_DATASOURCE_USERNAME=${PG_USER}
-      - SPRING_DATASOURCE_PASSWORD=${PG_PASSWORD}
-
   mail-sender:
     build: ./dnet-app/apps/dnet-email-sender
     expose: