fix after merge

apps/dnet-directindex-api/pom.xml

@@ -0,0 +1,77 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+	<parent>
+		<groupId>eu.dnetlib.dhp</groupId>
+		<artifactId>apps</artifactId>
+		<version>3.3.3-SNAPSHOT</version>
+		<relativePath>../</relativePath>
+	</parent>
+
+	<modelVersion>4.0.0</modelVersion>
+	<artifactId>dnet-directindex-api</artifactId>
+	<packaging>jar</packaging>
+
+	<dependencies>
+		<dependency>
+			<groupId>org.springframework.boot</groupId>
+			<artifactId>spring-boot-starter-data-jpa</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework.boot</groupId>
+			<artifactId>spring-boot-starter-json</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.commons</groupId>
+			<artifactId>commons-text</artifactId>
+			<version>1.9</version>
+		</dependency>
+		<dependency>
+			<groupId>org.postgresql</groupId>
+			<artifactId>postgresql</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.solr</groupId>
+			<artifactId>solr-solrj</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>eu.dnetlib.dhp</groupId>
+			<artifactId>dhp-schemas</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.velocity</groupId>
+			<artifactId>velocity</artifactId>
+			<version>1.7</version>
+			<exclusions>
+				<exclusion>
+					<artifactId>antlr</artifactId>
+					<groupId>antlr</groupId>
+				</exclusion>
+			</exclusions>
+		</dependency>
+
+		<!-- hot swapping, disable cache for template, enable live reload -->
+		<dependency>
+			<groupId>org.springframework.boot</groupId>
+			<artifactId>spring-boot-devtools</artifactId>
+			<optional>true</optional>
+		</dependency>
+
+	</dependencies>
+
+	<build>
+		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-help-plugin</artifactId>
+			</plugin>
+			<plugin>
+				<groupId>org.springframework.boot</groupId>
+				<artifactId>spring-boot-maven-plugin</artifactId>
+			</plugin>
+		</plugins>
+	</build>
+
+</project>

apps/dnet-directindex-api/src/main/java/eu/dnetlib/openaire/directindex/AdminController.java

@@ -0,0 +1,25 @@
+package eu.dnetlib.openaire.directindex;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.ResponseStatus;
+import org.springframework.web.bind.annotation.RestController;
+
+import eu.dnetlib.common.controller.AbstractDnetController;
+import eu.dnetlib.openaire.directindex.is.ISLookupClient;
+
+@RestController
+@RequestMapping("/api/admin")
+public class AdminController extends AbstractDnetController {
+
+	@Autowired
+	private ISLookupClient isLookupClient;
+
+	@GetMapping("/evictCache")
+	@ResponseStatus(HttpStatus.OK)
+	public void evictCache() {
+		isLookupClient.evictCache();
+	}
+}

apps/dnet-directindex-api/src/main/java/eu/dnetlib/openaire/directindex/DirectIndexApiException.java

@@ -0,0 +1,22 @@
+package eu.dnetlib.openaire.directindex;
+
+public class DirectIndexApiException extends Exception {
+
+	/**
+	 *
+	 */
+	private static final long serialVersionUID = -3888037031334809448L;
+
+	public DirectIndexApiException(final String string) {
+		super(string);
+	}
+
+	public DirectIndexApiException(final String string, final Throwable exception) {
+		super(string, exception);
+	}
+
+	public DirectIndexApiException(final Throwable exception) {
+		super(exception);
+	}
+
+}

apps/dnet-directindex-api/src/main/java/eu/dnetlib/openaire/directindex/DirectIndexApplication.java

@@ -0,0 +1,61 @@
+package eu.dnetlib.openaire.directindex;
+
+import java.util.Properties;
+
+import org.apache.velocity.app.VelocityEngine;
+import org.springdoc.core.GroupedOpenApi;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.cache.annotation.EnableCaching;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.scheduling.annotation.EnableScheduling;
+
+import eu.dnetlib.common.app.AbstractDnetApp;
+import eu.dnetlib.common.clients.ISLookupClientFactory;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+
+@SpringBootApplication
+@EnableCaching
+@EnableScheduling
+@ComponentScan(basePackages = "eu.dnetlib")
+public class DirectIndexApplication extends AbstractDnetApp {
+
+	@Value("${openaire.service.islookup.wsdl}")
+	private String isLookupUrl;
+
+	public static void main(final String[] args) {
+		SpringApplication.run(DirectIndexApplication.class, args);
+	}
+
+	@Bean
+	public GroupedOpenApi publicApi() {
+		return GroupedOpenApi.builder()
+			.group("D-Net DirectIndex API")
+			.pathsToMatch("/api/**")
+			.build();
+	}
+
+	@Bean
+	public ISLookUpService lookupServiceStub() {
+		return ISLookupClientFactory.getLookUpService(isLookupUrl);
+	}
+
+	@Bean
+	public VelocityEngine velocityEngine() {
+		final Properties props = new Properties();
+		props.setProperty("resource.loader", "class");
+		props.setProperty("class.resource.loader.class", "org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader");
+
+		final VelocityEngine ve = new VelocityEngine();
+		ve.init(props);
+		return ve;
+	}
+
+	@Override
+	protected String swaggerTitle() {
+		return "OpenAIRE DirectIndex API";
+	}
+
+}

apps/dnet-directindex-api/src/main/java/eu/dnetlib/openaire/directindex/DirectIndexController.java

@@ -0,0 +1,96 @@
+package eu.dnetlib.openaire.directindex;
+
+import java.time.OffsetDateTime;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.DeleteMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import eu.dnetlib.common.controller.AbstractDnetController;
+import eu.dnetlib.openaire.directindex.input.ResultEntry;
+import eu.dnetlib.openaire.directindex.mapping.XmlRecordConverter;
+import eu.dnetlib.openaire.directindex.repo.RecordInfo;
+import eu.dnetlib.openaire.directindex.repo.RecordInfoRepository;
+
+@RestController
+@RequestMapping("/api/results")
+public class DirectIndexController extends AbstractDnetController {
+
+	private static final Log log = LogFactory.getLog(DirectIndexController.class);
+
+	@Autowired
+	private RecordInfoRepository recordInfoRepository;
+
+	@Autowired
+	private XmlRecordConverter xmlRecordConverter;
+
+	@PostMapping("/")
+	public String feedResult(@RequestBody final ResultEntry r) throws DirectIndexApiException {
+		log.debug(r);
+
+		final RecordInfo info = new RecordInfo();
+		if (StringUtils.isNotBlank(r.getOpenaireId())) {
+			if (r.getOpenaireId().matches("^\\w{12}::\\w{32}$")) {
+				info.setId(r.getOpenaireId());
+				info.setOperation("UPDATE");
+			} else {
+				throw new DirectIndexApiException("Invalid openaireId: " + r.getOpenaireId() + " - regex ^\\w{12}::\\w{32}$ not matched");
+			}
+		} else if (StringUtils.isNoneBlank(r.getOriginalId(), r.getCollectedFromId())) {
+			final String openaireId = xmlRecordConverter.calculateOpenaireId(r.getOriginalId(), r.getCollectedFromId());
+			r.setOpenaireId(openaireId);
+			info.setId(openaireId);
+			info.setOperation("INSERT");
+		} else {
+			throw new DirectIndexApiException("Missing identifier fields: [openaireId] or [originalId, collectedFromId]");
+		}
+		if (StringUtils.isBlank(r.getTitle())) { throw new DirectIndexApiException("A required field is missing: title"); }
+		if (StringUtils.isBlank(r.getUrl())) { throw new DirectIndexApiException("A required field is missing: url"); }
+		if (StringUtils.isBlank(r.getAccessRightCode())) { throw new DirectIndexApiException("A required field is missing: accessRightCode"); }
+		if (StringUtils.isBlank(r.getResourceType())) { throw new DirectIndexApiException("A required field is missing: resourceType"); }
+		if (StringUtils.isBlank(r.getCollectedFromId())) { throw new DirectIndexApiException("A required field is missing: collectedFromId"); }
+		if (StringUtils.isBlank(r.getType())) { throw new DirectIndexApiException("A required field is missing: type"); }
+
+		info.setBody(r.toJson());
+		info.setType(r.getType());
+		info.setCreatedBy("TODO"); // TODO
+		info.setCreationDate(OffsetDateTime.now());
+		info.setExecutionDate(null);
+
+		recordInfoRepository.save(info);
+
+		return info.getId();
+
+	}
+
+	@DeleteMapping("/")
+	public void deleteResultWithOriginalId(@RequestParam final String originalId, @RequestParam final String collectedFromId) throws Exception {
+		deleteResult(xmlRecordConverter.calculateOpenaireId(originalId, collectedFromId));
+	}
+
+	@DeleteMapping("/{openaireId}")
+	public void deleteResultWithOpenaireId(@PathVariable final String openaireId) throws DirectIndexApiException {
+		deleteResult(openaireId);
+	}
+
+	private void deleteResult(final String openaireId) throws DirectIndexApiException {
+		final RecordInfo info = new RecordInfo();
+
+		info.setId(openaireId);
+		info.setOperation("DELETE");
+		info.setCreatedBy("TODO"); // TODO
+		info.setCreationDate(OffsetDateTime.now());
+		info.setExecutionDate(null);
+
+		recordInfoRepository.save(info);
+	}
+
+}

apps/dnet-directindex-api/src/main/java/eu/dnetlib/openaire/directindex/input/DatasourceEntry.java

@@ -0,0 +1,54 @@
+package eu.dnetlib.openaire.directindex.input;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * Created by michele on 02/12/15.
+ */
+public class DatasourceEntry {
+
+	private String id;
+	private String name;
+	private String prefix;
+
+	public DatasourceEntry() {}
+
+	public DatasourceEntry(final String id, final String name, final String prefix) {
+		this.id = id;
+		this.name = name;
+		this.prefix = prefix;
+	}
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getName() {
+		return name;
+	}
+
+	public void setName(final String name) {
+		this.name = name;
+	}
+
+	public String getPrefix() {
+		return prefix;
+	}
+
+	public void setPrefix(final String prefix) {
+		this.prefix = prefix;
+	}
+
+	public String calculateOpenaireId() {
+		if (StringUtils.isNotBlank(id)) {
+			final String[] arr = id.split("::");
+			if (arr.length == 2) { return String.format("%s::%s", arr[0], DigestUtils.md5Hex(arr[1])); }
+		}
+		return "";
+	}
+}

apps/dnet-directindex-api/src/main/java/eu/dnetlib/openaire/directindex/input/PidEntry.java

@@ -0,0 +1,38 @@
+package eu.dnetlib.openaire.directindex.input;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+
+/**
+ * Created by michele on 02/12/15.
+ */
+public class PidEntry {
+
+	@Schema(required = true, description = "E.g. doi, pmc, urn. See http://api.openaire.eu/vocabularies/dnet:pid_types")
+	private String type;
+
+	@Schema(required = true)
+	private String value;
+
+	public PidEntry() {}
+
+	public PidEntry(final String type, final String value) {
+		this.type = type;
+		this.value = value;
+	}
+
+	public String getType() {
+		return type;
+	}
+
+	public void setType(final String type) {
+		this.type = type;
+	}
+
+	public String getValue() {
+		return value;
+	}
+
+	public void setValue(final String value) {
+		this.value = value;
+	}
+}

apps/dnet-directindex-api/src/main/java/eu/dnetlib/openaire/directindex/input/ResultEntry.java

@@ -0,0 +1,234 @@
+package eu.dnetlib.openaire.directindex.input;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import eu.dnetlib.openaire.directindex.DirectIndexApiException;
+import io.swagger.v3.oas.annotations.media.Schema;
+
+/**
+ * Created by michele on 02/12/15.
+ */
+public class ResultEntry {
+
+	private String openaireId;
+	private String originalId;
+
+	@Schema(required = true)
+	private String title;
+	private List<String> authors = new ArrayList<>();
+	private String publisher;
+	private String description;
+	@Schema(description = "ISO Alpha-3 code. E.g. 'eng', 'ita'")
+	private String language;
+	private List<PidEntry> pids = new ArrayList<>();
+
+	@Schema(required = false, allowableValues = {
+		"OPEN", "CLOSED", "RESTRICTED", "EMBARGO", "UNKNOWN", "OTHER", "OPEN SOURCE"
+	})
+	private String accessRightCode;
+	private String embargoEndDate;
+
+	/**
+	 * One of publication, dataset, software, other. Default value is publication.
+	 */
+	@Schema(allowableValues = {
+		"publication", "dataset", "software", "other"
+	})
+	private String type = "publication";
+
+	@Schema(required = true, description = "Use 001 for articles, 021 for datasets, 0029 for software. See: http://api.openaire.eu/vocabularies/dnet:publication_resource for all the available resource types.")
+	private String resourceType;
+
+	@Schema(required = true)
+	private String url;
+
+	@Schema(required = true, description = "Use opendoar___::2659 for Zenodo Publications; re3data_____::r3d100010468 for Zenodo datasets; infrastruct::openaire for OpenAIRE portal.")
+	private String collectedFromId;
+	private String hostedById;
+
+	// String according to the EGI context profile, example: egi::classification::natsc::math
+
+	@Schema(description = "E.g. fet, egi::classification::natsc::math::pure, egi::projects::EMI")
+	private List<String> contexts = new ArrayList<>();
+
+	// String according to openaire guidelines:
+	// info:eu-repo/grantAgreement/Funder/FundingProgram/ProjectID/[Jurisdiction]/[ProjectName]/[ProjectAcronym]
+	@Schema(description = "E.g. info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus")
+	private List<String> linksToProjects = new ArrayList<>();
+
+	private static final Log log = LogFactory.getLog(ResultEntry.class);
+
+	public ResultEntry() {}
+
+	public String getOpenaireId() {
+		return openaireId;
+	}
+
+	public void setOpenaireId(final String openaireId) {
+		this.openaireId = openaireId;
+	}
+
+	public String getOriginalId() {
+		return originalId;
+	}
+
+	public void setOriginalId(final String originalId) {
+		this.originalId = originalId;
+	}
+
+	public String getTitle() {
+		return title;
+	}
+
+	public void setTitle(final String title) {
+		this.title = title;
+	}
+
+	public List<String> getAuthors() {
+		return authors;
+	}
+
+	public void setAuthors(final List<String> authors) {
+		this.authors = authors;
+	}
+
+	public String getPublisher() {
+		return publisher;
+	}
+
+	public void setPublisher(final String publisher) {
+		this.publisher = publisher;
+	}
+
+	public String getDescription() {
+		return description;
+	}
+
+	public void setDescription(final String description) {
+		this.description = description;
+	}
+
+	public String getLanguage() {
+		return language;
+	}
+
+	public void setLanguage(final String language) {
+		this.language = language;
+	}
+
+	public List<PidEntry> getPids() {
+		return pids;
+	}
+
+	public void setPids(final List<PidEntry> pids) {
+		this.pids = pids;
+	}
+
+	/**
+	 * Set required = true when the deprecated licenseCode is not used anymore by our client and it is removed
+	 *
+	 * @return access rights code
+	 */
+	public String getAccessRightCode() {
+		return accessRightCode;
+	}
+
+	public void setAccessRightCode(final String accessRightCode) {
+		this.accessRightCode = accessRightCode;
+	}
+
+	public String getResourceType() {
+		return resourceType;
+	}
+
+	public void setResourceType(final String resourceType) {
+		this.resourceType = resourceType;
+	}
+
+	public String getUrl() {
+		return url;
+	}
+
+	public void setUrl(final String url) {
+		this.url = url;
+	}
+
+	public String getCollectedFromId() {
+		return collectedFromId;
+	}
+
+	public void setCollectedFromId(final String collectedFromId) {
+		this.collectedFromId = collectedFromId;
+	}
+
+	public String getHostedById() {
+		return hostedById;
+	}
+
+	public void setHostedById(final String hostedById) {
+		this.hostedById = hostedById;
+	}
+
+	public List<String> getContexts() {
+		return contexts;
+	}
+
+	public void setContexts(final List<String> contexts) {
+		this.contexts = contexts;
+	}
+
+	public List<String> getLinksToProjects() {
+		return linksToProjects;
+	}
+
+	public void setLinksToProjects(final List<String> linksToProjects) {
+		this.linksToProjects = linksToProjects;
+	}
+
+	public String getType() {
+		return type;
+	}
+
+	public void setType(final String type) {
+		this.type = type;
+	}
+
+	public String getEmbargoEndDate() {
+		return embargoEndDate;
+	}
+
+	public void setEmbargoEndDate(final String embargoEndDate) {
+		this.embargoEndDate = embargoEndDate;
+	}
+
+	@Override
+	public String toString() {
+		try {
+			return toJson();
+		} catch (final DirectIndexApiException e) {
+			log.error("Error converting object in json", e);
+			throw new RuntimeException("Error converting object in json", e);
+		}
+	}
+
+	public String getAnyId() {
+		return StringUtils.isNotBlank(openaireId) ? openaireId : originalId;
+	}
+
+	public String toJson() throws DirectIndexApiException {
+		try {
+			return new ObjectMapper().writeValueAsString(this);
+		} catch (final JsonProcessingException e) {
+			log.error("Error converting object in json", e);
+			throw new DirectIndexApiException("Error converting object in json", e);
+		}
+	}
+}

apps/dnet-directindex-api/src/main/java/eu/dnetlib/openaire/directindex/input/ZenodoContextList.java

@@ -0,0 +1,32 @@
+package eu.dnetlib.openaire.directindex.input;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class ZenodoContextList {
+
+	private String zenodoid;
+
+	private List<String> openAirecommunitylist;
+
+	public List<String> getOpenAirecommunitylist() {
+		return openAirecommunitylist;
+	}
+
+	public void setOpenAirecommunitylist(final List<String> openAirecommunitylist) {
+		this.openAirecommunitylist = openAirecommunitylist;
+
+	}
+
+	public String getZenodoid() {
+		return zenodoid;
+	}
+
+	public void setZenodoid(final String zenodoid) {
+		this.zenodoid = zenodoid;
+
+	}
+
+}

apps/dnet-directindex-api/src/main/java/eu/dnetlib/openaire/directindex/is/ISLookupClient.java

@@ -0,0 +1,126 @@
+package eu.dnetlib.openaire.directindex.is;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.cache.annotation.CacheEvict;
+import org.springframework.cache.annotation.Cacheable;
+import org.springframework.stereotype.Component;
+
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+import eu.dnetlib.openaire.directindex.DirectIndexApiException;
+import eu.dnetlib.openaire.directindex.input.DatasourceEntry;
+
+@Component
+public class ISLookupClient {
+
+	private static final Log log = LogFactory.getLog(ISLookupClient.class);
+
+	@Autowired
+	private ISLookUpService lookupService;
+
+	@Cacheable("indexDsInfo")
+	public IndexDsInfo currentIndexDsInfo() {
+		try {
+			log.info("Not using cache");
+
+			final String queryUrl = IOUtils.toString(getClass().getResourceAsStream("/xquery/findSolrIndexUrl.xquery"));
+			final String queryDs = IOUtils.toString(getClass().getResourceAsStream("/xquery/findIndexDsInfo.xquery"));
+
+			final String indexBaseUrl = findOne(queryUrl);
+			final String idxDs = findOne(queryDs);
+
+			if (idxDs.isEmpty()) { throw new IllegalStateException(queryDs + "\n\nreturned no results, check IS profiles"); }
+
+			final String[] arr = idxDs.split("@@@");
+			return new IndexDsInfo(indexBaseUrl, arr[0].trim(), arr[1].trim(), arr[2].trim());
+		} catch (final Exception e) {
+			log.error(e.getMessage());
+			throw new RuntimeException(e);
+		}
+	}
+
+	@Cacheable("datasources")
+	public DatasourceEntry findDatasource(final String dsId) throws DirectIndexApiException {
+		final String query =
+			"collection('/db/DRIVER/RepositoryServiceResources/RepositoryServiceResourceType')//CONFIGURATION[./DATASOURCE_ORIGINAL_ID='" + dsId
+				+ "']/concat(./OFFICIAL_NAME, ' @@@ ', .//FIELD/value[../key='NamespacePrefix'])";
+		final String s = findOne(query);
+		final String[] arr = s.split("@@@");
+
+		final DatasourceEntry ds = new DatasourceEntry(dsId, arr[0].trim(), arr[1].trim());
+
+		if (StringUtils.isBlank(ds.getName()) || StringUtils.isBlank(ds.getPrefix())) {
+			log.error("Invalid datasource id: " + dsId);
+			throw new DirectIndexApiException("Invalid datasource id: " + dsId);
+		} else {
+			return ds;
+		}
+	}
+
+	@Cacheable("vocabularies")
+	public Map<String, String> findVocabulary(final String voc) throws DirectIndexApiException {
+
+		final String query = "collection('/db/DRIVER/VocabularyDSResources/VocabularyDSResourceType')[.//VOCABULARY_NAME/@code='" + voc
+			+ "']//TERM/concat(@code, ' @@@ ', @english_name)";
+
+		final Map<String, String> map = new HashMap<>();
+		for (final String s : find(query)) {
+			final String[] arr = s.split("@@@");
+			map.put(arr[0].trim(), arr[1].trim());
+		}
+
+		return map;
+	}
+
+	@Cacheable("contexts")
+	public Map<String, String> findContexts() throws DirectIndexApiException {
+		final String query =
+			"collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')[.//context/@type='community' or .//context/@type='ri']//*[name()='context' or name()='category' or name()='concept']/concat(@id, ' @@@ ', @label)";
+
+		final Map<String, String> map = new HashMap<>();
+		for (final String s : find(query)) {
+			final String[] arr = s.split("@@@");
+			map.put(arr[0].trim(), arr[1].trim());
+		}
+		return map;
+	}
+
+	@Cacheable("layouts")
+	public String findLayoutForFormat(final String format) throws Exception {
+		return findOne("collection('/db/DRIVER/MDFormatDSResources/MDFormatDSResourceType')[.//NAME='" + format + "']//LAYOUT[@name='index']");
+	}
+
+	@CacheEvict(allEntries = true, value = {
+		"indexDsInfo", "datasources", "vocabularies", "contexts", "layouts"
+	})
+	public void evictCache() {
+		log.info("Evicting indexDsInfo cache");
+	}
+
+	private String findOne(final String query) throws DirectIndexApiException {
+		try {
+			return lookupService.getResourceProfileByQuery(query);
+		} catch (final ISLookUpException e) {
+			log.error("Error executing xquery: " + query, e);
+			throw new DirectIndexApiException("Error executing xquery: " + query, e);
+		}
+	}
+
+	private List<String> find(final String query) throws DirectIndexApiException {
+		try {
+			return lookupService.quickSearchProfile(query);
+		} catch (final ISLookUpException e) {
+			log.error("Error executing xquery: " + query, e);
+			throw new DirectIndexApiException("Error executing xquery: " + query, e);
+		}
+	}
+
+}

apps/dnet-directindex-api/src/main/java/eu/dnetlib/openaire/directindex/is/IndexDsInfo.java

@@ -0,0 +1,44 @@
+package eu.dnetlib.openaire.directindex.is;
+
+public class IndexDsInfo {
+
+    private final String indexBaseUrl;
+    private final String indexDsId;
+    private final String format;
+    private final String coll;
+
+    public IndexDsInfo(final String indexBaseUrl, final String indexDsId, final String format, final String coll) {
+        this.indexBaseUrl = indexBaseUrl;
+        this.indexDsId = indexDsId;
+        this.format = format;
+        this.coll = coll;
+    }
+
+    public String getIndexBaseUrl() {
+        return indexBaseUrl;
+    }
+
+    public String getIndexDsId() {
+        return indexDsId;
+    }
+
+    public String getFormat() {
+        return format;
+    }
+
+    public String getColl() {
+        return coll;
+    }
+
+    @Override
+    public int hashCode() {
+        return getColl().hashCode();
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (!(other instanceof IndexDsInfo)) return false;
+
+        return getColl().equals(((IndexDsInfo) other).getColl());
+    }
+}

apps/dnet-directindex-api/src/main/java/eu/dnetlib/openaire/directindex/mapping/OpenAIRESubmitterUtils.java

@@ -0,0 +1,293 @@
+package eu.dnetlib.openaire.directindex.mapping;
+
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.web.client.RestClientException;
+import org.springframework.web.client.RestTemplate;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+
+import eu.dnetlib.openaire.directindex.input.ZenodoContextList;
+
+public class OpenAIRESubmitterUtils {
+
+	private static final org.apache.commons.logging.Log log = org.apache.commons.logging.LogFactory.getLog(OpenAIRESubmitterUtils.class);
+	private static final String ZENODO_COMMUNITY = "zenodo.org/communities/";
+	private final String community_api;
+
+	public OpenAIRESubmitterUtils(final String community_api) {
+		this.community_api = community_api;
+	}
+
+	public Map<String, String> calculateProjectInfo(final String link) {
+		final Map<String, String> info = new HashMap<>();
+		final String[] arr = link.split("/");
+		// info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble
+
+		if (arr.length > 4) {
+			final String acronym = arr.length > 7 ? arr[7] : "";
+			final String title = arr.length > 6 ? StringUtils.isNotBlank(arr[6]) ? arr[6] : acronym : "";
+			final String jurisdiction = arr.length > 5 ? arr[5] : "";
+			final String funderId = calculateFunderId(arr[2], arr[3]);
+			info.put("id", calculateProjectId(arr[2], arr[3], arr[4]));
+			info.put("funderShortName", fixFunderShortName(arr[2]));
+			info.put("fundingName", arr[3]);
+			info.put("code", unescape(arr[4]));
+			info.put("jurisdiction", jurisdiction);
+			info.put("title", title);
+			info.put("acronym", acronym);
+			info.put("funderId", funderId);
+			info.put("funderName", calculateFunderName(arr[2]));
+			if (StringUtils.isNotBlank(arr[3])) {
+				info.put("fundingId", funderId + "::" + arr[3]);
+			}
+		}
+		return info;
+	}
+
+	// TODO: remove me when Zenodo ingests the good UKRI projects
+	protected String fixFunderShortName(final String funderShortName) {
+		switch (funderShortName) {
+		case "RCUK":
+			return "UKRI";
+		default:
+			return funderShortName;
+		}
+	}
+
+	protected String calculateFunderPrefix(final String funderShortName, final String funding) {
+		switch (funderShortName.toLowerCase()) {
+		case "conicyt":
+			return "conicytf____::";
+		case "dfg":
+			return "dfgf________::";
+		case "ec":
+			if (funding.equalsIgnoreCase("fp7")) {
+				return "corda_______::";
+			} else {
+				return "corda__h2020::";
+			}
+		case "eea":
+			return "euenvagency_::";
+		case "hrzz":
+		case "mzos":
+			return "irb_hr______::";
+		case "tara":
+			return "taraexp_____::";
+		case "tubitak":
+			return "tubitakf____::";
+		case "rcuk":
+			return "ukri________::";
+		default:
+			String prefix = funderShortName.toLowerCase();
+			// ensure we have 12 chars
+			while (prefix.length() < 12) {
+				prefix += "_";
+			}
+			return prefix + "::";
+		}
+	}
+
+	protected String calculateProjectId(final String funderShortName, final String funding, final String code) {
+		final String suffix = DigestUtils.md5Hex(unescape(code));
+		final String funderPrefix = calculateFunderPrefix(funderShortName, funding);
+		return funderPrefix + suffix;
+	}
+
+	private String unescape(final String code) {
+		return StringUtils.replaceChars(code, "%2F", "/");
+	}
+
+	protected String calculateFunderId(final String funderShortName, final String funding) {
+		switch (funderShortName.toLowerCase()) {
+		case "ec":
+			return "ec__________::EC";
+		default:
+			final String fixedFunderShortName = fixFunderShortName(funderShortName);
+			final String prefix = calculateFunderPrefix(fixedFunderShortName, funding);
+			return prefix + fixedFunderShortName.toUpperCase();
+		}
+	}
+
+	protected String calculateFunderName(final String funderShortName) {
+
+		switch (funderShortName.toLowerCase()) {
+		case "aff":
+		case "aka":
+			return "Academy of Finland";
+		case "anr":
+			return "French National Research Agency (ANR)";
+		case "arc":
+			return "Australian Research Council (ARC)";
+		case "cihr":
+			return "Canadian Institutes of Health Research";
+		case "conicyt":
+			return "Comisión Nacional de Investigación Científica y Tecnológica";
+		case "dfg":
+			return "Deutsche Forschungsgemeinschaft";
+		case "ec":
+			return "European Commission";
+		case "eea":
+			return "European Environment Agency";
+		case "fct":
+			return "Fundação para a Ciência e a Tecnologia, I.P.";
+		case "fwf":
+			return "Austrian Science Fund (FWF)";
+		case "gsrt":
+			return "General Secretariat of Research and Technology (GSRT)";
+		case "hrzz":
+			return "Croatian Science Foundation (CSF)";
+		case "innoviris":
+			return "INNOVIRIS";
+		case "mestd":
+			return "Ministry of Education, Science and Technological Development of Republic of Serbia";
+		case "miur":
+			return "Ministero dell'Istruzione dell'Università e della Ricerca";
+		case "mzos":
+			return "Ministry of Science, Education and Sports of the Republic of Croatia (MSES)";
+		case "nhmrc":
+			return "National Health and Medical Research Council (NHMRC)";
+		case "nih":
+			return "National Institutes of Health";
+		case "nsf":
+			return "National Science Foundation";
+		case "nserc":
+			return "Natural Sciences and Engineering Research Council of Canada";
+		case "nwo":
+			return "Netherlands Organisation for Scientific Research (NWO)";
+		case "rcuk":
+		case "ukri":
+			return "UK Research and Innovation";
+		case "rif":
+		case "rpf":
+			return "Research and Innovation Foundation";
+		case "rsf":
+			return "Russian Science Foundation";
+		case "sfi":
+			return "Science Foundation Ireland";
+		case "sgov":
+			return "Gobierno de España";
+		case "snsf":
+			return "Swiss National Science Foundation";
+		case "sshrc":
+			return "Social Sciences and Humanities Research Council";
+		case "tara":
+			return "Tara Expeditions Foundation";
+		case "tubitak":
+			return "Türkiye Bilimsel ve Teknolojik Araştırma Kurumu";
+		case "wt":
+			return "Wellcome Trust";
+		default:
+			log.error("Funder short name '" + funderShortName + "' not managed");
+			return "";
+		}
+	}
+
+	private List<String> filterContexts(List<String> contexts) {
+
+		final List<String> zenodoContexts = contexts.stream()
+			.map(c -> {
+				if (c.contains(ZENODO_COMMUNITY)) { return c.substring(c.lastIndexOf("/") + 1); }
+				return null;
+			})
+			.collect(Collectors.toList());
+
+		if (zenodoContexts.size() > 0) {
+			contexts = contexts.stream().filter(c -> !c.contains(ZENODO_COMMUNITY)).collect(Collectors.toList());
+			final RestTemplate rt = new RestTemplate();
+			final Set<String> zenodoOpenAIRE = new HashSet<>();
+			for (final String context : zenodoContexts) {
+				// String ct = context.substring(context.lastIndexOf("/")+1);
+				try {
+					zenodoOpenAIRE
+						.addAll(rt.getForObject(community_api + context + "/openairecommunities", ZenodoContextList.class).getOpenAirecommunitylist());
+				} catch (final RestClientException rce) {
+					log.error("Unable to get object for " + community_api + context + "/openairecommunities");
+				}
+
+			}
+			contexts.addAll(zenodoOpenAIRE);
+		}
+
+		return contexts;
+
+	}
+
+	public List<ContextInfo> processContexts(final List<String> list) {
+		// filterContexts(list);
+		return Lists.newArrayList(Lists.transform(filterContexts(list), new Function<String, ContextInfo>() {
+
+			@Override
+			public ContextInfo apply(final String s) {
+				return createContextInfo(s.split("::"), 0);
+			}
+
+			private ContextInfo createContextInfo(final String[] arr, final int pos) {
+				final StringWriter id = new StringWriter();
+				id.write(arr[0]);
+				for (int i = 0; i < pos; i++) {
+					id.write("::");
+					id.write(arr[i + 1]);
+				}
+				final String elem = pos == 0 ? "context" : pos == 1 ? "category" : "concept";
+				final ContextInfo info = new ContextInfo(elem, id.toString());
+				if (pos + 1 < arr.length) {
+					info.getChildren().add(createContextInfo(arr, pos + 1));
+				}
+				return info;
+			}
+		}));
+
+	}
+
+	public class ContextInfo {
+
+		private String elem;
+		private String id;
+		private List<ContextInfo> children = new ArrayList<>();
+
+		public ContextInfo(final String elem,
+			final String id) {
+			this.elem = elem;
+			this.id = id;
+		}
+
+		public String getElem() {
+			return elem;
+		}
+
+		public void setElem(final String elem) {
+			this.elem = elem;
+		}
+
+		public String getId() {
+			return id;
+		}
+
+		public void setId(final String id) {
+			this.id = id;
+		}
+
+		public List<ContextInfo> getChildren() {
+			return children;
+		}
+
+		public void setChildren(final List<ContextInfo> children) {
+			this.children = children;
+		}
+
+		public boolean isRoot() {
+			return elem.equals("context");
+		}
+	}
+}

apps/dnet-directindex-api/src/main/java/eu/dnetlib/openaire/directindex/mapping/XmlRecordConverter.java

@@ -0,0 +1,110 @@
+package eu.dnetlib.openaire.directindex.mapping;
+
+import java.io.StringWriter;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.function.Function;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.text.StringEscapeUtils;
+import org.apache.velocity.VelocityContext;
+import org.apache.velocity.app.VelocityEngine;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import eu.dnetlib.openaire.directindex.DirectIndexApiException;
+import eu.dnetlib.openaire.directindex.input.DatasourceEntry;
+import eu.dnetlib.openaire.directindex.input.ResultEntry;
+import eu.dnetlib.openaire.directindex.is.ISLookupClient;
+
+@Component
+public class XmlRecordConverter {
+
+	private static final DatasourceEntry UNKNOWN_REPO =
+		new DatasourceEntry("openaire____::1256f046-bf1f-4afc-8b47-d0b147148b18", "Unknown Repository", "unknown_____");
+
+	@Value("${openaire.api.community}")
+	private String community_api;
+
+	@Value("${oaf.schema.location}")
+	private String oafSchemaLocation;
+
+	@Autowired
+	private ISLookupClient isLookupClient;
+
+	@Autowired
+	private VelocityEngine velocityEngine;
+
+	public String convert(final ResultEntry r) throws DirectIndexApiException {
+
+		final DatasourceEntry collectedFromEntry = getDatasourceInfo(r.getCollectedFromId());
+		final DatasourceEntry hostedByEntry = getDatasourceInfo(r.getHostedById());
+
+		final VelocityContext vc = new VelocityContext();
+		vc.put("esc", (Function<String, String>) s -> StringEscapeUtils.escapeXml11(s));
+		vc.put("util", new OpenAIRESubmitterUtils(community_api));
+		vc.put("pub", r);
+		vc.put("objIdentifier", r.getOpenaireId());
+		vc.put("oafSchemaLocation", oafSchemaLocation);
+		vc.put("resultTypes", isLookupClient.findVocabulary("dnet:result_typologies"));
+		vc.put("rights", isLookupClient.findVocabulary("dnet:access_modes"));
+		vc.put("resourceTypes", isLookupClient.findVocabulary("dnet:publication_resource"));
+		vc.put("pidTypes", isLookupClient.findVocabulary("dnet:pid_types"));
+		vc.put("languages", isLookupClient.findVocabulary("dnet:languages"));
+		vc.put("contexts", isLookupClient.findContexts());
+		vc.put("dateOfCollection", new SimpleDateFormat("yyyy-MM-dd\'T\'hh:mm:ss\'Z\'").format(new Date()));
+		vc.put("collectedFrom", collectedFromEntry);
+		vc.put("hostedBy", hostedByEntry);
+
+		final StringWriter sw = new StringWriter();
+		velocityEngine.getTemplate("/templates/indexRecord.xml.vm", "UTF-8").merge(vc, sw);
+		return sw.toString();
+	}
+
+	public String calculateOpenaireId(final String originalId, final String collectedFromId) throws DirectIndexApiException {
+		return calculateOpenaireId(originalId, getDatasourceInfo(collectedFromId));
+	}
+
+	private String calculateOpenaireId(final String originalId, final DatasourceEntry collectedFromEntry) {
+		return collectedFromEntry.getPrefix() + "::" + DigestUtils.md5Hex(originalId);
+	}
+
+	private synchronized DatasourceEntry getDatasourceInfo(final String dsId) throws DirectIndexApiException {
+		return StringUtils.isNotBlank(dsId) ? isLookupClient.findDatasource(dsId) : UNKNOWN_REPO;
+	}
+
+	protected String getCommunity_api() {
+		return community_api;
+	}
+
+	protected void setCommunity_api(final String community_api) {
+		this.community_api = community_api;
+	}
+
+	protected String getOafSchemaLocation() {
+		return oafSchemaLocation;
+	}
+
+	protected void setOafSchemaLocation(final String oafSchemaLocation) {
+		this.oafSchemaLocation = oafSchemaLocation;
+	}
+
+	protected ISLookupClient getIsLookupClient() {
+		return isLookupClient;
+	}
+
+	protected void setIsLookupClient(final ISLookupClient isLookupClient) {
+		this.isLookupClient = isLookupClient;
+	}
+
+	protected VelocityEngine getVelocityEngine() {
+		return velocityEngine;
+	}
+
+	protected void setVelocityEngine(final VelocityEngine velocityEngine) {
+		this.velocityEngine = velocityEngine;
+	}
+
+}

apps/dnet-directindex-api/src/main/java/eu/dnetlib/openaire/directindex/repo/RecordInfo.java

@@ -0,0 +1,95 @@
+package eu.dnetlib.openaire.directindex.repo;
+
+import java.time.OffsetDateTime;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+@Entity
+@Table(name = "records")
+public class RecordInfo {
+
+	@Id
+	@Column(name = "id")
+	private String id;
+
+	@Column(name = "type")
+	private String type;
+
+	/**
+	 * One of: DELETE, INSERT, UPDATE
+	 */
+	@Column(name = "operation")
+	private String operation;
+
+	@Column(name = "body")
+	private String body;
+
+	@Column(name = "created_by")
+	private String createdBy;
+
+	@Column(name = "creation_date")
+	private OffsetDateTime creationDate;
+
+	@Column(name = "execution_date")
+	private OffsetDateTime executionDate;
+
+	public String getId() {
+		return id;
+	}
+
+	public void setId(final String id) {
+		this.id = id;
+	}
+
+	public String getType() {
+		return type;
+	}
+
+	public void setType(final String type) {
+		this.type = type;
+	}
+
+	public String getOperation() {
+		return operation;
+	}
+
+	public void setOperation(final String operation) {
+		this.operation = operation;
+	}
+
+	public String getBody() {
+		return body;
+	}
+
+	public void setBody(final String body) {
+		this.body = body;
+	}
+
+	public String getCreatedBy() {
+		return createdBy;
+	}
+
+	public void setCreatedBy(final String createdBy) {
+		this.createdBy = createdBy;
+	}
+
+	public OffsetDateTime getCreationDate() {
+		return creationDate;
+	}
+
+	public void setCreationDate(final OffsetDateTime creationDate) {
+		this.creationDate = creationDate;
+	}
+
+	public OffsetDateTime getExecutionDate() {
+		return executionDate;
+	}
+
+	public void setExecutionDate(final OffsetDateTime executionDate) {
+		this.executionDate = executionDate;
+	}
+
+}

apps/dnet-directindex-api/src/main/java/eu/dnetlib/openaire/directindex/repo/RecordInfoRepository.java

@@ -0,0 +1,18 @@
+package eu.dnetlib.openaire.directindex.repo;
+
+import java.util.List;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Query;
+import org.springframework.stereotype.Repository;
+
+@Repository
+public interface RecordInfoRepository extends JpaRepository<RecordInfo, String> {
+
+	@Query(value = "select * from records where execution_date is null and (upper(operation) = 'INSERT' or upper(operation) = 'UPDATE')", nativeQuery = true)
+	List<RecordInfo> newRecords();
+
+	@Query(value = "select * from records where execution_date is null and upper(operation) = 'DELETE'", nativeQuery = true)
+	List<RecordInfo> toDeleteRecords();
+
+}

apps/dnet-directindex-api/src/main/java/eu/dnetlib/openaire/directindex/scheduled/ScheduledActions.java

@@ -0,0 +1,124 @@
+package eu.dnetlib.openaire.directindex.scheduled;
+
+import java.time.OffsetDateTime;
+import java.util.List;
+import java.util.Objects;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import eu.dnetlib.openaire.directindex.DirectIndexApiException;
+import eu.dnetlib.openaire.directindex.input.ResultEntry;
+import eu.dnetlib.openaire.directindex.is.ISLookupClient;
+import eu.dnetlib.openaire.directindex.is.IndexDsInfo;
+import eu.dnetlib.openaire.directindex.mapping.XmlRecordConverter;
+import eu.dnetlib.openaire.directindex.repo.RecordInfo;
+import eu.dnetlib.openaire.directindex.repo.RecordInfoRepository;
+import eu.dnetlib.openaire.directindex.solr.SolrIndexClient;
+import eu.dnetlib.openaire.directindex.solr.SolrIndexClientFactory;
+
+@Component
+@ConditionalOnProperty(value = "directindex.scheduling.enabled", havingValue = "true", matchIfMissing = false)
+public class ScheduledActions {
+
+	private static final Log log = LogFactory.getLog(ScheduledActions.class);
+
+	@Autowired
+	private ISLookupClient isLookupClient;
+
+	@Autowired
+	private SolrIndexClientFactory solrIndexClientFactory;
+
+	@Autowired
+	private XmlRecordConverter xmlRecordConverter;
+
+	@Autowired
+	private RecordInfoRepository recordInfoRepository;
+
+	@Scheduled(fixedDelay = 5 * 60 * 1000) // 5 minutes
+	public void indexNewRecords() throws DirectIndexApiException {
+		log.info("Indexing new records...");
+
+		final List<RecordInfo> list = recordInfoRepository.newRecords();
+
+		if (list.size() > 0) {
+			final IndexDsInfo info = isLookupClient.currentIndexDsInfo();
+			final SolrIndexClient solr = solrIndexClientFactory.getClient(info);
+
+			final ObjectMapper objectMapper = new ObjectMapper();
+
+			solr.addRecords(list.stream()
+				.map(RecordInfo::getBody)
+				.map(s -> {
+					try {
+						return objectMapper.readValue(s, ResultEntry.class);
+					} catch (final Exception e) {
+						log.error(e);
+						return null;
+					}
+				})
+				.filter(Objects::nonNull)
+				.map(t -> {
+					try {
+						return xmlRecordConverter.convert(t);
+					} catch (final DirectIndexApiException e) {
+						log.error("Error generating record for index: " + t, e);
+						return null;
+					}
+				})
+				.filter(StringUtils::isNotBlank));
+
+			solr.commit();
+
+			updateExecutionDate(list);
+		}
+
+		log.info(String.format("done (indexed records: %s)", list.size()));
+
+	}
+
+	@Scheduled(fixedDelay = 30 * 60 * 1000) // 30 minutes
+	public void deleteRecords() {
+		log.info("Deleting records from index...");
+
+		final List<RecordInfo> list = recordInfoRepository.toDeleteRecords();
+
+		if (list.size() > 0) {
+			final IndexDsInfo info = isLookupClient.currentIndexDsInfo();
+			final SolrIndexClient solr = solrIndexClientFactory.getClient(info);
+
+			list.stream().map(RecordInfo::getId).forEach(id -> {
+				try {
+					final String query = String.format("objidentifier:\"%s\" OR resultdupid:\"%s\"", id, id);
+					solr.deleteByQuery(query);
+				} catch (final DirectIndexApiException e) {
+					log.error(e);
+				}
+			});
+
+			updateExecutionDate(list);
+		}
+
+		log.info(String.format("done (deleted records: %s)", list.size()));
+	}
+
+	private void updateExecutionDate(final List<RecordInfo> list) {
+		final OffsetDateTime now = OffsetDateTime.now();
+		list.forEach(r -> r.setExecutionDate(now));
+		recordInfoRepository.saveAll(list);
+	}
+
+	@Scheduled(fixedDelay = 24 * 60 * 60 * 1000) // 24 hours
+	public void removeOldRecordsFromDB() {
+		// TODO
+
+	}
+
+}

apps/dnet-directindex-api/src/main/java/eu/dnetlib/openaire/directindex/solr/SolrIndexClient.java

@@ -0,0 +1,83 @@
+package eu.dnetlib.openaire.directindex.solr;
+
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.function.Function;
+import java.util.stream.Stream;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.common.SolrInputDocument;
+
+import eu.dnetlib.openaire.directindex.DirectIndexApiException;
+import eu.dnetlib.openaire.directindex.is.ISLookupClient;
+import eu.dnetlib.openaire.directindex.is.IndexDsInfo;
+
+public class SolrIndexClient {
+
+	private static final Log log = LogFactory.getLog(SolrIndexClient.class);
+
+	private final CloudSolrClient cloudSolrClient;
+	private final IndexDsInfo info;
+	private final ISLookupClient isLookupClient;
+
+	public SolrIndexClient(final CloudSolrClient cloudSolrClient, final IndexDsInfo info, final ISLookupClient isLookupClient) {
+		this.cloudSolrClient = cloudSolrClient;
+		this.info = info;
+		this.isLookupClient = isLookupClient;
+	}
+
+	public void commit() throws DirectIndexApiException {
+		try {
+			cloudSolrClient.commit();
+		} catch (SolrServerException | IOException e) {
+			throw new DirectIndexApiException(e);
+		}
+	}
+
+	public void deleteByQuery(final String query) throws DirectIndexApiException {
+		try {
+			cloudSolrClient.deleteByQuery(query);
+			cloudSolrClient.commit();
+		} catch (SolrServerException | IOException e) {
+			throw new DirectIndexApiException(e);
+		}
+	}
+
+	public void addRecords(final Stream<String> xmlRecords) throws DirectIndexApiException {
+		try {
+			final Function<String, String> transformer = XmlToIndexRecordFactory.newTransformerForMdFormat(info.getFormat(), isLookupClient);
+
+			final Iterator<SolrInputDocument> iter = xmlRecords
+				.map(xml -> prepareSolrDocument(xml, info.getIndexDsId(), transformer))
+				.iterator();
+
+			cloudSolrClient.add(iter);
+		} catch (final Throwable e) {
+			throw new DirectIndexApiException("Error creating solr document", e);
+		}
+	}
+
+	public void addRecords(final String... xmlRecords) throws Exception {
+		addRecords(Arrays.stream(xmlRecords));
+	}
+
+	protected SolrInputDocument prepareSolrDocument(final String xmlRecord, final String indexDsId, final Function<String, String> toIndexRecord) {
+
+		final String version = new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss'Z'").format(new Date());
+		final StreamingInputDocumentFactory documentFactory = new StreamingInputDocumentFactory(version, indexDsId);
+		final String indexRecord = toIndexRecord.apply(xmlRecord);
+		if (log.isDebugEnabled()) {
+			log.debug("***************************************\nSubmitting index record:\n" + indexRecord +
+				"\n***************************************\n");
+		}
+		return documentFactory.parseDocument(indexRecord);
+
+	}
+
+}

apps/dnet-directindex-api/src/main/java/eu/dnetlib/openaire/directindex/solr/SolrIndexClientFactory.java

@@ -0,0 +1,46 @@
+package eu.dnetlib.openaire.directindex.solr;
+
+import java.util.List;
+import java.util.Optional;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.Lists;
+
+import eu.dnetlib.openaire.directindex.is.ISLookupClient;
+import eu.dnetlib.openaire.directindex.is.IndexDsInfo;
+
+@Component
+public class SolrIndexClientFactory {
+
+	private static final Log log = LogFactory.getLog(SolrIndexClientFactory.class);
+
+	public static final String CHROOT_SEPARATOR = "/";
+
+	@Autowired
+	private ISLookupClient isLookupClient;
+
+	public SolrIndexClient getClient(final IndexDsInfo info) {
+		log.info(String.format("Initializing solr client (%s) with collection %s", info.getIndexBaseUrl(), info.getColl()));
+
+		// Example: quorum0:2182,quorum1:2182,quorum2:2182,quorum3:2182,quorum4:2182/solr-dev-openaire
+
+		final String s = StringUtils.substringAfterLast(info.getIndexBaseUrl(), CHROOT_SEPARATOR);
+		final String chroot = StringUtils.isNotBlank(s) ? CHROOT_SEPARATOR + s : null;
+		final String urls = chroot != null ? info.getIndexBaseUrl().replace(chroot, "") : info.getIndexBaseUrl();
+		final List<String> urlList = Lists.newArrayList(Splitter.on(",").omitEmptyStrings().split(urls));
+
+		final CloudSolrClient.Builder builder = new CloudSolrClient.Builder(urlList, Optional.of(chroot)).withParallelUpdates(true);
+		final CloudSolrClient cloudSolrClient = builder.build();
+		cloudSolrClient.setDefaultCollection(info.getColl());
+
+		return new SolrIndexClient(cloudSolrClient, info, isLookupClient);
+	}
+
+}

apps/dnet-directindex-api/src/main/java/eu/dnetlib/openaire/directindex/solr/StreamingInputDocumentFactory.java

@@ -0,0 +1,261 @@
+
+package eu.dnetlib.openaire.directindex.solr;
+
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+
+import javax.xml.stream.XMLEventFactory;
+import javax.xml.stream.XMLEventReader;
+import javax.xml.stream.XMLEventWriter;
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLOutputFactory;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.events.Namespace;
+import javax.xml.stream.events.StartElement;
+import javax.xml.stream.events.XMLEvent;
+
+import org.apache.solr.common.SolrInputDocument;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Optimized version of the document parser, drop in replacement of InputDocumentFactory.
+ * <p>
+ * Faster because:
+ * <ul>
+ * <li>Doesn't create a DOM for the full document
+ * <li>Doesn't execute xpaths agains the DOM
+ * <li>Quickly serialize the 'result' element directly in a string.
+ * <li>Uses less memory: less pressure on GC and allows more threads to process this in parallel
+ * </ul>
+ * <p>
+ * This class is fully reentrant and can be invoked in parallel.
+ *
+ * @author claudio
+ */
+public class StreamingInputDocumentFactory {
+
+	private static final String INDEX_FIELD_PREFIX = "__";
+
+	private static final String DS_VERSION = INDEX_FIELD_PREFIX + "dsversion";
+
+	private static final String DS_ID = INDEX_FIELD_PREFIX + "dsid";
+
+	private static final String RESULT = "result";
+
+	private static final String INDEX_RESULT = INDEX_FIELD_PREFIX + RESULT;
+
+	private static final String INDEX_RECORD_ID = INDEX_FIELD_PREFIX + "indexrecordidentifier";
+
+	private static final String DEFAULTDNETRESULT = "dnetResult";
+
+	private static final String TARGETFIELDS = "targetFields";
+
+	private static final String INDEX_RECORD_ID_ELEMENT = "indexRecordIdentifier";
+
+	private static final String ROOT_ELEMENT = "indexRecord";
+
+	private static final int MAX_FIELD_LENGTH = 25000;
+
+	private final ThreadLocal<XMLInputFactory> inputFactory = ThreadLocal
+		.withInitial(() -> XMLInputFactory.newInstance());
+
+	private final ThreadLocal<XMLOutputFactory> outputFactory = ThreadLocal
+		.withInitial(() -> XMLOutputFactory.newInstance());
+
+	private final ThreadLocal<XMLEventFactory> eventFactory = ThreadLocal
+		.withInitial(() -> XMLEventFactory.newInstance());
+
+	private final String version;
+
+	private final String dsId;
+
+	private String resultName = DEFAULTDNETRESULT;
+
+	public StreamingInputDocumentFactory(final String version, final String dsId) {
+		this(version, dsId, DEFAULTDNETRESULT);
+	}
+
+	public StreamingInputDocumentFactory(
+		final String version, final String dsId, final String resultName) {
+		this.version = version;
+		this.dsId = dsId;
+		this.resultName = resultName;
+	}
+
+	public SolrInputDocument parseDocument(final String inputDocument) {
+
+		final StringWriter results = new StringWriter();
+		final List<Namespace> nsList = Lists.newLinkedList();
+		try {
+
+			final XMLEventReader parser = inputFactory.get().createXMLEventReader(new StringReader(inputDocument));
+
+			final SolrInputDocument indexDocument = new SolrInputDocument(new HashMap<>());
+
+			while (parser.hasNext()) {
+				final XMLEvent event = parser.nextEvent();
+				if (event != null && event.isStartElement()) {
+					final String localName = event.asStartElement().getName().getLocalPart();
+
+					if (ROOT_ELEMENT.equals(localName)) {
+						nsList.addAll(getNamespaces(event));
+					} else if (INDEX_RECORD_ID_ELEMENT.equals(localName)) {
+						final XMLEvent text = parser.nextEvent();
+						final String recordId = getText(text);
+						indexDocument.addField(INDEX_RECORD_ID, recordId);
+					} else if (TARGETFIELDS.equals(localName)) {
+						parseTargetFields(indexDocument, parser);
+					} else if (resultName.equals(localName)) {
+						copyResult(indexDocument, results, parser, nsList, resultName);
+					}
+				}
+			}
+
+			if (version != null) {
+				indexDocument.addField(DS_VERSION, version);
+			}
+
+			if (dsId != null) {
+				indexDocument.addField(DS_ID, dsId);
+			}
+
+			if (!indexDocument.containsKey(INDEX_RECORD_ID)) { throw new IllegalStateException("cannot extract record ID from: " + inputDocument); }
+
+			return indexDocument;
+		} catch (final XMLStreamException e) {
+			throw new IllegalStateException(e);
+		}
+	}
+
+	private List<Namespace> getNamespaces(final XMLEvent event) {
+		final List<Namespace> res = Lists.newLinkedList();
+		@SuppressWarnings("unchecked")
+		final Iterator<Namespace> nsIter = event.asStartElement().getNamespaces();
+		while (nsIter.hasNext()) {
+			final Namespace ns = nsIter.next();
+			res.add(ns);
+		}
+		return res;
+	}
+
+	/**
+	 * Parse the targetFields block and add fields to the solr document.
+	 *
+	 * @param indexDocument
+	 * @param parser
+	 * @throws XMLStreamException
+	 */
+	protected void parseTargetFields(
+		final SolrInputDocument indexDocument,
+		final XMLEventReader parser)
+		throws XMLStreamException {
+
+		boolean hasFields = false;
+
+		while (parser.hasNext()) {
+			final XMLEvent targetEvent = parser.nextEvent();
+			if (targetEvent.isEndElement()
+				&& targetEvent.asEndElement().getName().getLocalPart().equals(TARGETFIELDS)) {
+				break;
+			}
+
+			if (targetEvent.isStartElement()) {
+				final String fieldName = targetEvent.asStartElement().getName().getLocalPart();
+				final XMLEvent text = parser.nextEvent();
+
+				final String data = getText(text);
+
+				addField(indexDocument, fieldName, data);
+				hasFields = true;
+			}
+		}
+
+		if (!hasFields) {
+			indexDocument.clear();
+		}
+	}
+
+	/**
+	 * Copy the /indexRecord/result element and children, preserving namespace declarations etc.
+	 *
+	 * @param indexDocument
+	 * @param results
+	 * @param parser
+	 * @param nsList
+	 * @throws XMLStreamException
+	 */
+	protected void copyResult(
+		final SolrInputDocument indexDocument,
+		final StringWriter results,
+		final XMLEventReader parser,
+		final List<Namespace> nsList,
+		final String dnetResult)
+		throws XMLStreamException {
+		final XMLEventWriter writer = outputFactory.get().createXMLEventWriter(results);
+
+		for (final Namespace ns : nsList) {
+			eventFactory.get().createNamespace(ns.getPrefix(), ns.getNamespaceURI());
+		}
+
+		final StartElement newRecord = eventFactory.get().createStartElement("", null, RESULT, null, nsList.iterator());
+
+		// new root record
+		writer.add(newRecord);
+
+		// copy the rest as it is
+		while (parser.hasNext()) {
+			final XMLEvent resultEvent = parser.nextEvent();
+
+			// TODO: replace with depth tracking instead of close tag tracking.
+			if (resultEvent.isEndElement()
+				&& resultEvent.asEndElement().getName().getLocalPart().equals(dnetResult)) {
+				writer.add(eventFactory.get().createEndElement("", null, RESULT));
+				break;
+			}
+
+			writer.add(resultEvent);
+		}
+		writer.close();
+		indexDocument.addField(INDEX_RESULT, results.toString());
+	}
+
+	/**
+	 * Helper used to add a field to a solr doc. It avoids to add empy fields
+	 *
+	 * @param indexDocument
+	 * @param field
+	 * @param value
+	 */
+	private final void addField(
+		final SolrInputDocument indexDocument,
+		final String field,
+		final String value) {
+		final String cleaned = value.trim();
+		if (!cleaned.isEmpty()) {
+			// log.info("\n\n adding field " + field.toLowerCase() + " value: " + cleaned + "\n");
+			indexDocument.addField(field.toLowerCase(), cleaned);
+		}
+	}
+
+	/**
+	 * Helper used to get the string from a text element.
+	 *
+	 * @param text
+	 * @return the
+	 */
+	protected final String getText(final XMLEvent text) {
+		if (text.isEndElement()) {
+			// text.asEndElement().getName().getLocalPart());
+			return "";
+		}
+
+		final String data = text.asCharacters().getData();
+		if (data != null && data.length() > MAX_FIELD_LENGTH) { return data.substring(0, MAX_FIELD_LENGTH); }
+
+		return data;
+	}
+}

apps/dnet-directindex-api/src/main/java/eu/dnetlib/openaire/directindex/solr/XmlToIndexRecordFactory.java

@@ -0,0 +1,48 @@
+package eu.dnetlib.openaire.directindex.solr;
+
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.util.function.Function;
+
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerConfigurationException;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.transform.stream.StreamSource;
+
+import org.apache.commons.io.IOUtils;
+
+import eu.dnetlib.openaire.directindex.is.ISLookupClient;
+
+public class XmlToIndexRecordFactory {
+
+	public static Function<String, String> newTransformerForMdFormat(final String format, final ISLookupClient isLookupClient) throws Exception {
+		final Transformer layoutTransformer =
+			newTransformer(IOUtils.toString(XmlToIndexRecordFactory.class.getResourceAsStream("/xslt/openaireLayoutToRecordStylesheet.xslt")));
+
+		layoutTransformer.setParameter("format", format);
+
+		final String xslt = transform(layoutTransformer, isLookupClient.findLayoutForFormat(format));
+
+		final Transformer recordTransformer = newTransformer(xslt);
+
+		return s -> transform(recordTransformer, s);
+	}
+
+	private static Transformer newTransformer(final String xslt) throws TransformerConfigurationException {
+		final TransformerFactory factory = TransformerFactory.newInstance();
+		return factory.newTransformer(new StreamSource(new StringReader(xslt)));
+	}
+
+	private static String transform(final Transformer t, final String xml) {
+		try {
+			final StreamResult res = new StreamResult(new StringWriter());
+			t.transform(new StreamSource(new StringReader(xml)), res);
+			return res.getWriter().toString();
+		} catch (final TransformerException e) {
+			throw new RuntimeException("Error transforming record: " + xml, e);
+		}
+	}
+
+}

apps/dnet-directindex-api/src/main/resources/application.properties

@@ -0,0 +1,34 @@
+server.port=8080
+
+spring.profiles.active=dev
+
+maven.pom.path = /META-INF/maven/eu.dnetlib.dhp/dnet-directindex-api/effective-pom.xml
+
+spring.main.banner-mode = off
+
+logging.level.root      = INFO
+#logging.level.org.springframework = DEBUG
+
+management.endpoints.web.exposure.include   =   prometheus,health
+management.endpoints.web.base-path                  =   /
+management.endpoints.web.path-mapping.prometheus    =   metrics
+management.endpoints.web.path-mapping.health        =   health
+
+spring.datasource.url=jdbc:postgresql://localhost:5432/oa_directindex
+spring.datasource.username=oa_directindex
+spring.datasource.password=
+
+# Hibernate ddl auto (create, create-drop, validate, update)
+spring.jpa.hibernate.ddl-auto = validate
+spring.jpa.properties.hibernate.dialect = org.hibernate.dialect.PostgreSQLDialect
+spring.jpa.properties.hibernate.hbm2dll.extra_physical_table_types = MATERIALIZED VIEW
+spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true
+spring.jpa.open-in-view=true
+spring.jpa.properties.hibernate.show_sql=false
+spring.jpa.properties.hibernate.use_sql_comments=false
+spring.jpa.properties.hibernate.format_sql=false
+
+directindex.scheduling.enabled = false
+openaire.api.community         = http://..../openaire/community/
+oaf.schema.location            = https://www.openaire.eu/schema/1.0/oaf-1.0.xsd
+openaire.service.islookup.wsdl = http://beta.services.openaire.eu:8280/is/services/isLookUp?wsdl

apps/dnet-directindex-api/src/main/resources/sql/schema.sql

@@ -0,0 +1,11 @@
+DROP TABLE IF EXISTS records;
+
+CREATE TABLE records(
+	id             text PRIMARY KEY,
+	type           text,
+	operation      text NOT NULL,
+	body           text,
+	created_by     text,
+	creation_date  timestamp with time zone NOT NULL DEFAULT now(),
+	execution_date timestamp with time zone
+);

apps/dnet-directindex-api/src/main/resources/templates/indexRecord.xml.vm

@@ -0,0 +1,135 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+#macro(context $ctx)
+        <$!ctx.elem id="$!esc.evaluate($!ctx.id)" label="$!esc.evaluate($contexts.get($!ctx.id))" #if($!ctx.isRoot()) type="community"#end>
+	#foreach($child in $!ctx.children)#context($child)#end
+        </$!ctx.elem>
+#end
+-->
+
+<record xmlns:dri="http://www.driver-repository.eu/namespace/dri" xmlns:oaf="http://namespace.openaire.eu/oaf">
+	<result>
+		<header>
+			<dri:objIdentifier>$!esc.evaluate($!objIdentifier)</dri:objIdentifier>
+			<dri:dateOfCollection>$!esc.evaluate($!dateOfCollection)</dri:dateOfCollection>
+			<dri:status>under curation</dri:status>
+			<counters/>
+		</header>
+		<metadata>
+			<oaf:entity schemaLocation="http://namespace.openaire.eu/oaf $!oafSchemaLocation">
+				<oaf:result>
+					<title classid="main title" classname="main title" schemeid="dnet:dataCite_title" schemename="dnet:dataCite_title">
+						$!esc.evaluate($!pub.title)
+					</title>
+					#foreach($author in $!pub.authors)
+                    	<creator rank="$velocityCount" name="" surname="">$!esc.evaluate($!author)</creator>
+                    #end
+					<dateofacceptance/>
+					<resulttype classid="$!esc.evaluate($!pub.type)" classname="$!esc.evaluate($!resultTypes.get($!pub.type))" schemeid="dnet:result_typologies" schemename="dnet:result_typologies"/>
+					<language classid="$!esc.evaluate($!pub.language)" classname="$!esc.evaluate($!languages.get($!pub.language))" schemeid="dnet:languages"
+					          schemename="dnet:languages"/>
+					<description>
+						$!esc.evaluate($!pub.description)
+					</description>
+					<country classid="" classname="" schemeid="" schemename=""/>
+					<subject classid="" classname="" schemeid="" schemename=""/>
+					<relevantdate classid="" classname="" schemeid="" schemename=""/>
+					<publisher>$!esc.evaluate($!pub.publisher)</publisher>
+					<embargoenddate>$!esc.evaluate($!pub.embargoEndDate)</embargoenddate>
+					<journal issn="" eissn="" lissn="" ep="" iss="" sp="" vol=""/>
+					<source/>
+					<fulltext/>
+					<format/>
+					<storagedate/>
+					<resourcetype classid="" classname="" schemeid="" schemename=""/>
+					<device/>
+					<size/>
+					<version/>
+					<lastmetadataupdate/>
+					<metadataversionnumber/>
+					<documentationUrl/>
+					<codeRepositoryUrl/>
+					<programmingLanguage classid="" classname="" schemeid="" schemename="" />
+					<contactperson />
+					<contactgroup />
+					<tool />
+					<originalId>$!esc.evaluate($!pub.originalId)</originalId>
+					<collectedfrom name="$!esc.evaluate($!collectedFrom.name)" id="$!esc.evaluate($!collectedFrom.calculateOpenaireId())"/>
+
+					#foreach($pid in $!pub.pids)
+						<pid classid="$!esc.evaluate($!pid.type)" classname="$!esc.evaluate($!pidTypes.get($!pid.type))" schemeid="dnet:pid_types"
+						     schemename="dnet:pid_types">$!esc.evaluate($pid.value)</pid>
+					#end
+					#if($!pub.accessRightCode)
+						<bestaccessright classid="$!esc.evaluate($!pub.accessRightCode)" classname="$!esc.evaluate($!rights.get($!pub.accessRightCode))"
+						                 schemeid="dnet:access_modes" schemename="dnet:access_modes"/>
+					#elseif($!pub.licenseCode)
+							<bestaccessright classid="$!esc.evaluate($!pub.licenseCode)" classname="$!esc.evaluate($!rights.get($!pub.licenseCode))"
+							                 schemeid="dnet:access_modes" schemename="dnet:access_modes"/>
+					#else
+						<bestaccessright classid="UNKNOWN" classname="not available" schemeid="dnet:access_modes" schemename="dnet:access_modes" />
+					#end
+
+					#foreach($ctx in $util.processContexts($!pub.contexts))
+						#context($ctx)
+					#end
+
+					<datainfo>
+						<inferred>false</inferred>
+						<deletedbyinference>false</deletedbyinference>
+						<trust>0.9</trust>
+						<inferenceprovenance/>
+						<provenanceaction classid="user:insert" classname="user:insert" schemeid="dnet:provenanceActions" schemename="dnet:provenanceActions"/>
+					</datainfo>
+
+					<rels>
+						#foreach($link in $!pub.linksToProjects)
+
+							#set( $info = $!util.calculateProjectInfo($!link) )
+							<rel inferred="false" trust="0.9" inferenceprovenance="" provenanceaction="user:claim">
+								<to class="isProducedBy" scheme="dnet:result_project_relations" type="project">$!esc.evaluate($!info.id)</to>
+								<code>$!esc.evaluate($!info.code)</code>
+								<acronym>$!esc.evaluate($!info.acronym)</acronym>
+								<title>$!esc.evaluate($!info.title)</title>
+								<contracttype classid="" classname="" schemeid="" schemename=""/>
+
+								<funding>
+									<funder id="$!esc.evaluate($!info.funderId)"
+									        shortname="$!esc.evaluate($!info.funderShortName)"
+									        name="$!esc.evaluate($!info.funderName)"
+									        jurisdiction="$!esc.evaluate($!info.jurisdiction)"/>
+									#if($!info.fundingId)
+									<funding_level_0 name="$!esc.evaluate($!info.fundingName)">$!esc.evaluate($!info.fundingId)</funding_level_0>
+									#end
+								</funding>
+								<websiteurl/>
+							</rel>
+						#end
+					</rels>
+					<children>
+						<instance id="$!esc.evaluate($!objIdentifier)">
+							<instancetype classid="$!esc.evaluate($!pub.resourceType)" classname="$!esc.evaluate($resourceTypes.get($!pub.resourceType))"
+							              schemeid="dnet:publication_resource" schemename="dnet:publication_resource"/>
+							<collectedfrom name="$!esc.evaluate($!collectedFrom.name)" id="$!esc.evaluate($!collectedFrom.calculateOpenaireId())"/>
+							<hostedby name="$!esc.evaluate($!hostedBy.name)" id="$!esc.evaluate($!hostedBy.calculateOpenaireId())"/>
+							#if($!pub.accessRightCode)
+								<accessright classid="$!esc.evaluate($!pub.accessRightCode)" classname="$!esc.evaluate($!rights.get($!pub.accessRightCode))"
+								                 schemeid="dnet:access_modes" schemename="dnet:access_modes"/>
+							#elseif($!pub.licenseCode)
+								<accessright classid="$!esc.evaluate($!pub.licenseCode)" classname="$!esc.evaluate($!rights.get($!pub.licenseCode))"
+								                 schemeid="dnet:access_modes" schemename="dnet:access_modes"/>
+							#else
+								<accessright classid="UNKNOWN" classname="not available" schemeid="dnet:access_modes" schemename="dnet:access_modes" />
+							#end
+							<dateofacceptance/>
+							<webresource>
+								<url>$!esc.evaluate($!pub.url)</url>
+							</webresource>
+						</instance>
+					</children>
+				</oaf:result>
+			</oaf:entity>
+		</metadata>
+	</result>
+</record>

apps/dnet-directindex-api/src/main/resources/xqueries/findIndexDsInfo.xquery

@@ -0,0 +1,11 @@
+distinct-values(for $s in collection('/db/DRIVER/ServiceResources/SearchServiceResourceType')//SERVICE_PROPERTIES[./PROPERTY[@key = 'infrastructure' and @value = 'public'] ]
+let $format := $s/PROPERTY[@key = "mdformat"]/@value/string()
+return(
+for $x in collection('/db/DRIVER/IndexDSResources/IndexDSResourceType')
+where
+    $x//METADATA_FORMAT =$format and
+            $x//METADATA_FORMAT_INTERPRETATION = 'openaire' and
+            $x//METADATA_FORMAT_LAYOUT = 'index'
+return
+    concat($x//RESOURCE_IDENTIFIER/@value/string(), ' @@@ ', $format , ' @@@ ', $format , '-index-openaire')
+))

apps/dnet-directindex-api/src/main/resources/xqueries/findSolrIndexUrl.xquery

@@ -0,0 +1 @@
+distinct-values(collection("/db/DRIVER/ServiceResources/IndexServiceResourceType")//PROTOCOL[@name = "solr" or @name = "SOLR"]/@address/string())

apps/dnet-directindex-api/src/main/resources/xslt/openaireLayoutToRecordStylesheet.xslt

@@ -0,0 +1,113 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet version="1.0"
+                xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:nxsl="http://www.w3.org/1999/XSL/TransformXX">
+
+	<xsl:output omit-xml-declaration="yes" method="xml"/>
+	<xsl:namespace-alias stylesheet-prefix="nxsl" result-prefix="xsl"/>
+
+	<xsl:param name="format"/>
+
+	<xsl:template match="/">
+		<xsl:apply-templates select="//LAYOUT"/>
+	</xsl:template>
+
+	<xsl:template match="LAYOUT">
+		<nxsl:stylesheet version="1.0"
+		                 xmlns:dnet="eu.dnetlib.miscutils.functional.xml.DnetXsltFunctions"
+		                 xmlns:dri="http://www.driver-repository.eu/namespace/dri"
+		                 exclude-result-prefixes="dnet">
+
+			<nxsl:output version="1.0" omit-xml-declaration="yes"
+			             method="xml"/>
+
+			<nxsl:variable name="format">
+				<xsl:value-of select="$format"/>
+			</nxsl:variable>
+
+			<nxsl:template match="/">
+				<indexRecord>
+					<indexRecordIdentifier>
+						<nxsl:value-of select="//dri:objIdentifier"/>
+					</indexRecordIdentifier>
+					<targetFields>
+						<nxsl:if test="count(//*[local-name()='metadata']/*) &gt; 0">
+							<xsl:apply-templates select="FIELDS/FIELD[@indexable='true']"/>
+						</nxsl:if>
+					</targetFields>
+					<dnetResult>
+						<nxsl:copy-of select="/*[local-name()='record']/*[local-name()='result']/*[local-name()='header']"/>
+						<metadata>
+							<xsl:apply-templates select="FIELDS/FIELD"
+							                     mode="result"/>
+						</metadata>
+					</dnetResult>
+				</indexRecord>
+			</nxsl:template>
+		</nxsl:stylesheet>
+	</xsl:template>
+
+	<xsl:template match="FIELD[@indexable='true']">
+		<xsl:choose>
+			<xsl:when test="@constant">
+				<xsl:element name="{@name}">
+					<xsl:value-of select="@constant"/>
+				</xsl:element>
+			</xsl:when>
+			<xsl:when test="@value and not(@xpath)">
+				<xsl:choose>
+					<xsl:when test="contains(@type, 'date')">
+						<nxsl:variable name="{@name}" select="dnet:normalizeDate(normalize-space({normalize-space(@value)}), false())"/>
+						<nxsl:if test="string-length(${@name}) &gt; 0">
+							<nxsl:element name="{@name}">
+								<nxsl:value-of select="${@name}"/>
+							</nxsl:element>
+						</nxsl:if>
+					</xsl:when>
+					<xsl:otherwise>
+						<nxsl:element name="{@name}">
+							<nxsl:value-of select="{@value}"/>
+						</nxsl:element>
+					</xsl:otherwise>
+				</xsl:choose>
+			</xsl:when>
+			<xsl:otherwise>
+				<xsl:variable name="value">
+					<xsl:choose>
+						<xsl:when test="@value">
+							<xsl:value-of select="@value"/>
+						</xsl:when>
+						<xsl:otherwise>
+							.
+						</xsl:otherwise>
+					</xsl:choose>
+				</xsl:variable>
+				<nxsl:for-each select="{@xpath}">
+					<xsl:element name="{@name}">
+						<xsl:choose>
+							<xsl:when test="@tokenizable='false'">
+								<nxsl:value-of select="normalize-space({normalize-space($value)})"/>
+							</xsl:when>
+							<xsl:otherwise>
+								<nxsl:value-of select="{normalize-space($value)}"/>
+							</xsl:otherwise>
+						</xsl:choose>
+					</xsl:element>
+				</nxsl:for-each>
+			</xsl:otherwise>
+		</xsl:choose>
+	</xsl:template>
+
+
+	<xsl:template match="FIELD" mode="result">
+		<xsl:if test="@result='true'">
+			<nxsl:copy-of select="{@xpath}"/>
+		</xsl:if>
+	</xsl:template>
+
+	<xsl:template match="FIELD" mode="header">
+		<xsl:if test="@header='true'">
+			<nxsl:copy-of select="{@xpath}"/>
+		</xsl:if>
+	</xsl:template>
+
+</xsl:stylesheet>

apps/dnet-directindex-api/src/test/java/eu/dnetlib/openaire/directindex/OpenAIRESubmitterUtilsTest.java

@@ -0,0 +1,87 @@
+package eu.dnetlib.openaire.directindex;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.junit.jupiter.api.Test;
+
+import eu.dnetlib.openaire.directindex.mapping.OpenAIRESubmitterUtils;
+
+/**
+ * Created by Alessia Bardi on 26/05/2017.
+ *
+ * @author Alessia Bardi
+ */
+public class OpenAIRESubmitterUtilsTest {
+
+	private final OpenAIRESubmitterUtils utils = new OpenAIRESubmitterUtils(
+		"http://beta.services.openaire.eu/openaire/community/");
+
+	final String fullProject = "info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble";
+	final String minimalProject = "info:eu-repo/grantAgreement/EC/FP7/244909///WorkAble";
+	final String onlyId = "info:eu-repo/grantAgreement/EC/FP7/244909/";
+	final String onlyTitle = "info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work";
+	final String escapedId = "info:eu-repo/grantAgreement/RCUK//NE%2FL003066%2F1/";
+
+	@Test
+	public void testCalculateProjectInfoEscaped() {
+		final Map<String, String> project = utils.calculateProjectInfo(escapedId);
+		print(project);
+	}
+
+	@Test
+	public void testCalculateProjectInfoFull() {
+		final Map<String, String> project = utils.calculateProjectInfo(fullProject);
+		print(project);
+	}
+
+	@Test
+	public void testCalculateProjectInfoOnlyId() {
+		final Map<String, String> project = utils.calculateProjectInfo(onlyId);
+		print(project);
+	}
+
+	@Test
+	public void testCalculateProjectInfoMinimalAcro() {
+		final Map<String, String> project = utils.calculateProjectInfo(minimalProject);
+		print(project);
+	}
+
+	@Test
+	public void testCalculateProjectInfoOnlyTitle() {
+		final Map<String, String> project = utils.calculateProjectInfo(onlyTitle);
+		print(project);
+	}
+
+	@Test
+	public void testJerusalem() {
+		final String s =
+			"info:eu-repo/grantAgreement/EC/FP7/337895/EU/Opening Jerusalem Archives: For a connected History of ‘Citadinité’ in the Holy City (1840-1940)/OPEN-JERUSALEM";
+		final Map<String, String> project = utils.calculateProjectInfo(s);
+		print(project);
+	}
+
+	private void print(final Map<String, String> map) {
+		for (final Entry<String, String> e : map.entrySet()) {
+			System.out.println(e.getKey() + " = " + e.getValue());
+		}
+	}
+
+	@Test
+	public void testContext() {
+		final List<String> contexts = new ArrayList<>();
+		contexts.add("https://zenodo.org/communities/dimpo");
+		contexts.add("https://zenodo.org/communities/aoo_beopen");
+		final List<OpenAIRESubmitterUtils.ContextInfo> tmp = utils.processContexts(contexts);
+
+		assertEquals(2, tmp.size());
+
+		assertTrue(tmp.get(0).getId().equals("dh-ch") || tmp.get(1).getId().equalsIgnoreCase("dh-ch"));
+		assertTrue(tmp.get(0).getId().equals("dariah") || tmp.get(1).getId().equalsIgnoreCase("dariah"));
+	}
+}

apps/dnet-directindex-api/src/test/java/eu/dnetlib/openaire/directindex/mapping/XmlRecordConverterTest.java

@@ -0,0 +1,294 @@
+package eu.dnetlib.openaire.directindex.mapping;
+
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.when;
+
+import java.io.StringReader;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.text.StringEscapeUtils;
+import org.apache.velocity.app.VelocityEngine;
+import org.dom4j.Document;
+import org.dom4j.io.OutputFormat;
+import org.dom4j.io.SAXReader;
+import org.dom4j.io.XMLWriter;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.mockito.junit.jupiter.MockitoSettings;
+import org.mockito.quality.Strictness;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import eu.dnetlib.openaire.directindex.input.DatasourceEntry;
+import eu.dnetlib.openaire.directindex.input.PidEntry;
+import eu.dnetlib.openaire.directindex.input.ResultEntry;
+import eu.dnetlib.openaire.directindex.is.ISLookupClient;
+
+@ExtendWith(MockitoExtension.class)
+@MockitoSettings(strictness = Strictness.LENIENT)
+public class XmlRecordConverterTest {
+
+	// private VelocityEngine ve;
+
+	private final String community_api = "https://dev-openaire.d4science.org/openaire/community/";
+
+	@Mock
+	private ISLookupClient isLookupClient;
+
+	// Class Under Test
+	private XmlRecordConverter xmlRecordConverter;
+
+	@BeforeEach
+	public void setUp() throws Exception {
+		final VelocityEngine ve = new VelocityEngine();
+		final Properties props = new Properties();
+		props.setProperty("resource.loader", "class");
+		props.setProperty("class.resource.loader.class", "org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader");
+		ve.init(props);
+
+		xmlRecordConverter = new XmlRecordConverter();
+		xmlRecordConverter.setCommunity_api(community_api);
+		xmlRecordConverter.setOafSchemaLocation("http://oaf/oaf.xsd");
+		xmlRecordConverter.setIsLookupClient(isLookupClient);
+		xmlRecordConverter.setVelocityEngine(ve);
+
+		// final Properties props = new Properties();
+		// props.setProperty("resource.loader", "class");
+		// props.setProperty("class.resource.loader.class", "org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader");
+		// props.setProperty("runtime.log.logsystem.class", "org.apache.velocity.runtime.log.Log4JLogChute");
+
+		when(isLookupClient.findVocabulary(anyString())).thenAnswer(invocation -> {
+			final String query = invocation.getArguments()[0].toString();
+			if (query.contains("dnet:result_typologies")) {
+				return asMap("publication @@@ publication", "dataset @@@ dataset", "software @@@ software", "other @@@ other");
+			} else if (query.contains("dnet:access_modes")) {
+				return asMap("OPEN @@@ Open Access");
+			} else if (query.contains("dnet:publication_resource")) {
+				return asMap("0001 @@@ Journal Article");
+			} else if (query.contains("dnet:pid_types")) {
+				return asMap("oai @@@ Open Archive Initiative", "doi @@@ Digital object identifier");
+			} else if (query.contains("dnet:languages")) {
+				return asMap("ita @@@ Italian");
+			} else {
+				return new HashMap<String, String>();
+			}
+		});
+
+		when(isLookupClient.findContexts())
+			.thenAnswer(invocation -> asMap("egi::classification::natsc::math::stats @@@ Statistics and Probability", "egi::classification::natsc::math::pure @@@ Pure Mathematics", "egi::classification::natsc::math @@@ Mathematics", "egi::classification::natsc @@@ Natural Sciences", "egi::classification @@@ EGI classification scheme", "egi @@@ EGI", "enermaps::selection @@@ Selected by the H2020 EnerMaps project"));
+
+		when(isLookupClient.findDatasource(anyString()))
+			.thenAnswer(invocation -> new DatasourceEntry(invocation.getArguments()[0].toString(), "REPO NAME", "repo________"));
+	}
+
+	private Map<String, String> asMap(final String... arr) {
+		return Arrays
+			.stream(arr)
+			.collect(Collectors.toMap(s -> StringUtils.substringBefore(s, "@@@").trim(), s -> StringUtils.substringAfter(s, "@@@").trim()));
+	}
+
+	@Test
+	public void testAsIndexRecord() throws Exception {
+		final ResultEntry pub = new ResultEntry();
+		pub.setOriginalId("ORIG_ID_1234");
+		pub.setTitle("TEST TITLE <test>");
+		pub.getAuthors().add("Michele Artini");
+		pub.getAuthors().add("Claudio Atzori");
+		pub.getAuthors().add("Alessia Bardi");
+		pub.setPublisher("Test publisher");
+		pub.setDescription("DESCRIPTION & DESCRIPTION & DESCRIPTION & DESCRIPTION & DESCRIPTION & DESCRIPTION & DESCRIPTION ");
+		pub.setLanguage("ita");
+		pub.setResourceType("0001");
+		pub.setUrl("http://test.it/a=1&b=2");
+		pub.getPids().add(new PidEntry("doi", "10.000/xyz-123"));
+		pub.getPids().add(new PidEntry("oai", "oai:1234"));
+		pub.setCollectedFromId("test________::zenodo");
+		pub.setHostedById("test________::UNKNOWN");
+		pub.getLinksToProjects().add("info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus");
+		pub.getLinksToProjects().add("info:eu-repo/grantAgreement/RCUK//244%2F909/EU/Making Capabilities Work/WorkAble");
+		pub.getContexts().add("egi::classification::natsc::math::pure");
+		pub.getContexts().add("egi::classification::natsc::math::stats");
+		pub.getContexts().add("enermaps");
+		pub.getContexts().add("enermaps::selection");
+		pub.setAccessRightCode("OPEN");
+		final String xml = xmlRecordConverter.convert(pub);
+
+		final Document doc = new SAXReader().read(new StringReader(xml));
+
+		final OutputFormat format = OutputFormat.createPrettyPrint();
+
+		final XMLWriter writer = new XMLWriter(System.out, format);
+
+		writer.write(doc);
+
+		/* writer.close(); */
+
+	}
+
+	@Test
+	public void testAsIndexRecordAccessRight() throws Exception {
+		final ResultEntry pub = new ResultEntry();
+		pub.setOriginalId("ORIG_ID_1234");
+		pub.setTitle("TEST TITLE <test>");
+		pub.getAuthors().add("Michele Artini");
+		pub.getAuthors().add("Claudio Atzori");
+		pub.getAuthors().add("Alessia Bardi");
+		pub.setPublisher("Test publisher");
+		pub.setDescription("DESCRIPTION & DESCRIPTION & DESCRIPTION & DESCRIPTION & DESCRIPTION & DESCRIPTION & DESCRIPTION ");
+		pub.setLanguage("ita");
+		pub.setAccessRightCode("OPEN");
+		pub.setResourceType("0001");
+		pub.setUrl("http://test.it/a=1&b=2");
+		pub.getPids().add(new PidEntry("doi", "10.000/xyz-123"));
+		pub.getPids().add(new PidEntry("oai", "oai:1234"));
+		pub.setCollectedFromId("test________::zenodo");
+		pub.setHostedById("test________::UNKNOWN");
+		pub.getLinksToProjects().add("info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus");
+		pub.getLinksToProjects().add("info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble");
+		pub.getContexts().add("egi::classification::natsc::math::pure");
+		pub.getContexts().add("egi::classification::natsc::math::stats");
+
+		final String xml = xmlRecordConverter.convert(pub);
+
+		final Document doc = new SAXReader().read(new StringReader(xml));
+
+		final OutputFormat format = OutputFormat.createPrettyPrint();
+
+		final XMLWriter writer = new XMLWriter(System.out, format);
+
+		writer.write(doc);
+
+		/* writer.close(); */
+
+	}
+
+	@Test
+	public void testAsORP() throws Exception {
+		final ResultEntry pub = new ResultEntry();
+		pub.setOriginalId("ORIG_ID_1234");
+		pub.setTitle("TEST TITLE <test>");
+		pub.getAuthors().add("Michele Artini");
+		pub.getAuthors().add("Claudio Atzori");
+		pub.getAuthors().add("Alessia Bardi");
+		pub.setPublisher("Test publisher");
+		pub.setDescription("DESCRIPTION & DESCRIPTION & DESCRIPTION & DESCRIPTION & DESCRIPTION & DESCRIPTION & DESCRIPTION ");
+		pub.setLanguage("ita");
+		pub.setAccessRightCode("OPEN");
+		pub.setResourceType("0020");
+		pub.setType("other");
+		pub.setUrl("http://test.it/a=1&b=2");
+		pub.getPids().add(new PidEntry("doi", "10.000/xyz-123"));
+		pub.getPids().add(new PidEntry("oai", "oai:1234"));
+		pub.setCollectedFromId("test________::zenodo");
+		pub.setHostedById("test________::UNKNOWN");
+		pub.getLinksToProjects().add("info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus");
+		pub.getLinksToProjects().add("info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble");
+		pub.getContexts().add("egi::classification::natsc::math::pure");
+		pub.getContexts().add("egi::classification::natsc::math::stats");
+
+		final String xml = xmlRecordConverter.convert(pub);
+
+		final Document doc = new SAXReader().read(new StringReader(xml));
+
+		final OutputFormat format = OutputFormat.createPrettyPrint();
+
+		final XMLWriter writer = new XMLWriter(System.out, format);
+
+		writer.write(doc);
+
+		/* writer.close(); */
+
+	}
+
+	@Test
+	public void testAsIndexRecord_json() throws Exception {
+		testAsIndexRecord_json("test_record.json");
+
+	}
+
+	@Test
+	public void testAsIndexRecord_json_with_greek_chars() throws Exception {
+
+		testAsIndexRecord_json("test_record_with_greek_chars.json");
+
+	}
+
+	@Test
+	public void testAsIndexRecord_openaireId() throws Exception {
+
+		testAsIndexRecord_json("test_record_openaireId.json");
+
+	}
+
+	@Test
+	public void testAsIndexRecord_json_zenodo() throws Exception {
+
+		testAsIndexRecord_json("test_zenodo.json");
+
+	}
+
+	@Test
+	public void testAsIndexRecord_json_zenodoWithAmpersand() throws Exception {
+
+		testAsIndexRecord_json("test_zenodoAmpersandEverywhere.json");
+
+	}
+
+	@Test
+	public void testAsIndexRecord_json_software() throws Exception {
+
+		testAsIndexRecord_json("test_record_software.json");
+
+	}
+
+	@Test
+	public void testAsIndexRecord_json_orp() throws Exception {
+
+		testAsIndexRecord_json("test_record_orp.json");
+
+	}
+
+	private void testAsIndexRecord_json(final String filename) throws Exception {
+		final ResultEntry pub =
+			new ObjectMapper().readValue(getClass().getResourceAsStream("/samples/" + filename), ResultEntry.class);
+
+		final String xml = xmlRecordConverter.convert(pub);
+		// System.out.println(xml);
+
+		final Document doc = new SAXReader().read(new StringReader(xml));
+
+		final OutputFormat format = OutputFormat.createPrettyPrint();
+
+		final XMLWriter writer = new XMLWriter(System.out, format);
+
+		writer.write(doc);
+
+		/* writer.close(); */
+	}
+
+	@Test
+	public void testAsIndexRecord_json_zenodocommunities() throws Exception {
+		testAsIndexRecord_json("test_zenodo_community.json");
+	}
+
+	@Test
+	public void testAsIndexRecordFromSandbox_json_zenodocommunities() throws Exception {
+		testAsIndexRecord_json("test_zenodo_community2.json");
+	}
+
+	@Test
+	public void testEscapeUnicode() {
+		final String unicodeTxt =
+			"i.e. closed curves of the form $t\ud835\udfc4 [0,2\u03c0] \u21a6 (\\cos t)u + (\\sin t)v$ for suitable orthogonal vectors $u$";
+		System.out.println(StringEscapeUtils.escapeXml11(unicodeTxt));
+	}
+
+}

apps/dnet-directindex-api/src/test/resources/log4j.properties

@@ -0,0 +1,10 @@
+### Root Level ###
+log4j.rootLogger=WARN, CONSOLE
+### Configuration for the CONSOLE appender ###
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %d %c - %m%n
+org.apache.cxf.Logger=org.apache.cxf.common.logging.Log4jLogger
+### Application Level ###
+log4j.logger.eu.dnetlib=INFO
+log4j.logger.eu.dnetlib.msro.openaireplus.workflows.nodes.contexts=DEBUG

apps/dnet-directindex-api/src/test/resources/samples/test_record.json

@@ -0,0 +1,35 @@
+{
+	"originalId": "ORIG_ID_12345",
+	"title": "TEST TITLE",
+	"authors": [
+		"Michele Artini",
+		"Claudio Atzori",
+		"Alessia Bardi"
+	],
+	"publisher": "Test publisher",
+	"description": "DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION",
+	"language": "ita",
+	"pids": [
+		{
+			"type": "doi",
+			"value": "10.000/xyz-123"
+		},
+		{
+			"type": "oai",
+			"value": "oai:1234"
+		}
+	],
+	"accessRightCode": "OPEN",
+	"resourceType": "0001",
+	"url": "http://test.it/xyz",
+	"collectedFromId": "opendoar____::2659",
+	"hostedById": "opendoar____::2367",
+	"linksToProjects": [
+		"info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus",
+		"info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble"
+	],
+	"contexts": [
+		"egi::classification::natsc::math::pure",
+		"egi::classification::natsc::math::stats"
+	]
+}

apps/dnet-directindex-api/src/test/resources/samples/test_record_ariadne.json

@@ -0,0 +1,25 @@
+{
+  "openaireId":"od_______278::d5b39693ac6c2197e8cf48008a982950",
+  "originalId": "oai:rudar.ruc.dk:1800/24690",
+  "type": "publication",
+  "title": "Autofiktion",
+  "authors": [
+	"Sloan, Patrick Alexander Raaby",
+	"Strange, Gustav Valdemar",
+	"Freund-Jensen, Sofie",
+	"Canvin, Emilie Meistrup",
+	"Faaborg, Ida Selvejer",
+	"Kruse, Mikkel"
+  ],
+  "accessRightCode" : "OPEN",
+  "publisher": "",
+  "description": "The following paper is a project on the autofictional book Færdig med Eddy Bellegueule by Édouard Louis (2015). The main focus is to ascertain how the novel is an expression of the author’s effort to connect social life and literature, and what part autofiction as a genre plays in this process. A textual analysis will describe the narrator’s position, and the aesthetic grip of the novel. Furthermore, we will analyze the central themes in Louis’ novel with theories by sociologist Pierre Bourdieu, gender theorist Judith Butler and postcolonial theorist Gayatri Spivak. This includes symbolic violence, gender roles, language and suppression. With use of classic literary theory and the more recent and specific theory on autofiction, the paper concludes, that Færdig med Eddy Bellegueule is based on non-fictional events, but presents these events through a fictionalization that raise the story into a literary work of art. The genre autofiction encircles a current tendency to blend non-fictional events and characters into literature, and Færdig med Eddy Bellegueule writes itself perfectly into this twilight zone between fact and fiction.",
+  "language": "eng",
+  "pids": [],
+  "resourceType": "0006",
+  "url": "http://rudar.ruc.dk/handle/1800/24690",
+  "collectedFromId": "opendoar____::278",
+  "hostedById": "opendoar____::278",
+  "linksToProjects": [],
+  "contexts": []
+}

apps/dnet-directindex-api/src/test/resources/samples/test_record_complete.json

@@ -0,0 +1,37 @@
+{
+  "originalId": "ORIG_ID_TEST",
+  "type": "publication",
+  "title": "TEST TITLE",
+  "authors": [
+	"Michele Artini",
+	"Claudio Atzori",
+	"Alessia Bardi"
+  ],
+  "publisher": "Test publisher",
+  "description": "DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION",
+  "language": "ita",
+  "pids": [
+	{
+	  "type": "doi",
+	  "value": "10.000/xyz-123"
+	},
+	{
+	  "type": "oai",
+	  "value": "oai:1234"
+	}
+  ],
+  "accessRightCode": "EMBARGO",
+  "embargoEndDate": "2018-02-02",
+  "resourceType": "0001",
+  "url": "http://test.it/xyz",
+  "collectedFromId": "opendoar____::2659",
+  "hostedById": "opendoar____::2659",
+  "linksToProjects": [
+	"info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus",
+	"info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble"
+  ],
+  "contexts": [
+	"egi::classification::natsc::math::pure",
+	"egi::classification::natsc::math::stats"
+  ]
+}

apps/dnet-directindex-api/src/test/resources/samples/test_record_openaireId.json

@@ -0,0 +1,38 @@
+{
+  "openaireId":"dedup_wf_001::ab42e8116f4ae3c4021fb7c643a8167a",
+  "originalId": "ORIG_ID_TEST",
+  "type": "publication",
+  "title": "TEST TITLE",
+  "authors": [
+	"Michele Artini",
+	"Claudio Atzori",
+	"Alessia Bardi"
+  ],
+  "publisher": "Test publisher",
+  "description": "DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION",
+  "language": "ita",
+  "pids": [
+	{
+	  "type": "doi",
+	  "value": "10.000/xyz-123"
+	},
+	{
+	  "type": "oai",
+	  "value": "oai:1234"
+	}
+  ],
+  "accessRightCode": "EMBARGO",
+  "embargoEndDate": "2018-02-02",
+  "resourceType": "0001",
+  "url": "http://test.it/xyz",
+  "collectedFromId": "opendoar____::2659",
+  "hostedById": "opendoar____::2659",
+  "linksToProjects": [
+	"info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus",
+	"info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble"
+  ],
+  "contexts": [
+	"egi::classification::natsc::math::pure",
+	"egi::classification::natsc::math::stats"
+  ]
+}

apps/dnet-directindex-api/src/test/resources/samples/test_record_orp.json

@@ -0,0 +1,21 @@
+{
+	"openaireId": "od_______278::d5b39693ac6c2197e8cf48008a982951",
+	"originalId": "oai:rudar.ruc.dk:1800/24691",
+	"type": "other",
+	"title": "Other test record",
+	"authors": [
+		"Sloan, Patrick Alexander Raaby",
+		"Strange, Gustav Valdemar"
+	],
+	"accessRightCode": "OPEN",
+	"publisher": "",
+	"description": "This is for testing software",
+	"language": "eng",
+	"pids": [],
+	"resourceType": "0000",
+	"url": "http://rudar.ruc.dk/handle/1800/24691",
+	"collectedFromId": "opendoar____::278",
+	"hostedById": "opendoar____::278",
+	"linksToProjects": [],
+	"contexts": []
+}

apps/dnet-directindex-api/src/test/resources/samples/test_record_software.json

@@ -0,0 +1,21 @@
+{
+	"openaireId": "od_______278::d5b39693ac6c2197e8cf48008a982951",
+	"originalId": "oai:rudar.ruc.dk:1800/24691",
+	"type": "software",
+	"title": "Software test",
+	"authors": [
+		"Sloan, Patrick Alexander Raaby",
+		"Strange, Gustav Valdemar"
+	],
+	"accessRightCode": "OPEN",
+	"publisher": "",
+	"description": "This is for testing software",
+	"language": "eng",
+	"pids": [],
+	"resourceType": "0000",
+	"url": "http://rudar.ruc.dk/handle/1800/24691",
+	"collectedFromId": "opendoar____::278",
+	"hostedById": "opendoar____::278",
+	"linksToProjects": [],
+	"contexts": []
+}

apps/dnet-directindex-api/src/test/resources/samples/test_record_with_greek_chars.json

@@ -0,0 +1,29 @@
+{
+	"originalId": "ORIG_ID_12345",
+	"title": "TEST TITLE WITH Greek Characters",
+	"authors": [
+		"αβγδεζηθικλμ νξοπρσςτυφχψω",
+		"ΑΒΓΔΕΖΗΘΙΚΛ ΜΝΞΟΠΡΣΤΥΦΧΨΩ"
+	],
+	"publisher": "âââââââ",
+	"description": "Αν περιμένατε να βρίσκεται εδώ μια σελίδα και δεν υπάρχει, η σελίδα μπορεί να μην εμφανίζεται λόγω καθυστέρησης στην ανανέωση της βάσης δεδομένων, ή μπορεί να έχει διαγραφεί. (Δείτε την γρήγορη διαγραφή σελίδων για πιθανούς λόγους). Μπορείτε να δοκιμάστε την λειτουργία εκκαθάρισης, και να ελέγξετε το αρχείο διαγραφών.",
+	"language": "ell",
+	"pids": [
+		{
+			"type": "doi",
+			"value": "10.000/xyz-123-gr"
+		}
+	],
+	"accessRightCode": "EMBARGO",
+	"embargoEndDate": "2018-02-02",
+	"resourceType": "0001",
+	"url": "http://test.it/xyz",
+	"collectedFromId": "opendoar____::2659",
+	"hostedById": "opendoar____::2367",
+	"linksToProjects": [
+		"info:eu-repo/grantAgreement/EC/FP7/123456/EU//Test"
+	],
+	"contexts": [
+		"egi::classification::natsc::math::stats"
+	]
+}

apps/dnet-directindex-api/src/test/resources/samples/test_record_wrong_openaireId.json

@@ -0,0 +1,38 @@
+{
+	"openaireId": "dedup_wf_001::ab42e811",
+	"originalId": "ORIG_ID_TEST",
+	"type": "publication",
+	"title": "TEST TITLE",
+	"authors": [
+		"Michele Artini",
+		"Claudio Atzori",
+		"Alessia Bardi"
+	],
+	"publisher": "Test publisher",
+	"description": "DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION DESCRIPTION",
+	"language": "ita",
+	"pids": [
+		{
+			"type": "doi",
+			"value": "10.000/xyz-123"
+		},
+		{
+			"type": "oai",
+			"value": "oai:1234"
+		}
+	],
+	"accessRightCode": "EMBARGO",
+	"embargoEndDate": "2018-02-02",
+	"resourceType": "0001",
+	"url": "http://test.it/xyz",
+	"collectedFromId": "opendoar____::2659",
+	"hostedById": "opendoar____::2659",
+	"linksToProjects": [
+		"info:eu-repo/grantAgreement/EC/FP7/283595/EU//OpenAIREplus",
+		"info:eu-repo/grantAgreement/EC/FP7/244909/EU/Making Capabilities Work/WorkAble"
+	],
+	"contexts": [
+		"egi::classification::natsc::math::pure",
+		"egi::classification::natsc::math::stats"
+	]
+}

apps/dnet-directindex-api/src/test/resources/samples/test_zenodo.json

@@ -0,0 +1,26 @@
+{
+	"authors": [
+		"Alouges, Fran\u00e7ois",
+		"Di Fratta, Giovanni"
+	],
+	"collectedFromId": "opendoar____::2659",
+	"description": "The paper is about the parking 3-sphere swimmer ($\\text{sPr}_3$). This is a low-Reynolds number model swimmer composed of three balls of equal radii. The three balls can move along three horizontal axes (supported in the same plane) that mutually meet at the center of $\\text{sPr}_3$ with angles of $120^{\u2218}$ . The governing dynamical system is introduced and the implications of its geometric symmetries revealed. It is then shown that, in the first order range of small strokes, optimal periodic strokes are ellipses embedded in 3d space, i.e. closed curves of the form $t\ud835\udfc4 [0,2\u03c0] \u21a6 (\\cos t)u + (\\sin t)v$ for suitable orthogonal vectors $u$ and $v$ of $\u211d^3$. A simple analytic expression for the vectors $u$ and $v$ is derived. The results of the paper are used in a second article where the real physical dynamics of $\\text{sPr}_3$ is analyzed in the asymptotic range of very long arms. ; Comment: 17 pages, 4 figures",
+	"hostedById": "opendoar____::2659",
+	"originalId": "oai:zenodo.org:996201",
+	"pids": [
+		{
+			"type": "oai",
+			"value": "oai:zenodo.org:996201"
+		},
+		{
+			"type": "doi",
+			"value": "10.5281/zenodo.996201"
+		}
+	],
+	"accessRightCode": "OPEN",
+	"publisher": "Zenodo",
+	"resourceType": "0020",
+	"title": "Parking 3-sphere swimmer. I. Energy minimizing strokes",
+	"type": "publication",
+	"url": "https://zenodo.org/record/996201"
+}

apps/dnet-directindex-api/src/test/resources/samples/test_zenodoAmpersandEverywhere.json

@@ -0,0 +1,29 @@
+{
+	"authors": [
+		"Conradt, Tobias",
+		"& members of the ISIMIP project (original data provision), cf. Hempel et al. 2013, https://doi.org/10.5194/esd-4-219-2013"
+	],
+	"collectedFromId": "re3data_____::r3d100010468",
+	"description": "<p>ISIMIP-2a climate data cutout provided for Sardinia in the framework of SIM4NEXUS & X<\/p>",
+	"hostedById": "re3data_____::r3d100010468",
+	"linksToProjects": [
+		"info:eu-repo/grantAgreement/EC/H2020/689150//Sustainable Integrated Management FOR the NEXUS of water-land-food-energy-climate for a resource-efficient Europe & beyond/SIM4NEXUS"
+	],
+	"accessRightCode": "OPEN",
+	"originalId": "10.5281/zenodo.1460665",
+	"pids": [
+		{
+			"type": "oai",
+			"value": "oai:zenodo.org:1460665"
+		},
+		{
+			"type": "doi",
+			"value": "10.5281/zenodo.1460665"
+		}
+	],
+	"publisher": "Zenodo & Zenodo",
+	"resourceType": "0021",
+	"title": "sardinia_tasmax_MIROC-ESM-CHEM_rcp4p5_2006-2099_daily.nc4 & the title has && in it",
+	"type": "dataset",
+	"url": "https://zenodo.org/record/1460665"
+}

apps/dnet-directindex-api/src/test/resources/samples/test_zenodo_community.json

@@ -0,0 +1,30 @@
+{
+	"authors": [
+		"Dag Haug",
+		"Marius Jhndal"
+	],
+	"collectedFromId": "re3data_____::r3d100010468",
+	"contexts": [
+		"https://sandbox.zenodo.org/communities/oac-ni",
+		"pippo"
+	],
+	"accessRightCode": "OPEN",
+	"description": "<p>Official releases of the PROIEL treebank of ancient Indo-European languages</p>",
+	"hostedById": "re3data_____::r3d100010468",
+	"originalId": "10.5281/zenodo.11003",
+	"pids": [
+		{
+			"type": "oai",
+			"value": "oai:zenodo.org:11003"
+		},
+		{
+			"type": "doi",
+			"value": "10.5281/zenodo.11003"
+		}
+	],
+	"publisher": "Zenodo",
+	"resourceType": "0021",
+	"title": "proiel-treebank: 20140723 version",
+	"type": "dataset",
+	"url": "https://zenodo.org/record/11003"
+}

apps/dnet-directindex-api/src/test/resources/samples/test_zenodo_community2.json

@@ -0,0 +1,27 @@
+{
+	"authors": [
+		"test"
+	],
+	"collectedFromId": "re3data_____::r3d100010468",
+	"contexts": [
+		"https://zenodo.org/communities/oac-ni"
+	],
+	"description": "<p>test</p>",
+	"hostedById": "re3data_____::r3d100010468",
+	"originalId": "oai:zenodo.org:315784",
+	"accessRightCode": "OPEN",
+	"pids": [
+		{
+			"type": "oai",
+			"value": "oai:zenodo.org:315784"
+		},
+		{
+			"type": "doi",
+			"value": "10.5072/zenodo.315784"
+		}
+	],
+	"resourceType": "0020",
+	"title": "test",
+	"type": "other",
+	"url": "https://zenodo.org/record/315784"
+}

apps/dnet-directindex-api/src/test/resources/samples/test_zenodo_problems.json

@@ -0,0 +1,29 @@
+{
+	"authors": [
+		"Milanese, Alessio",
+		"Mende, Daniel",
+		"Zeller, Georg",
+		"Sunagawa, Shinichi"
+	],
+	"collectedFromId": "re3data_____::r3d100010468",
+	"description": "<p>We simulated ten human gut metagenomic samples to assess the taxonomic quantification accuracy of the mOTUs tool (<a href=\"http: \// motu-tool.org \/\">link</a>). In this directory you can find the metagenomic samples, the gold standard (used to produce them) and the profiles obtained with four metagenomic profiler tools.</p>\\n\\n<p>Check README.txt for more information.</p>",
+	"hostedById": "re3data_____::r3d100010468",
+	"originalId": "10.5281/zenodo.1473645",
+	"pids": [
+		{
+			"type": "oai",
+			"value": "oai:zenodo.org:1473645"
+		},
+		{
+			"type": "doi",
+			"value": "10.5281/zenodo.1473645"
+		}
+	],
+	"accessRightCode": "OPEN",
+	"publisher": "Zenodo",
+	"resourceType": "0021",
+	"title": "Simulated Human gut metagenomic samples to benchmark mOTUs v2",
+	"type": "dataset",
+	"url": "https://zenodo.org/record/1473645",
+	"version": "2"
+}

apps/pom.xml

@@ -16,6 +16,7 @@
 		<module>dhp-broker-public-application</module>
 		<module>dhp-mdstore-manager</module>
 		<module>dnet-orgs-database-application</module>
+		<module>dnet-directindex-api</module>
 		<module>dnet-exporter-api</module>
 		<module>scholexplorer-api</module>
 		<module>bioschemas-api</module>

libs/dnet-apps-common/pom.xml

@@ -1,5 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 
 	<parent>
 		<groupId>eu.dnetlib.dhp</groupId>
@@ -78,6 +80,15 @@
 			<artifactId>simpleclient_spring_web</artifactId>
 			<version>0.3.0</version>
 		</dependency>
+		<dependency>
+			<groupId>eu.dnetlib</groupId>
+			<artifactId>cnr-rmi-api</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.cxf</groupId>
+			<artifactId>cxf-rt-transports-http</artifactId>
+			<version>3.1.5</version>
+		</dependency>
 
 		<!-- Tests -->
 		<dependency>

libs/dnet-apps-common/src/main/java/eu/dnetlib/common/clients/ISLookupClientFactory.java

@@ -0,0 +1,55 @@
+
+package eu.dnetlib.common.clients;
+
+import org.apache.cxf.endpoint.Client;
+import org.apache.cxf.frontend.ClientProxy;
+import org.apache.cxf.jaxws.JaxWsProxyFactoryBean;
+import org.apache.cxf.transport.http.HTTPConduit;
+import org.apache.cxf.transports.http.configuration.HTTPClientPolicy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
+
+public class ISLookupClientFactory {
+
+	private static final Logger log = LoggerFactory.getLogger(ISLookupClientFactory.class);
+
+	private static final int requestTimeout = 60000 * 10;
+	private static final int connectTimeout = 60000 * 10;
+
+	public static ISLookUpService getLookUpService(final String isLookupUrl) {
+		return getServiceStub(ISLookUpService.class, isLookupUrl);
+	}
+
+	@SuppressWarnings("unchecked")
+	private static <T> T getServiceStub(final Class<T> clazz, final String endpoint) {
+		log.info(String.format("creating %s stub from %s", clazz.getName(), endpoint));
+		final JaxWsProxyFactoryBean jaxWsProxyFactory = new JaxWsProxyFactoryBean();
+		jaxWsProxyFactory.setServiceClass(clazz);
+		jaxWsProxyFactory.setAddress(endpoint);
+
+		final T service = (T) jaxWsProxyFactory.create();
+
+		Client client = ClientProxy.getClient(service);
+		if (client != null) {
+			HTTPConduit conduit = (HTTPConduit) client.getConduit();
+			HTTPClientPolicy policy = new HTTPClientPolicy();
+
+			log
+				.info(
+					String
+						.format(
+							"setting connectTimeout to %s, requestTimeout to %s for service %s",
+							connectTimeout,
+							requestTimeout,
+							clazz.getCanonicalName()));
+
+			policy.setConnectionTimeout(connectTimeout);
+			policy.setReceiveTimeout(requestTimeout);
+			conduit.setClient(policy);
+		}
+
+		return service;
+	}
+}

pom.xml

@@ -1,4 +1,6 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
 
 	<parent>
 		<groupId>org.springframework.boot</groupId>
@@ -183,12 +185,24 @@
 				</exclusions>
 			</dependency>
 
+			<dependency>
+				<groupId>eu.dnetlib</groupId>
+				<artifactId>cnr-rmi-api</artifactId>
+				<version>${cnr-rmi-api.version}</version>
+			</dependency>
+
 			<dependency>
 				<groupId>commons-cli</groupId>
 				<artifactId>commons-cli</artifactId>
 				<version>1.4</version>
 			</dependency>
 
+			<dependency>
+				<groupId>org.apache.commons</groupId>
+				<artifactId>commons-lang3</artifactId>
+				<version>3.12.0</version>
+			</dependency>
+
 			<dependency>
 				<groupId>commons-codec</groupId>
 				<artifactId>commons-codec</artifactId>
@@ -453,6 +467,7 @@
 		<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
 		<maven.compiler.plugin.version>3.6.0</maven.compiler.plugin.version>
 		<java.version>1.8</java.version>
+		<cnr-rmi-api.version>2.6.1</cnr-rmi-api.version>
 		<dhp-schemas-version>2.14.0</dhp-schemas-version>
 		<apache.solr.version>7.1.0</apache.solr.version>
 		<mongodb.driver.version>3.4.2</mongodb.driver.version>