diff --git a/src/main/java/eu/dnetlib/app/directindex/DirectIndexApplication.java b/src/main/java/eu/dnetlib/app/directindex/DirectIndexApplication.java index 15142f6..f66c229 100644 --- a/src/main/java/eu/dnetlib/app/directindex/DirectIndexApplication.java +++ b/src/main/java/eu/dnetlib/app/directindex/DirectIndexApplication.java @@ -46,11 +46,14 @@ public class DirectIndexApplication { @Value("${maven.pom.path}") private ClassPathResource pom; - @Value("${server.public_url}") - private String serverPublicUrl; + @Value("${dnet.directindex.baseurl}") + private String publicUrl; - @Value("${server.public_desc}") - private String serverPublicDesc; + @Value("${dnet.directindex.title}") + private String publicTitle; + + @Value("${dnet.directindex.description}") + private String publicDesc; @Value("${openaire.service.islookup.wsdl}") private String isLookupUrl; @@ -80,24 +83,24 @@ public class DirectIndexApplication { @Bean public OpenAPI newSwaggerDocket() { final List servers = new ArrayList<>(); - if (StringUtils.isNotBlank(serverPublicUrl)) { + if (StringUtils.isNotBlank(publicUrl)) { final Server server = new Server(); - server.setUrl(serverPublicUrl); - server.setDescription(serverPublicDesc); + server.setUrl(publicUrl); + server.setDescription(publicDesc); servers.add(server); } return new OpenAPI() .servers(servers) .info(new Info() - .title(swaggerTitle()) + .title(publicTitle) .description(DIRECT_INDEX_DESC) - .version(swaggerVersion()) + .version(currentVersion()) .license(AGPL_3_LICENSE)) .tags(new ArrayList<>()); } - private String swaggerVersion() { + private String currentVersion() { try { return new MavenXpp3Reader().read(new InputStreamReader(pom.getInputStream())).getVersion(); } catch (IOException | XmlPullParserException e) { @@ -108,7 +111,7 @@ public class DirectIndexApplication { @Bean public GroupedOpenApi publicApi() { return GroupedOpenApi.builder() - .group("D-Net DirectIndex API") + .group(publicTitle) .pathsToMatch("/api/**") .build(); } @@ -118,8 +121,4 @@ public class DirectIndexApplication { return ISLookupClientFactory.getLookUpService(isLookupUrl); } - protected String swaggerTitle() { - return "OpenAIRE DirectIndex API"; - } - } diff --git a/src/main/java/eu/dnetlib/app/directindex/clients/CommunityClient.java b/src/main/java/eu/dnetlib/app/directindex/clients/CommunityClient.java new file mode 100644 index 0000000..a71f263 --- /dev/null +++ b/src/main/java/eu/dnetlib/app/directindex/clients/CommunityClient.java @@ -0,0 +1,39 @@ +package eu.dnetlib.app.directindex.clients; + +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestClientException; +import org.springframework.web.client.RestTemplate; + +import eu.dnetlib.app.directindex.input.ZenodoContextList; + +@Component +public class CommunityClient { + + private static final String ZENODO_COMMUNITY = "zenodo.org/communities/"; + + private static final Log log = LogFactory.getLog(CommunityClient.class); + + @Value("${dnet.directindex.community.url") + private String communityApiUrl; + + public Collection translateZenodoCommunity(final String community) { + if (!community.contains(ZENODO_COMMUNITY)) { return Arrays.asList(community); } + final String context = community.substring(community.lastIndexOf("/") + 1); + final RestTemplate rt = new RestTemplate(); + try { + return new HashSet<>(rt.getForObject(communityApiUrl + context + "/openairecommunities", ZenodoContextList.class) + .getOpenAirecommunitylist()); + } catch (final RestClientException rce) { + log.error("Unable to get object for " + communityApiUrl + context + "/openairecommunities"); + log.error(rce.getMessage()); + return new HashSet<>(); + } + } +} diff --git a/src/main/java/eu/dnetlib/app/directindex/clients/DatasourceManagerClient.java b/src/main/java/eu/dnetlib/app/directindex/clients/DatasourceManagerClient.java new file mode 100644 index 0000000..cf8fb18 --- /dev/null +++ b/src/main/java/eu/dnetlib/app/directindex/clients/DatasourceManagerClient.java @@ -0,0 +1,109 @@ +package eu.dnetlib.app.directindex.clients; + +import java.io.Serializable; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestClientException; +import org.springframework.web.client.RestTemplate; + +import eu.dnetlib.app.directindex.input.DatasourceEntry; + +@Component +public class DatasourceManagerClient { + + @Value("${dnet.directindex.dsm.url") + private String dsmApiUrl; + + private static final Log log = LogFactory.getLog(DatasourceManagerClient.class); + + public DatasourceEntry findDatasource(final String dsId) { + final RestTemplate rt = new RestTemplate(); + final String url = dsmApiUrl + "/searchdetails/0/1?requestSortBy=id&order=ASCENDING"; + try { + final DsmSearchResponse res = rt.postForObject(url, new DsmSearchRequest(dsId), DsmSearchResponse.class); + return res.getDatasourceInfo() + .stream() + .map(ds -> new DatasourceEntry(ds.getId(), ds.getOfficialname(), ds.getNamespaceprefix())) + .findFirst() + .orElse(null); + } catch (final RestClientException rce) { + log.error("Unable to get object for " + url); + return null; + } + } + + public class DsmDatasourceInfo implements Serializable { + + private static final long serialVersionUID = -593392381920400974L; + + private String id; + private String officialname; + private String namespaceprefix; + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + public String getOfficialname() { + return officialname; + } + + public void setOfficialname(final String officialname) { + this.officialname = officialname; + } + + public String getNamespaceprefix() { + return namespaceprefix; + } + + public void setNamespaceprefix(final String namespaceprefix) { + this.namespaceprefix = namespaceprefix; + } + + } + + public class DsmSearchRequest implements Serializable { + + private static final long serialVersionUID = -2532361140043817319L; + + private String id; + + public DsmSearchRequest() {} + + public DsmSearchRequest(final String id) { + this.id = id; + } + + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + } + + public class DsmSearchResponse implements Serializable { + + private static final long serialVersionUID = 8944122902111813747L; + + private List datasourceInfo; + + public List getDatasourceInfo() { + return datasourceInfo; + } + + public void setDatasourceInfo(final List datasourceInfo) { + this.datasourceInfo = datasourceInfo; + } + } + +} diff --git a/src/main/java/eu/dnetlib/app/directindex/controllers/LegacyApiController.java b/src/main/java/eu/dnetlib/app/directindex/controllers/LegacyApiController.java new file mode 100644 index 0000000..1015678 --- /dev/null +++ b/src/main/java/eu/dnetlib/app/directindex/controllers/LegacyApiController.java @@ -0,0 +1,97 @@ +package eu.dnetlib.app.directindex.controllers; + +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; + +import eu.dnetlib.app.directindex.errors.DirectIndexApiException; +import eu.dnetlib.app.directindex.input.ResultEntry; +import eu.dnetlib.app.directindex.mapping.OafMapper; +import eu.dnetlib.app.directindex.service.DirectIndexService; + +@RestController("/api") +@ConditionalOnProperty(value = "dnet.directindex.legacy.enabled", havingValue = "true", matchIfMissing = false) +public class LegacyApiController { + + private static final Log log = LogFactory.getLog(LegacyApiController.class); + + @Autowired + private DirectIndexService service; + + @Autowired + private OafMapper oafMapper; + + @PostMapping("/results/feedObject") + public String feedResult(@RequestBody final ResultEntry pub, @RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit) + throws DirectIndexApiException { + + return service.prepareMetadataInsertion(pub); + + } + + @DeleteMapping("/result/{openaireId}") + public boolean deleteResultWithOpenaireId(@PathVariable(value = "openaireId") final String openaireId, + @RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit) throws DirectIndexApiException { + + service.prepareMetadataDeletion(openaireId); + + return true; + } + + @DeleteMapping("/results") + public boolean deleteResultWithOriginalId( + @RequestParam(value = "originalId", required = true) final String originalId, + @RequestParam(value = "collectedFromId", required = true) final String collectedFromId, + @RequestParam(value = "commit", required = false, defaultValue = "true") final boolean commit) throws DirectIndexApiException { + + final String openaireId = oafMapper.calculateOpenaireId(originalId, collectedFromId); + + service.prepareMetadataDeletion(openaireId); + + return true; + } + + @ExceptionHandler(Exception.class) + @ResponseStatus(value = HttpStatus.INTERNAL_SERVER_ERROR) + public @ResponseBody ErrorMessage handleException(final Exception e) { + log.error("Error in direct index API", e); + return new ErrorMessage(e); + } + + public class ErrorMessage { + + private final String message; + private final String stacktrace; + + public ErrorMessage(final Exception e) { + this(e.getMessage(), ExceptionUtils.getStackTrace(e)); + } + + public ErrorMessage(final String message, final String stacktrace) { + this.message = message; + this.stacktrace = stacktrace; + } + + public String getMessage() { + return message; + } + + public String getStacktrace() { + return stacktrace; + } + + } + +} diff --git a/src/main/java/eu/dnetlib/app/directindex/controllers/SwordApiController.java b/src/main/java/eu/dnetlib/app/directindex/controllers/SwordApiController.java index c94d1de..c4b00c6 100644 --- a/src/main/java/eu/dnetlib/app/directindex/controllers/SwordApiController.java +++ b/src/main/java/eu/dnetlib/app/directindex/controllers/SwordApiController.java @@ -5,6 +5,7 @@ import java.io.IOException; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; @@ -28,8 +29,6 @@ import eu.dnetlib.app.directindex.errors.SwordError; import eu.dnetlib.app.directindex.errors.SwordErrorType; import eu.dnetlib.app.directindex.errors.SwordException; import eu.dnetlib.app.directindex.input.ResultEntry; -import eu.dnetlib.app.directindex.is.ISLookupClient; -import eu.dnetlib.app.directindex.is.IndexDsInfo; import eu.dnetlib.app.directindex.mapping.OafMapper; import eu.dnetlib.app.directindex.service.DirectIndexService; import eu.dnetlib.app.directindex.solr.SolrIndexClient; @@ -40,16 +39,17 @@ import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; @RestController("/api/sword/3.0") +@ConditionalOnProperty(value = "dnet.directindex.sword.enabled", havingValue = "true", matchIfMissing = false) public class SwordApiController { @Autowired private DirectIndexService service; @Autowired - private ISLookupClient isLookupClient; + private SolrIndexClientFactory solrIndexClientFactory; @Autowired - private SolrIndexClientFactory solrIndexClientFactory; + private OafMapper oafMapper; @Value("${dnet.directindex.baseurl}") private String baseUrl; @@ -103,14 +103,13 @@ public class SwordApiController { @GetMapping("/objects/{id}/metadata") public ResponseEntity getMetadata(@PathVariable final String id) throws SwordException { - final IndexDsInfo info = isLookupClient.currentIndexDsInfo(); - final SolrIndexClient solr = solrIndexClientFactory.getClient(info); + final SolrIndexClient solr = solrIndexClientFactory.getClient(); final String metadata = solr.findRecord(id); if (StringUtils.isBlank(metadata)) { throw new SwordException(SwordErrorType.NotFound); } - return new ResponseEntity<>(OafMapper.toResultEntry(metadata), HttpStatus.OK); + return new ResponseEntity<>(oafMapper.toResultEntry(metadata), HttpStatus.OK); } @PutMapping("/objects/{id}/metadata") @@ -123,8 +122,7 @@ public class SwordApiController { @PathVariable final String id, @RequestBody final ResultEntry result) throws SwordException { - final IndexDsInfo info = isLookupClient.currentIndexDsInfo(); - final SolrIndexClient solr = solrIndexClientFactory.getClient(info); + final SolrIndexClient solr = solrIndexClientFactory.getClient(); if (!solr.existsRecord(id)) { throw new SwordException(SwordErrorType.NotFound); } diff --git a/src/main/java/eu/dnetlib/app/directindex/input/DatasourceEntry.java b/src/main/java/eu/dnetlib/app/directindex/input/DatasourceEntry.java index cf8fc02..e010ad7 100644 --- a/src/main/java/eu/dnetlib/app/directindex/input/DatasourceEntry.java +++ b/src/main/java/eu/dnetlib/app/directindex/input/DatasourceEntry.java @@ -1,9 +1,13 @@ package eu.dnetlib.app.directindex.input; +import java.io.Serializable; + import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; -public class DatasourceEntry { +public class DatasourceEntry implements Serializable { + + private static final long serialVersionUID = -8627323814467461993L; private String id; private String name; diff --git a/src/main/java/eu/dnetlib/app/directindex/input/PidEntry.java b/src/main/java/eu/dnetlib/app/directindex/input/PidEntry.java index a741d09..3a19e9a 100644 --- a/src/main/java/eu/dnetlib/app/directindex/input/PidEntry.java +++ b/src/main/java/eu/dnetlib/app/directindex/input/PidEntry.java @@ -1,13 +1,18 @@ package eu.dnetlib.app.directindex.input; +import java.io.Serializable; + import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.media.Schema.RequiredMode; -public class PidEntry { +public class PidEntry implements Serializable { - @Schema(required = true, description = "E.g. doi, pmc, urn. See http://api.openaire.eu/vocabularies/dnet:pid_types") + private static final long serialVersionUID = -4282430427274772667L; + + @Schema(requiredMode = RequiredMode.REQUIRED, description = "E.g. doi, pmc, urn. See http://api.openaire.eu/vocabularies/dnet:pid_types") private String type; - @Schema(required = true) + @Schema(requiredMode = RequiredMode.REQUIRED) private String value; public PidEntry() {} diff --git a/src/main/java/eu/dnetlib/app/directindex/input/ResultEntry.java b/src/main/java/eu/dnetlib/app/directindex/input/ResultEntry.java index bb23844..d3b6170 100644 --- a/src/main/java/eu/dnetlib/app/directindex/input/ResultEntry.java +++ b/src/main/java/eu/dnetlib/app/directindex/input/ResultEntry.java @@ -1,5 +1,6 @@ package eu.dnetlib.app.directindex.input; +import java.io.Serializable; import java.util.ArrayList; import java.util.List; @@ -13,7 +14,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.media.Schema.RequiredMode; -public class ResultEntry { +public class ResultEntry implements Serializable { + + private static final long serialVersionUID = 3171122009783163319L; private String openaireId; private String originalId; diff --git a/src/main/java/eu/dnetlib/app/directindex/input/ZenodoContextList.java b/src/main/java/eu/dnetlib/app/directindex/input/ZenodoContextList.java index 81c64e7..270e764 100644 --- a/src/main/java/eu/dnetlib/app/directindex/input/ZenodoContextList.java +++ b/src/main/java/eu/dnetlib/app/directindex/input/ZenodoContextList.java @@ -1,11 +1,14 @@ package eu.dnetlib.app.directindex.input; +import java.io.Serializable; import java.util.List; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @JsonIgnoreProperties(ignoreUnknown = true) -public class ZenodoContextList { +public class ZenodoContextList implements Serializable { + + private static final long serialVersionUID = -8575901008472098218L; private String zenodoid; diff --git a/src/main/java/eu/dnetlib/app/directindex/is/ISLookupClient.java b/src/main/java/eu/dnetlib/app/directindex/is/ISLookupClient.java index 3b4fcab..daec5db 100644 --- a/src/main/java/eu/dnetlib/app/directindex/is/ISLookupClient.java +++ b/src/main/java/eu/dnetlib/app/directindex/is/ISLookupClient.java @@ -57,26 +57,8 @@ public class ISLookupClient { return map; } - @Cacheable("contexts") - public Map findContexts() throws DirectIndexApiException { - final String query = - "collection('/db/DRIVER/ContextDSResources/ContextDSResourceType')[.//context/@type='community' or .//context/@type='ri']//*[name()='context' or name()='category' or name()='concept']/concat(@id, ' @@@ ', @label)"; - - final Map map = new HashMap<>(); - for (final String s : find(query)) { - final String[] arr = s.split("@@@"); - map.put(arr[0].trim(), arr[1].trim()); - } - return map; - } - - @Cacheable("layouts") - public String findLayoutForFormat(final String format) throws Exception { - return findOne("collection('/db/DRIVER/MDFormatDSResources/MDFormatDSResourceType')[.//NAME='" + format + "']//LAYOUT[@name='index']"); - } - @CacheEvict(allEntries = true, value = { - "indexDsInfo", "datasources", "vocabularies", "contexts", "layouts" + "datasources", "vocabularies" }) public void evictCache() { log.info("Evicting indexDsInfo cache"); diff --git a/src/main/java/eu/dnetlib/app/directindex/mapping/OafMapper.java b/src/main/java/eu/dnetlib/app/directindex/mapping/OafMapper.java index 8bbdcb6..5f36461 100644 --- a/src/main/java/eu/dnetlib/app/directindex/mapping/OafMapper.java +++ b/src/main/java/eu/dnetlib/app/directindex/mapping/OafMapper.java @@ -1,18 +1,35 @@ package eu.dnetlib.app.directindex.mapping; +import org.apache.commons.codec.digest.DigestUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import eu.dnetlib.app.directindex.clients.DatasourceManagerClient; +import eu.dnetlib.app.directindex.input.DatasourceEntry; import eu.dnetlib.app.directindex.input.ResultEntry; +@Component public class OafMapper { - public static String toOAF(final ResultEntry result) { + @Autowired + private DatasourceManagerClient dsmClient; + + public String toOAF(final ResultEntry result) { // TODO return null; } - public static ResultEntry toResultEntry(final String oaf) { + public ResultEntry toResultEntry(final String oaf) { // TODO Auto-generated method stub return null; } + public String calculateOpenaireId(final String originalId, final String collectedFromId) { + return calculateOpenaireId(originalId, dsmClient.findDatasource(collectedFromId)); + } + + private String calculateOpenaireId(final String originalId, final DatasourceEntry collectedFromEntry) { + return collectedFromEntry.getPrefix() + "::" + DigestUtils.md5Hex(originalId); + } } diff --git a/src/main/java/eu/dnetlib/app/directindex/tasks/ScheduledActions.java b/src/main/java/eu/dnetlib/app/directindex/tasks/ScheduledActions.java index a98c4ab..7f9fb69 100644 --- a/src/main/java/eu/dnetlib/app/directindex/tasks/ScheduledActions.java +++ b/src/main/java/eu/dnetlib/app/directindex/tasks/ScheduledActions.java @@ -16,7 +16,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import eu.dnetlib.app.directindex.errors.DirectIndexApiException; import eu.dnetlib.app.directindex.input.ResultEntry; -import eu.dnetlib.app.directindex.is.ISLookupClient; import eu.dnetlib.app.directindex.mapping.OafMapper; import eu.dnetlib.app.directindex.repo.PendingAction; import eu.dnetlib.app.directindex.repo.PendingActionRepository; @@ -28,14 +27,14 @@ public class ScheduledActions { private static final Log log = LogFactory.getLog(ScheduledActions.class); - @Value(value = "${dnet.directindex.scheduling.enabled}") + @Value("${dnet.directindex.scheduling.enabled}") private boolean enabled; @Autowired - private ISLookupClient isLookupClient; + private SolrIndexClientFactory solrIndexClientFactory; @Autowired - private SolrIndexClientFactory solrIndexClientFactory; + private OafMapper oafMapper; @Autowired private PendingActionRepository pendingActionRepository; @@ -67,7 +66,7 @@ public class ScheduledActions { } }) .filter(Objects::nonNull) - .map(OafMapper::toOAF) + .map(oafMapper::toOAF) .filter(StringUtils::isNotBlank)); solr.commit(); diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 26f2ef7..d3f28cc 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -2,9 +2,19 @@ server.port=8080 dnet.directindex.baseurl = http://localhost:8280 +dnet.directindex.scheduling.enabled}") + dnet.directindex.title = D-Net Direct Index Service dnet.directindex.description = Service that permits the indexing according to the SWORD v.3 protocol dnet.directindex.scheduling.enabled = false +dnet.directindex.legacy.enabled=true +dnet.directindex.sword.enabled=false + +dnet.directindex.solr.collection = +dnet.directindex.solr.url = +dnet.directindex.community.url = +dnet.directindex.dsm.url = + spring.profiles.active=dev