package eu.dnetlib.openaire.dsm; import static eu.dnetlib.openaire.common.ExporterConstants.API; import static eu.dnetlib.openaire.common.ExporterConstants.DS; import static eu.dnetlib.openaire.common.ExporterConstants.M; import static eu.dnetlib.openaire.common.ExporterConstants.R; import static eu.dnetlib.openaire.common.ExporterConstants.W; import java.util.List; import javax.validation.Valid; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.StopWatch; import org.apache.http.HttpStatus; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.web.bind.annotation.CrossOrigin; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import eu.dnetlib.enabling.datasources.common.DsmException; import eu.dnetlib.enabling.datasources.common.DsmForbiddenException; import eu.dnetlib.enabling.datasources.common.DsmNotFoundException; import eu.dnetlib.openaire.common.AbstractExporterController; import eu.dnetlib.openaire.common.OperationManager; import eu.dnetlib.openaire.dsm.dao.ResponseUtils; import eu.dnetlib.openaire.dsm.domain.AggregationHistoryResponse; import eu.dnetlib.openaire.dsm.domain.ApiDetails; import eu.dnetlib.openaire.dsm.domain.ApiDetailsResponse; import eu.dnetlib.openaire.dsm.domain.DatasourceDetailResponse; import eu.dnetlib.openaire.dsm.domain.DatasourceDetails; import eu.dnetlib.openaire.dsm.domain.DatasourceDetailsUpdate; import eu.dnetlib.openaire.dsm.domain.DatasourceDetailsWithApis; import eu.dnetlib.openaire.dsm.domain.DatasourceSnippetResponse; import eu.dnetlib.openaire.dsm.domain.RegisteredDatasourceInfo; import eu.dnetlib.openaire.dsm.domain.RequestFilter; import eu.dnetlib.openaire.dsm.domain.RequestSort; import eu.dnetlib.openaire.dsm.domain.RequestSortOrder; import eu.dnetlib.openaire.dsm.domain.Response; import eu.dnetlib.openaire.dsm.domain.SimpleDatasourceInfo; import eu.dnetlib.openaire.dsm.domain.SimpleResponse; import eu.dnetlib.openaire.vocabularies.Country; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; import io.swagger.v3.oas.annotations.tags.Tag; @RestController @CrossOrigin(origins = { "*" }) @ConditionalOnProperty(value = "openaire.exporter.enable.dsm", havingValue = "true") @Tag(name = "OpenAIRE DSM API", description = "the OpenAIRE Datasource Manager API") public class DsmApiController extends AbstractExporterController { @Autowired private DsmCore dsmCore; @RequestMapping(value = "/ds/countries", produces = { "application/json" }, method = RequestMethod.GET) @Operation(summary = "list the datasource countries", description = "list the datasource countries", tags = { DS, R }) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) public List listCountries() throws DsmException { return dsmCore.listCountries(); } @RequestMapping(value = "/ds/searchdetails/{page}/{size}", produces = { "application/json" }, method = RequestMethod.POST) @Operation(summary = "search datasources", description = "Returns list of Datasource details.", tags = { DS, R }) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) public DatasourceDetailResponse searchDsDetails( @RequestParam final RequestSort requestSortBy, @RequestParam final RequestSortOrder order, @RequestBody final RequestFilter requestFilter, @PathVariable final int page, @PathVariable final int size) throws DsmException { final StopWatch stop = StopWatch.createStarted(); final DatasourceDetailResponse rsp = dsmCore.searchDsDetails(requestSortBy, order, requestFilter, page, size); return prepareResponse(page, size, stop, rsp); } @RequestMapping(value = "/ds/aggregationhistory/{dsId}", produces = { "application/json" }, method = RequestMethod.GET) @Operation(summary = "search datasources", description = "Returns list of Datasource details.", tags = { DS, R }) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) public AggregationHistoryResponse aggregationHistory(@PathVariable final String dsId) throws DsmException { final StopWatch stop = StopWatch.createStarted(); final AggregationHistoryResponse rsp = dsmCore.aggregationhistory(dsId); return prepareResponse(0, rsp.getAggregationInfo().size(), stop, rsp); } @RequestMapping(value = "/ds/searchsnippet/{page}/{size}", produces = { "application/json" }, method = RequestMethod.POST) @Operation(summary = "search datasources", description = "Returns list of Datasource basic info.", tags = { DS, R }) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) public DatasourceSnippetResponse searchSnippet( @RequestParam final RequestSort requestSortBy, @RequestParam final RequestSortOrder order, @RequestBody final RequestFilter requestFilter, @PathVariable final int page, @PathVariable final int size) throws DsmException { final StopWatch stop = StopWatch.createStarted(); final DatasourceSnippetResponse rsp = dsmCore.searchSnippet(requestSortBy, order, requestFilter, page, size); return prepareResponse(page, size, stop, rsp); } @RequestMapping(value = "/ds/searchregistered/{page}/{size}", produces = { "application/json" }, method = RequestMethod.POST) @Operation(summary = "search among registered datasources", description = "Returns list of Datasource basic info.", tags = { DS, R }) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) public DatasourceSnippetResponse searchRegistered( @RequestParam final RequestSort requestSortBy, @RequestParam final RequestSortOrder order, @RequestBody final RequestFilter requestFilter, @PathVariable final int page, @PathVariable final int size) throws DsmException { final StopWatch stop = StopWatch.createStarted(); final DatasourceSnippetResponse rsp = dsmCore.searchRegistered(requestSortBy, order, requestFilter, page, size); return prepareResponse(page, size, stop, rsp); } @RequestMapping(value = "/ds/recentregistered/{size}", produces = { "application/json" }, method = RequestMethod.GET) @Operation(summary = "return the latest datasources that were registered through Provide", description = "Returns list of Datasource basic info.", tags = { DS, R }) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) public SimpleResponse recentRegistered(@PathVariable final int size) throws Throwable { final StopWatch stop = StopWatch.createStarted(); final SimpleResponse rsp = dsmCore.searchRecentRegistered(size); return prepareResponse(1, size, stop, rsp); } @RequestMapping(value = "/ds/countregistered", produces = { "application/json" }, method = RequestMethod.GET) @Operation(summary = "return the number of datasources registered after the given date", description = "Returns a number.", tags = { DS, R }) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) public Long countRegistered(@RequestParam final String fromDate, @RequestParam(required = false) final String typologyFilter) throws Throwable { return dsmCore.countRegisteredAfter(fromDate, typologyFilter); } @RequestMapping(value = "/ds/api/{dsId}", produces = { "application/json" }, method = RequestMethod.GET) @Operation(summary = "get the list of API for a given datasource", description = "Returns the list of API for a given datasource.", tags = { API, R }) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) public ApiDetailsResponse getApi( @PathVariable final String dsId) throws DsmException { final StopWatch stop = StopWatch.createStarted(); final ApiDetailsResponse rsp = dsmCore.getApis(dsId); return prepareResponse(0, rsp.getApi().size(), stop, rsp); } @RequestMapping(value = "/api/baseurl/{page}/{size}", produces = { "application/json" }, method = RequestMethod.POST) @Operation(summary = "search for the list of base URLs of Datasource APIs managed by a user", description = "Returns the list of base URLs of Datasource APIs managed by a user", tags = { DS, API, R }) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) public List searchBaseUrls( @RequestBody final RequestFilter requestFilter, @PathVariable final int page, @PathVariable final int size) throws DsmException { return dsmCore.findBaseURLs(requestFilter, page, size); } @RequestMapping(value = "/ds/api/{apiId}", method = RequestMethod.DELETE) @Operation(summary = "delete an API", description = "delete an API, if removable", tags = { API, W }) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Api not found"), @ApiResponse(responseCode = "403", description = "Api not removable"), @ApiResponse(responseCode = "500", description = "DSM Server error") }) public void deleteApi(@PathVariable final String apiId) throws DsmForbiddenException, DsmNotFoundException { dsmCore.deleteApi(apiId); } @RequestMapping(value = "/ds/manage", method = RequestMethod.POST) @Operation(summary = "set the managed status for a given datasource", description = "set the managed status for a given datasource", tags = { DS, W }) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) public void setManaged( @RequestParam final String id, @RequestParam final boolean managed) throws DsmException { dsmCore.setManaged(id, managed); } @RequestMapping(value = "/ds/managed/{id}", method = RequestMethod.GET) @Operation(summary = "get the datasource managed status", description = "get the datasource managed status", tags = { DS, R }) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) public boolean isManaged(@PathVariable final String id) throws DsmException { return dsmCore.isManaged(id); } @RequestMapping(value = "/ds/add", method = RequestMethod.POST) @Operation(summary = "add a new Datasource", description = "add a new Datasource", tags = { DS, W }) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Malformed request"), @ApiResponse(responseCode = "500", description = "Unexpected error") }) public void saveDs(@Valid @RequestBody final DatasourceDetails datasource) throws DsmException { if (dsmCore.exist(datasource)) { // TODO further check that the DS doesn't have any API throw new DsmException(HttpStatus.SC_CONFLICT, String.format("cannot register, datasource already defined '%s'", datasource.getId())); } dsmCore.save(datasource); } @RequestMapping(value = "/ds/addWithApis", method = RequestMethod.POST) @Operation(summary = "add a new Datasource and its apis", description = "add a new Datasource and its apis", tags = { DS, W }) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "400", description = "Malformed request"), @ApiResponse(responseCode = "500", description = "Unexpected error") }) public void saveDsWithApis(@Valid @RequestBody final DatasourceDetailsWithApis d) throws DsmException { if (d.getDatasource() == null) { throw new DsmException(HttpStatus.SC_BAD_REQUEST, "Datasource field is null"); } if (dsmCore.exist(d.getDatasource())) { // TODO further check that the DS doesn't have any API throw new DsmException(HttpStatus.SC_CONFLICT, String.format("cannot register, datasource already defined '%s'", d.getDatasource().getId())); } dsmCore.save(d); } @RequestMapping(value = "/ds/update", method = RequestMethod.POST) @Operation(summary = "update Datasource details", description = "update Datasource details", tags = { DS, W }) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) public void updateDatasource( @RequestBody final DatasourceDetailsUpdate ds) throws DsmException, DsmNotFoundException { dsmCore.updateDatasource(ds); } @RequestMapping(value = "/ds/api/baseurl", method = RequestMethod.POST) @Operation(summary = "update the base URL of a datasource interface", description = "update the base URL of a datasource interface", tags = { API, W }) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) public void updateBaseUrl( @RequestParam final String dsId, @RequestParam final String apiId, @RequestParam final String baseUrl) throws DsmException { dsmCore.updateApiBaseurl(dsId, apiId, baseUrl); } @RequestMapping(value = "/ds/api/compliance", method = RequestMethod.POST) @Operation(summary = "update the compatibility of a datasource interface", description = "update the compatibility of a datasource interface", tags = { API, W }) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) public void updateCompliance( @RequestParam final String dsId, @RequestParam final String apiId, @RequestParam final String compliance, @RequestParam(required = false, defaultValue = "false") final boolean override) throws DsmException { dsmCore.updateApiCompatibility(dsId, apiId, compliance, override); } @RequestMapping(value = "/ds/api/oaiset", method = RequestMethod.POST) @Operation(summary = "update the OAI set of a datasource interface", description = "update the OAI set of a datasource interface", tags = { API, W }) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) public void updateOaiSetl( @RequestParam final String dsId, @RequestParam final String apiId, @RequestParam final String oaiSet) throws DsmException { dsmCore.updateApiOaiSet(dsId, apiId, oaiSet); } @RequestMapping(value = "/ds/api/add", method = RequestMethod.POST) @Operation(summary = "adds a new Interface to one Datasource", description = "adds an Interface to one Datasource", tags = { API, W }) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) public void addApi(@RequestBody final ApiDetails api) throws DsmException { if (StringUtils.isBlank(api.getDatasource())) { throw new DsmException(HttpStatus.SC_BAD_REQUEST, "missing datasource id"); } dsmCore.addApi(api); } // MANAGEMENT @Autowired private OperationManager operationManager; @RequestMapping(value = "/dsm/ops", method = RequestMethod.GET) @Operation(summary = "get the number of pending operations", description = "get the number of pending operations", tags = { R, M }) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) public int getOps() throws DsmException { return operationManager.getOpSize(); } @RequestMapping(value = "/dsm/killops", method = RequestMethod.POST) @Operation(summary = "interrupts the pending operations", description = "return the number of interrupted operations", tags = { W, M }) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) public int killOps() throws DsmException { return operationManager.dropAll(); } @RequestMapping(value = "/dsm/dropcache", method = RequestMethod.POST) @Operation(summary = "drop the caches", description = "drop the internal caches", tags = { W, M }) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) public void dropCache() throws DsmException { dsmCore.dropCaches(); } // HELPERS private T prepareResponse(final int page, final int size, final StopWatch stopWatch, final T rsp) { rsp.getHeader() .setTime(stopWatch.getTime()) .setPage(page) .setSize(size); return rsp; } // ------------------------------ @RequestMapping(value = "/ds/recentregistered/v2/{size}", produces = { "application/json" }, method = RequestMethod.GET) @Operation(summary = "return the latest datasources that were registered through Provide (v2)", description = "Returns list of Datasource basic info.", tags = { DS, R }) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) public SimpleResponse recentRegisteredV2(@PathVariable final int size) throws Throwable { final StopWatch stop = StopWatch.createStarted(); final SimpleResponse rsp = dsmCore.searchRecentRegisteredV2(size); return prepareResponse(1, size, stop, rsp); } @RequestMapping(value = "/ds/countfirstcollect", produces = { "application/json" }, method = RequestMethod.GET) @Operation(summary = "return the number of datasources registered after the given date", description = "Returns a number.", tags = { DS, R }) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) public Long countFirstCollectAfter(@RequestParam final String fromDate, @RequestParam(required = false) final String typologyFilter) throws Throwable { return dsmCore.countFirstCollect(fromDate, typologyFilter); } @RequestMapping(value = "/ds/firstCollected", produces = { "application/json" }, method = RequestMethod.GET) @Operation(summary = "return the datasources that were collected for the first time after the specified date", description = "Returns list of Datasource basic info.", tags = { DS, R }) @ApiResponses(value = { @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "500", description = "unexpected error") }) public SimpleResponse firstCollectedAfter(@RequestParam final String fromDate, @RequestParam(required = false) final String typologyFilter) throws Throwable { final StopWatch stop = StopWatch.createStarted(); final List list = dsmCore.getFirstCollectedAfter(fromDate, typologyFilter); final SimpleResponse rsp = ResponseUtils.simpleResponse(list); return prepareResponse(1, list.size(), stop, rsp); } }